feat(admin): add audit target APIs and harden target source handling (#2350)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

Cargo.lock

@@ -7703,6 +7703,7 @@ dependencies = [
  "rustfs-targets",
  "serde",
  "serde_json",
+ "temp-env",
  "thiserror 2.0.18",
  "tokio",
  "tracing",
@@ -7897,6 +7898,7 @@ dependencies = [
  "rustfs-utils",
  "s3s",
  "serde",
+ "tempfile",
  "thiserror 2.0.18",
  "time",
  "tokio",
@@ -8396,6 +8398,7 @@ name = "rustfs-targets"
 version = "0.0.5"
 dependencies = [
  "async-trait",
+ "criterion",
  "reqwest 0.13.2",
  "rumqttc",
  "rustfs-config",

crates/audit/Cargo.toml

@@ -44,6 +44,8 @@ tracing = { workspace = true, features = ["std", "attributes"] }
 url = { workspace = true }
 rumqttc = { workspace = true }
 
+[dev-dependencies]
+temp-env = { workspace = true }
 
 [lints]
 workspace = true

crates/audit/src/registry.rs

@@ -109,9 +109,6 @@ impl AuditRegistry {
         let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
         // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // let final_config = config.clone(); // Clone a configuration for aggregating the final result
-        // Record the defaults for each segment so that the segment can eventually be rebuilt
-        let mut section_defaults: HashMap<String, KVS> = HashMap::new();
         // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
@@ -125,9 +122,6 @@ impl AuditRegistry {
             let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
             debug!(?default_cfg, "Get the default configuration");
 
-            // Save defaults for eventual write back
-            section_defaults.insert(section_name.clone(), default_cfg.clone());
-
             // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
             debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -235,103 +229,28 @@ impl AuditRegistry {
                 if enabled {
                     info!(instance_id = %id, "Target is enabled, ready to create a task");
                     // 5.3. Create asynchronous tasks for enabled instances
-                    let target_type_clone = target_type.clone();
                     let tid = id.clone();
                     let merged_config_arc = Arc::new(merged_config);
                     tasks.push(async move {
                         let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (target_type_clone, tid, result, Arc::clone(&merged_config_arc))
+                        (tid, result)
                     });
                 } else {
-                    info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
+                    info!(instance_id = %id, "Skip disabled target");
-                    // Remove disabled target from final configuration
-                    // final_config.0.entry(section_name.clone()).or_default().remove(&id);
                 }
             }
         }
 
         // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
-        let mut successful_configs = Vec::new();
+        while let Some((id, result)) = tasks.next().await {
-        while let Some((target_type, id, result, final_config)) = tasks.next().await {
             match result {
                 Ok(target) => {
-                    info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
+                    info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
                     successful_targets.push(target);
-                    successful_configs.push((target_type, id, final_config));
                 }
                 Err(e) => {
-                    error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
+                    error!(instance_id = %id, error = %e, "Failed to create a target");
-                }
-            }
-        }
-
-        // 7. Aggregate new configuration and write back to system configuration
-        if !successful_configs.is_empty() || !section_defaults.is_empty() {
-            info!(
-                "Prepare to update {} successfully created target configurations to the system configuration...",
-                successful_configs.len()
-            );
-
-            let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
-
-            for (target_type, id, kvs) in successful_configs {
-                let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
-                successes_by_section
-                    .entry(section_name)
-                    .or_default()
-                    .insert(id.to_lowercase(), (*kvs).clone());
-            }
-
-            let mut new_config = config.clone();
-            // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
-            let mut sections: HashSet<String> = HashSet::new();
-            sections.extend(section_defaults.keys().cloned());
-            sections.extend(successes_by_section.keys().cloned());
-
-            for section in sections {
-                let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
-                // Add default item
-                if let Some(default_kvs) = section_defaults.get(&section)
-                    && !default_kvs.is_empty()
-                {
-                    section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
-                }
-
-                // Add successful instance item
-                if let Some(instances) = successes_by_section.get(&section) {
-                    for (id, kvs) in instances {
-                        section_map.insert(id.clone(), kvs.clone());
-                    }
-                }
-
-                // Empty breaks are removed and non-empty breaks are replaced entirely.
-                if section_map.is_empty() {
-                    new_config.0.remove(&section);
-                } else {
-                    new_config.0.insert(section, section_map);
-                }
-            }
-
-            if &new_config == config {
-                info!("Audit target configuration unchanged, skip persisting server config");
-                info!(count = successful_targets.len(), "All target processing completed");
-                return Ok(successful_targets);
-            }
-
-            let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
-                return Err(AuditError::StorageNotAvailable(
-                    "Failed to save target configuration: server storage not initialized".to_string(),
-                ));
-            };
-
-            match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
-                Ok(_) => {
-                    info!("The new configuration was saved to the system successfully.")
-                }
-                Err(e) => {
-                    error!("Failed to save the new configuration: {}", e);
-                    return Err(AuditError::SaveConfig(Box::new(e)));
                 }
             }
         }
@@ -371,6 +290,11 @@ impl AuditRegistry {
         self.targets.get(id).map(|t| t.as_ref())
     }
 
+    /// Lists cloned target values for runtime inspection without exposing mutable registry access.
+    pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        self.targets.values().map(|target| target.clone_dyn()).collect()
+    }
+
     /// Lists all target IDs
     ///
     /// # Returns

crates/audit/src/system.rs

@@ -13,14 +13,15 @@
 //  limitations under the License.
 
 use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
+use hashbrown::HashMap;
 use rustfs_ecstore::config::Config;
 use rustfs_targets::{
     StoreError, Target, TargetError,
     store::{Key, Store},
-    target::EntityTarget,
+    target::{EntityTarget, QueuedPayload},
 };
 use std::sync::Arc;
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{Mutex, RwLock, mpsc};
 use tracing::{error, info, warn};
 
 /// State of the audit system
@@ -39,6 +40,8 @@ pub struct AuditSystem {
     registry: Arc<Mutex<AuditRegistry>>,
     state: Arc<RwLock<AuditSystemState>>,
     config: Arc<RwLock<Option<Config>>>,
+    /// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
+    stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
 }
 
 impl Default for AuditSystem {
@@ -54,6 +57,7 @@ impl AuditSystem {
             registry: Arc::new(Mutex::new(AuditRegistry::new())),
             state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
             config: Arc::new(RwLock::new(None)),
+            stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
         }
     }
 
@@ -110,27 +114,7 @@ impl AuditSystem {
 
                 // Initialize all targets
                 for target in targets {
-                    let target_id = target.id().to_string();
+                    self.init_and_register_target(target, &mut registry).await;
-                    if let Err(e) = target.init().await {
-                        error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
-                    } else {
-                        // After successful initialization, if enabled and there is a store, start the send from storage task
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
-                        }
-                        registry.add_target(target_id, target);
-                    }
                 }
 
                 // Update state to running
@@ -214,6 +198,9 @@ impl AuditSystem {
 
         info!("Stopping audit system");
 
+        // Stop all stream tasks first
+        self.stop_all_streams().await;
+
         // Close all targets
         let mut registry = self.registry.lock().await;
         if let Err(e) = registry.close_all().await {
@@ -258,52 +245,49 @@ impl AuditSystem {
         let state = self.state.read().await;
 
         match *state {
-            AuditSystemState::Running => {
+            AuditSystemState::Running => {}
-                // Continue with dispatch
-                info!("Dispatching audit log entry");
-            }
             AuditSystemState::Paused => {
-                // Skip dispatch when paused
                 return Ok(());
             }
             _ => {
-                // Don't dispatch when not running
                 return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
             }
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
+        // Collect cloned targets under lock, then dispatch without holding it
-        let target_keys = registry.list_targets();
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
+            if target_keys.is_empty() {
-            warn!("No audit targets configured for dispatch");
+                warn!("No audit targets configured for dispatch");
-            return Ok(());
+                return Ok(());
-        }
+            }
 
-        // Dispatch to all targets concurrently
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
+
+        // Dispatch to all targets concurrently (no lock held)
         let mut tasks = Vec::new();
 
-        for target_key in target_keys {
+        for (target_key, target) in targets {
-            if let Some(target) = registry.get_target(&target_key) {
+            let entity_target = EntityTarget {
-                let entry_clone = Arc::clone(&entry);
+                object_name: entry.api.name.clone().unwrap_or_default(),
-                let target_key_clone = target_key.clone();
+                bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                event_name: entry.event,
+                data: (*entry).clone(),
+            };
 
-                // Create EntityTarget for the audit log entry
+            let task = async move {
-                let entity_target = EntityTarget {
+                let result = target.save(Arc::new(entity_target)).await;
-                    object_name: entry.api.name.clone().unwrap_or_default(),
+                (target_key, result)
-                    bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+            };
-                    event_name: entry.event, // Default, should be derived from entry
-                    data: (*entry_clone).clone(),
-                };
 
-                let task = async move {
+            tasks.push(task);
-                    let result = target.save(Arc::new(entity_target)).await;
-                    (target_key_clone, result)
-                };
-
-                tasks.push(task);
-            }
         }
 
         // Execute all dispatch tasks
@@ -359,39 +343,45 @@ impl AuditSystem {
         }
         drop(state);
 
-        let registry = self.registry.lock().await;
+        // Collect targets under lock, then dispatch without holding it
-        let target_keys = registry.list_targets();
+        let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
+            let registry = self.registry.lock().await;
+            let target_keys = registry.list_targets();
 
-        if target_keys.is_empty() {
+            if target_keys.is_empty() {
-            warn!("No audit targets configured for batch dispatch");
+                warn!("No audit targets configured for batch dispatch");
-            return Ok(());
+                return Ok(());
-        }
+            }
+
+            target_keys
+                .into_iter()
+                .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
+                .collect()
+        };
 
         let mut tasks = Vec::new();
-        for target_key in target_keys {
+        for (target_key, target) in targets {
-            if let Some(target) = registry.get_target(&target_key) {
+            let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
-                let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
+            let target_key_clone = target_key.clone();
-                let target_key_clone = target_key.clone();
 
-                let task = async move {
+            let task = async move {
-                    let mut success_count = 0;
+                let mut success_count = 0;
-                    let mut errors = Vec::new();
+                let mut errors = Vec::new();
-                    for entry in entries_clone {
+                for entry in entries_clone {
-                        let entity_target = EntityTarget {
+                    let entity_target = EntityTarget {
-                            object_name: entry.api.name.clone().unwrap_or_default(),
+                        object_name: entry.api.name.clone().unwrap_or_default(),
-                            bucket_name: entry.api.bucket.clone().unwrap_or_default(),
+                        bucket_name: entry.api.bucket.clone().unwrap_or_default(),
-                            event_name: entry.event,
+                        event_name: entry.event,
-                            data: (*entry).clone(),
+                        data: (*entry).clone(),
-                        };
+                    };
-                        match target.save(Arc::new(entity_target)).await {
+                    match target.save(Arc::new(entity_target)).await {
-                            Ok(_) => success_count += 1,
+                        Ok(_) => success_count += 1,
-                            Err(e) => errors.push(e),
+                        Err(e) => errors.push(e),
-                        }
                     }
-                    (target_key_clone, success_count, errors)
+                }
-                };
+                (target_key_clone, success_count, errors)
-                tasks.push(task);
+            };
-            }
+            tasks.push(task);
         }
 
         let results = futures::future::join_all(tasks).await;
@@ -417,6 +407,60 @@ impl AuditSystem {
         Ok(())
     }
 
+    /// Stops all active audit stream tasks by sending cancellation signals.
+    async fn stop_all_streams(&self) {
+        let mut cancellers = self.stream_cancellers.write().await;
+        for (target_id, cancel_tx) in cancellers.drain() {
+            info!(target_id = %target_id, "Stopping audit stream");
+            let _ = cancel_tx.send(()).await;
+        }
+    }
+
+    /// Initializes a single target: runs init(), starts stream if store is present,
+    /// and adds it to the registry. For store-backed targets, registration and stream
+    /// startup proceed even if init() fails so queued entries can be drained later.
+    async fn init_and_register_target(
+        &self,
+        target: Box<dyn Target<AuditEntry> + Send + Sync>,
+        registry: &mut AuditRegistry,
+    ) -> Option<String> {
+        let target_id = target.id().to_string();
+        let has_store = target.store().is_some();
+
+        if let Err(e) = target.init().await {
+            error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
+            // Non-store targets: init failure is fatal.
+            if !has_store {
+                return None;
+            }
+            // Store-backed targets: still register and start the stream so queued
+            // entries can be drained when connectivity recovers.
+            warn!(
+                target_id = %target_id,
+                "Proceeding with store-backed audit target despite init failure"
+            );
+        }
+
+        if target.is_enabled() {
+            if let Some(store) = target.store() {
+                info!(target_id = %target_id, "Start audit stream processing for target");
+                let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
+                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
+                let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
+
+                self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
+                info!(target_id = %target_id, "Audit stream processing started");
+            } else {
+                info!(target_id = %target_id, "No store configured, skip audit stream processing");
+            }
+        } else {
+            info!(target_id = %target_id, "Target disabled, skip audit stream processing");
+        }
+
+        registry.add_target(target_id.clone(), target);
+        Some(target_id)
+    }
+
     /// Starts the audit stream processing for a target with batching and retry logic
     ///
     /// # Arguments
@@ -427,9 +471,10 @@ impl AuditSystem {
     /// and attempts to send them to the specified target. It implements retry logic with exponential backoff
     fn start_audit_stream_with_batching(
         &self,
-        store: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send>,
+        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
         target: Arc<dyn Target<AuditEntry> + Send + Sync>,
-    ) {
+    ) -> mpsc::Sender<()> {
+        let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
         let state = self.state.clone();
 
         tokio::spawn(async move {
@@ -442,6 +487,12 @@ impl AuditSystem {
             const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
 
             loop {
+                // Check for cancellation signal
+                if cancel_rx.try_recv().is_ok() {
+                    info!("Audit stream cancelled for target: {}", target.id());
+                    break;
+                }
+
                 match *state.read().await {
                     AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
                     _ => {
@@ -452,11 +503,22 @@ impl AuditSystem {
 
                 let keys: Vec<Key> = store.list();
                 if keys.is_empty() {
-                    sleep(Duration::from_millis(500)).await;
+                    tokio::select! {
+                        _ = sleep(Duration::from_millis(500)) => {},
+                        _ = cancel_rx.recv() => {
+                            info!("Audit stream cancelled during idle for target: {}", target.id());
+                            return;
+                        }
+                    }
                     continue;
                 }
 
                 for key in keys {
+                    if cancel_rx.try_recv().is_ok() {
+                        info!("Audit stream cancelled during processing for target: {}", target.id());
+                        return;
+                    }
+
                     let mut retries = 0usize;
                     let mut success = false;
 
@@ -497,6 +559,8 @@ impl AuditSystem {
                 sleep(Duration::from_millis(100)).await;
             }
         });
+
+        cancel_tx
     }
 
     /// Enables a specific target
@@ -594,6 +658,12 @@ impl AuditSystem {
         registry.list_targets()
     }
 
+    /// Returns cloned target values for read-only runtime inspection.
+    pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
+        let registry = self.registry.lock().await;
+        registry.list_target_values()
+    }
+
     /// Gets information about a specific target
     ///
     /// # Arguments
@@ -616,9 +686,11 @@ impl AuditSystem {
     pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> {
         info!("Reloading audit system configuration");
 
-        // Record config reload
         observability::record_config_reload();
 
+        // Stop all existing stream tasks first
+        self.stop_all_streams().await;
+
         // Store new configuration
         {
             let mut config_guard = self.config.write().await;
@@ -636,28 +708,9 @@ impl AuditSystem {
             Ok(targets) => {
                 info!(target_count = targets.len(), "Reloaded audit targets successfully");
 
-                // Initialize all new targets
                 for target in targets {
-                    let target_id = target.id().to_string();
+                    if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
-                    if let Err(e) = target.init().await {
+                        info!(target_id = %target_id, "Target initialized (reload)");
-                        error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target");
-                    } else {
-                        // Same starts the storage stream after a heavy load
-                        if target.is_enabled() {
-                            if let Some(store) = target.store() {
-                                info!(target_id = %target_id, "Start audit stream processing for target (reload)");
-                                let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
-                                    store.boxed_clone();
-                                let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
-                                self.start_audit_stream_with_batching(store_clone, target_arc);
-                                info!(target_id = %target_id, "Audit stream processing started (reload)");
-                            } else {
-                                info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)");
-                            }
-                        } else {
-                            info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)");
-                        }
-                        registry.add_target(target.id().to_string(), target);
                     }
                 }
 

crates/audit/tests/integration_test.rs

@@ -15,6 +15,7 @@
 use rustfs_audit::*;
 use rustfs_ecstore::config::{Config, KVS};
 use std::collections::HashMap;
+use temp_env::with_vars;
 
 #[tokio::test]
 async fn test_audit_system_creation() {
@@ -35,34 +36,42 @@ async fn test_config_parsing_webhook() {
     let mut config = Config(HashMap::new());
     let mut audit_webhook_section = HashMap::new();
 
-    // Create default configuration
     let mut default_kvs = KVS::new();
-    default_kvs.insert("enable".to_string(), "on".to_string());
+    default_kvs.insert("enable".to_string(), "off".to_string());
-    default_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
+    default_kvs.insert("endpoint".to_string(), "".to_string());
-
     audit_webhook_section.insert("_".to_string(), default_kvs);
+    let mut instance_kvs = KVS::new();
+    instance_kvs.insert("enable".to_string(), "on".to_string());
+    instance_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
+    audit_webhook_section.insert("primary".to_string(), instance_kvs);
     config.0.insert("audit_webhook".to_string(), audit_webhook_section);
 
     let registry = AuditRegistry::new();
 
-    // This should not fail even if server storage is not initialized
-    // as it's an integration test
     let result = registry.create_audit_targets_from_config(&config).await;
+    assert!(result.is_ok(), "audit target creation should not require server storage");
+}
 
-    // We expect this to fail due to server storage not being initialized
+#[test]
-    // but the parsing should work correctly
+fn test_env_only_audit_target_does_not_require_server_storage() {
-    match result {
+    with_vars(
-        Err(AuditError::StorageNotAvailable(_)) => {
+        [
-            // This is expected in test environment
+            ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
-        }
+            ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("http://localhost:3020/webhook")),
-        Err(e) => {
+        ],
-            // Other errors might indicate parsing issues
+        || {
-            println!("Unexpected error: {e}");
+            let runtime = tokio::runtime::Builder::new_current_thread()
-        }
+                .enable_all()
-        Ok(_) => {
+                .build()
-            // Unexpected success in test environment without server storage
+                .expect("failed to create tokio runtime");
-        }
+            runtime.block_on(async {
-    }
+                let config = Config(HashMap::new());
+                let registry = AuditRegistry::new();
+                let result = registry.create_audit_targets_from_config(&config).await;
+                assert!(result.is_ok(), "env-only audit target creation should not require server storage");
+            });
+        },
+    )
 }
 
 #[test]

crates/config/src/constants/targets.rs

@@ -34,3 +34,10 @@ pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
 pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
 pub const MQTT_QUEUE_DIR: &str = "queue_dir";
 pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
+
+/// Environment variable controlling whether target queue files are Snappy-compressed.
+/// Applies to both notify and audit target queue stores.
+pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
+
+/// Queue-store compression is enabled by default to reduce disk footprint.
+pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;

crates/e2e_test/src/object_lambda_test.rs

@@ -313,6 +313,31 @@ async fn list_target_arns(env: &RustFSTestEnvironment) -> Result<Vec<String>, Bo
     Ok(serde_json::from_slice(&body)?)
 }
 
+async fn delete_webhook_target(env: &RustFSTestEnvironment, target_name: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+    let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}/reset", env.url);
+    let response = signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await?;
+    let status = response.status();
+    let body = response.text().await.unwrap_or_default();
+    if status != StatusCode::OK {
+        return Err(format!("failed to delete webhook target {target_name}: {status} {body}").into());
+    }
+
+    Ok(())
+}
+
+fn notification_target_is_listed(targets: &serde_json::Value, target_name: &str) -> bool {
+    targets["notification_endpoints"]
+        .as_array()
+        .into_iter()
+        .flatten()
+        .any(|entry| {
+            entry["account_id"].as_str() == Some(target_name)
+                && entry["service"]
+                    .as_str()
+                    .is_some_and(|service| service == "webhook" || service.starts_with("webhook-"))
+        })
+}
+
 async fn wait_for_target_visibility(
     env: &RustFSTestEnvironment,
     target_name: &str,
@@ -324,18 +349,7 @@ async fn wait_for_target_visibility(
         last_targets = list_notification_targets(env).await?;
         last_arns = list_target_arns(env).await?;
 
-        let listed = last_targets["notification_endpoints"]
+        if notification_target_is_listed(&last_targets, target_name) {
-            .as_array()
-            .into_iter()
-            .flatten()
-            .any(|entry| {
-                entry["account_id"].as_str() == Some(target_name)
-                    && entry["service"]
-                        .as_str()
-                        .is_some_and(|service| service == "webhook" || service.starts_with("webhook-"))
-            });
-
-        if listed {
             return Ok((last_targets, last_arns));
         }
 
@@ -345,10 +359,51 @@ async fn wait_for_target_visibility(
     Err(format!("target {target_name} did not become visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
 }
 
+async fn wait_for_target_absence(
+    env: &RustFSTestEnvironment,
+    target_name: &str,
+) -> Result<(serde_json::Value, Vec<String>), Box<dyn Error + Send + Sync>> {
+    let mut last_targets = serde_json::Value::Null;
+    let mut last_arns = Vec::new();
+
+    for _ in 0..20 {
+        last_targets = list_notification_targets(env).await?;
+        last_arns = list_target_arns(env).await?;
+
+        let listed = notification_target_is_listed(&last_targets, target_name);
+        let arn_listed = last_arns.iter().any(|arn| arn.ends_with(&format!(":{target_name}:webhook")));
+        if !listed && !arn_listed {
+            return Ok((last_targets, last_arns));
+        }
+
+        tokio::time::sleep(Duration::from_millis(250)).await;
+    }
+
+    Err(format!("target {target_name} remained visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
+}
+
+async fn restart_rustfs_server(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn Error + Send + Sync>> {
+    env.stop_server();
+    env.start_rustfs_server_without_cleanup(vec![]).await
+}
+
 async fn read_persisted_server_config(env: &RustFSTestEnvironment) -> String {
     let path = format!("{}/.rustfs.sys/config/config.json", env.temp_dir);
     match tokio::fs::read_to_string(&path).await {
         Ok(content) => content,
+        Err(err) if err.kind() == std::io::ErrorKind::IsADirectory => {
+            let mut entries = Vec::new();
+            match tokio::fs::read_dir(&path).await {
+                Ok(mut dir) => {
+                    while let Ok(Some(entry)) = dir.next_entry().await {
+                        entries.push(entry.file_name().to_string_lossy().to_string());
+                    }
+                    entries.sort();
+                    format!("persisted config stored as object directory at {path}; entries={entries:?}")
+                }
+                Err(dir_err) => format!("persisted config directory exists at {path} but could not be listed: {dir_err}"),
+            }
+        }
         Err(err) => format!("failed to read persisted config at {path}: {err}"),
     }
 }
@@ -400,6 +455,66 @@ async fn read_listen_notification_event(
     }
 }
 
+#[tokio::test]
+#[serial]
+async fn test_notification_target_persists_across_restart_and_delete() -> Result<(), Box<dyn Error + Send + Sync>> {
+    init_logging();
+
+    let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
+
+    let mut env = RustFSTestEnvironment::new().await?;
+    env.start_rustfs_server(vec![]).await?;
+
+    let target_name = "restart-target";
+    configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
+
+    let (visible_targets, visible_arns) = wait_for_target_visibility(&env, target_name).await?;
+    assert!(notification_target_is_listed(&visible_targets, target_name));
+    assert!(
+        visible_arns
+            .iter()
+            .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
+        "target ARN missing after initial configure: {visible_arns:?}"
+    );
+
+    restart_rustfs_server(&mut env).await?;
+
+    let (targets_after_restart, arns_after_restart) = wait_for_target_visibility(&env, target_name).await?;
+    assert!(notification_target_is_listed(&targets_after_restart, target_name));
+    assert!(
+        arns_after_restart
+            .iter()
+            .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
+        "target ARN missing after restart: {arns_after_restart:?}"
+    );
+
+    delete_webhook_target(&env, target_name).await?;
+    let (targets_after_delete, arns_after_delete) = wait_for_target_absence(&env, target_name).await?;
+    assert!(!notification_target_is_listed(&targets_after_delete, target_name));
+    assert!(
+        !arns_after_delete
+            .iter()
+            .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
+        "target ARN still visible after delete: {arns_after_delete:?}"
+    );
+
+    restart_rustfs_server(&mut env).await?;
+
+    let (targets_after_delete_restart, arns_after_delete_restart) = wait_for_target_absence(&env, target_name).await?;
+    assert!(!notification_target_is_listed(&targets_after_delete_restart, target_name));
+    assert!(
+        !arns_after_delete_restart
+            .iter()
+            .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
+        "target ARN still visible after delete + restart: {arns_after_delete_restart:?}"
+    );
+
+    webhook_handle.abort();
+    let _ = webhook_handle.await;
+
+    Ok(())
+}
+
 #[tokio::test]
 #[serial]
 async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box<dyn Error + Send + Sync>> {

crates/ecstore/src/config/audit.rs

@@ -16,8 +16,8 @@ use crate::config::{KV, KVS};
 use rustfs_config::{
     COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
     MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY,
+    WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
-    WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
+    WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -51,6 +51,16 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: "".to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: WEBHOOK_CLIENT_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
         KV {
             key: WEBHOOK_BATCH_SIZE.to_owned(),
             value: "1".to_owned(),
@@ -81,6 +91,11 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: "5s".to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: COMMENT_KEY.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
     ])
 });
 

crates/ecstore/src/config/com.rs

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, oidc, storageclass};
+use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, storageclass};
 use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::global::is_first_cluster_node_local;
 use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
 use http::HeaderMap;
+use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
+use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
 use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
 use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
 use rustfs_utils::path::SLASH_SEPARATOR;
@@ -261,6 +263,159 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
     applied
 }
 
+fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
+    match value {
+        Value::String(v) => Some(v.trim().to_string()),
+        Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
+            EnableState::On.to_string()
+        } else {
+            EnableState::Off.to_string()
+        }),
+        Value::Bool(v) => Some(v.to_string()),
+        Value::Number(v) => Some(v.to_string()),
+        Value::Null => None,
+        _ => None,
+    }
+}
+
+fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
+    let mut kvs = KVS::new();
+
+    for (key, value) in instance {
+        if !valid_keys.contains(&key.as_str()) || key == COMMENT_KEY {
+            continue;
+        }
+
+        if let Some(parsed) = parse_notify_scalar_value(key, value) {
+            kvs.insert(key.clone(), parsed);
+        }
+    }
+
+    kvs
+}
+
+fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
+    match value {
+        Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
+        Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
+        _ => None,
+    }
+}
+
+fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
+    section
+        .iter()
+        .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
+}
+
+fn apply_external_notify_section(
+    cfg: &mut Config,
+    notify_obj: &Map<String, Value>,
+    external_key: &str,
+    subsystem_key: &str,
+    default_kvs: &KVS,
+    valid_keys: &[&str],
+) -> bool {
+    let Some(Value::Object(section_obj)) = notify_obj.get(external_key).or_else(|| notify_obj.get(subsystem_key)) else {
+        return false;
+    };
+
+    if section_obj.is_empty() {
+        return false;
+    }
+
+    let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
+    let mut applied = false;
+
+    if is_notify_instance_shorthand(section_obj, valid_keys) {
+        let kvs = decode_notify_instance_object(section_obj, valid_keys);
+        if !kvs.is_empty() {
+            let mut merged = default_kvs.clone();
+            merged.extend(kvs);
+            subsystem.insert(DEFAULT_DELIMITER.to_string(), merged);
+            applied = true;
+        }
+        return applied;
+    }
+
+    for (raw_instance, value) in section_obj {
+        let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
+            continue;
+        };
+        if kvs.is_empty() {
+            continue;
+        }
+
+        let instance_key = if raw_instance == "default" {
+            DEFAULT_DELIMITER.to_string()
+        } else {
+            raw_instance.to_string()
+        };
+
+        if instance_key == DEFAULT_DELIMITER {
+            let mut merged = default_kvs.clone();
+            merged.extend(kvs);
+            kvs = merged;
+        }
+
+        subsystem.insert(instance_key, kvs);
+        applied = true;
+    }
+
+    applied
+}
+
+fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
+    let Some(Value::Object(notify_obj)) = root.get("notify") else {
+        return false;
+    };
+
+    let mut applied = false;
+    applied |= apply_external_notify_section(
+        cfg,
+        notify_obj,
+        "webhook",
+        NOTIFY_WEBHOOK_SUB_SYS,
+        &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
+        NOTIFY_WEBHOOK_KEYS,
+    );
+    applied |= apply_external_notify_section(
+        cfg,
+        notify_obj,
+        "mqtt",
+        NOTIFY_MQTT_SUB_SYS,
+        &notify::DEFAULT_NOTIFY_MQTT_KVS,
+        NOTIFY_MQTT_KEYS,
+    );
+    applied
+}
+
+fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
+    let audit_root = root.get("audit").or_else(|| root.get("logger")).and_then(Value::as_object);
+    let Some(audit_obj) = audit_root else {
+        return false;
+    };
+
+    let mut applied = false;
+    applied |= apply_external_notify_section(
+        cfg,
+        audit_obj,
+        "webhook",
+        AUDIT_WEBHOOK_SUB_SYS,
+        &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
+        AUDIT_WEBHOOK_KEYS,
+    );
+    applied |= apply_external_notify_section(
+        cfg,
+        audit_obj,
+        "mqtt",
+        AUDIT_MQTT_SUB_SYS,
+        &audit::DEFAULT_AUDIT_MQTT_KVS,
+        AUDIT_MQTT_KEYS,
+    );
+    applied
+}
+
 fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
     let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
     let Some(Value::Object(sc_obj)) = sc else {
@@ -305,8 +460,10 @@ fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
     let mut cfg = Config::new();
     let has_storage = apply_external_storage_class_map(&mut cfg, &root);
     let has_oidc = apply_external_oidc_map(&mut cfg, &root);
+    let has_notify = apply_external_notify_map(&mut cfg, &root);
+    let has_audit = apply_external_audit_map(&mut cfg, &root);
     let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
-    if !has_storage && !has_oidc && !has_header {
+    if !has_storage && !has_oidc && !has_notify && !has_audit && !has_header {
         return Err(Error::other("unrecognized external server config shape"));
     }
     Ok(cfg)
@@ -449,6 +606,139 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
     oidc_obj
 }
 
+fn is_notify_bool_key(key: &str) -> bool {
+    key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
+}
+
+fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
+    if is_notify_bool_key(key) {
+        if let Ok(state) = value.parse::<EnableState>() {
+            return Value::Bool(state.is_enabled());
+        }
+        if let Ok(boolean) = value.parse::<bool>() {
+            return Value::Bool(boolean);
+        }
+    }
+
+    Value::String(value.to_string())
+}
+
+fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
+    default_kvs
+        .0
+        .iter()
+        .find(|kv| kv.key == key)
+        .map(|kv| kv.hidden_if_empty)
+        .unwrap_or(false)
+}
+
+fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
+    let mut instance = Map::new();
+
+    for key in valid_keys {
+        if *key == COMMENT_KEY {
+            continue;
+        }
+
+        let baseline_value = baseline.lookup(key).unwrap_or_default();
+        let effective_value = kvs.lookup(key).unwrap_or_else(|| baseline_value.clone());
+
+        if effective_value == baseline_value {
+            continue;
+        }
+
+        if effective_value.trim().is_empty() && baseline_value.trim().is_empty() {
+            continue;
+        }
+
+        if is_hidden_if_empty(default_kvs, key) && effective_value.trim().is_empty() && baseline_value.trim().is_empty() {
+            continue;
+        }
+
+        instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
+    }
+
+    instance
+}
+
+fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
+    let mut merged = default_kvs.clone();
+    if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
+        merged.extend(kvs.clone());
+    }
+    merged
+}
+
+fn build_notify_subsystem_object(
+    cfg: &Config,
+    subsystem_key: &str,
+    default_kvs: &KVS,
+    valid_keys: &[&str],
+) -> Map<String, Value> {
+    let Some(subsystem) = cfg.0.get(subsystem_key) else {
+        return Map::new();
+    };
+
+    let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
+    let mut subsystem_obj = Map::new();
+
+    if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
+        let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
+        if !default_obj.is_empty() {
+            subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
+        }
+    }
+
+    let mut instances = subsystem
+        .iter()
+        .filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
+        .collect::<Vec<_>>();
+    instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
+
+    for (instance_key, kvs) in instances {
+        let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
+        if !instance_obj.is_empty() {
+            subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
+        }
+    }
+
+    subsystem_obj
+}
+
+fn build_notify_object(cfg: &Config) -> Map<String, Value> {
+    let mut notify_obj = Map::new();
+
+    let webhook_obj =
+        build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, &notify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
+    if !webhook_obj.is_empty() {
+        notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
+    }
+
+    let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, &notify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
+    if !mqtt_obj.is_empty() {
+        notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
+    }
+
+    notify_obj
+}
+
+fn build_audit_object(cfg: &Config) -> Map<String, Value> {
+    let mut audit_obj = Map::new();
+
+    let webhook_obj =
+        build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
+    if !webhook_obj.is_empty() {
+        audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
+    }
+
+    let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
+    if !mqtt_obj.is_empty() {
+        audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
+    }
+
+    audit_obj
+}
+
 fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
     let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
 
@@ -478,6 +768,73 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
         root.remove(IDENTITY_OPENID_SUB_SYS);
     }
 
+    let mut notify_obj = match root.remove("notify") {
+        Some(Value::Object(v)) => v,
+        _ => Map::new(),
+    };
+    let rendered_notify = build_notify_object(cfg);
+    match rendered_notify.get("webhook") {
+        Some(Value::Object(v)) => {
+            notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
+            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
+        }
+        _ => {
+            notify_obj.remove("webhook");
+            notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
+        }
+    }
+    match rendered_notify.get("mqtt") {
+        Some(Value::Object(v)) => {
+            notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
+            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
+        }
+        _ => {
+            notify_obj.remove("mqtt");
+            notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
+        }
+    }
+    if notify_obj.is_empty() {
+        root.remove("notify");
+    } else {
+        root.insert("notify".to_string(), Value::Object(notify_obj));
+    }
+    root.remove(NOTIFY_WEBHOOK_SUB_SYS);
+    root.remove(NOTIFY_MQTT_SUB_SYS);
+
+    let mut logger_obj = match root.remove("logger") {
+        Some(Value::Object(v)) => v,
+        _ => Map::new(),
+    };
+    let rendered_audit = build_audit_object(cfg);
+    match rendered_audit.get("webhook") {
+        Some(Value::Object(v)) => {
+            logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
+            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
+        }
+        _ => {
+            logger_obj.remove("webhook");
+            logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
+        }
+    }
+    match rendered_audit.get("mqtt") {
+        Some(Value::Object(v)) => {
+            logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
+            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
+        }
+        _ => {
+            logger_obj.remove("mqtt");
+            logger_obj.remove(AUDIT_MQTT_SUB_SYS);
+        }
+    }
+    if logger_obj.is_empty() {
+        root.remove("logger");
+    } else {
+        root.insert("logger".to_string(), Value::Object(logger_obj));
+    }
+    root.remove("audit");
+    root.remove(AUDIT_WEBHOOK_SUB_SYS);
+    root.remove(AUDIT_MQTT_SUB_SYS);
+
     Ok(serde_json::to_vec(&Value::Object(root))?)
 }
 
@@ -496,6 +853,8 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
 fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
     build_storageclass_object(lhs) == build_storageclass_object(rhs)
         && build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs)
+        && build_notify_object(lhs) == build_notify_object(rhs)
+        && build_audit_object(lhs) == build_audit_object(rhs)
 }
 
 fn is_object_not_found(err: &Error) -> bool {
@@ -712,7 +1071,7 @@ mod tests {
         configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
         read_config_with_metadata, storage_class_kvs_mut,
     };
-    use crate::config::{Config, oidc};
+    use crate::config::{Config, audit, notify, oidc};
     use crate::disk::endpoint::Endpoint;
     use crate::endpoints::SetupType;
     use crate::error::{Error, Result};
@@ -725,6 +1084,8 @@ mod tests {
         ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
     };
     use http::HeaderMap;
+    use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+    use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
     use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
     use rustfs_filemeta::FileInfo;
@@ -1332,6 +1693,152 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_decode_server_config_reads_notify_targets() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "notify":{
+            "webhook":{
+              "primary":{
+                "enable":true,
+                "endpoint":"https://example.com/hook",
+                "queue_dir":"/tmp/webhook-queue"
+              }
+            },
+            "mqtt":{
+              "default":{
+                "enable":true,
+                "topic":"events"
+              },
+              "analytics":{
+                "enable":true,
+                "broker":"tcp://127.0.0.1:1883",
+                "topic":"events",
+                "queue_dir":""
+              }
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+
+        let webhook = cfg
+            .get_value(NOTIFY_WEBHOOK_SUB_SYS, "primary")
+            .expect("webhook target should be decoded");
+        assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/hook");
+        assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/webhook-queue");
+
+        let mqtt_default = cfg
+            .get_value(NOTIFY_MQTT_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("mqtt default should be decoded");
+        assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "events");
+        assert_eq!(
+            mqtt_default.get(rustfs_config::MQTT_QUEUE_DIR),
+            notify::DEFAULT_NOTIFY_MQTT_KVS.get(rustfs_config::MQTT_QUEUE_DIR)
+        );
+
+        let mqtt = cfg
+            .get_value(NOTIFY_MQTT_SUB_SYS, "analytics")
+            .expect("mqtt target should be decoded");
+        assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+        assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
+    }
+
+    #[test]
+    fn test_decode_server_config_reads_notify_shorthand_default() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "notify":{
+            "webhook":{
+              "enable":true,
+              "endpoint":"https://example.com/shorthand"
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+        let webhook_default = cfg
+            .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("default webhook config should be decoded");
+        assert_eq!(webhook_default.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(webhook_default.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/shorthand");
+    }
+
+    #[test]
+    fn test_decode_server_config_keeps_instance_named_like_field() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "notify":{
+            "webhook":{
+              "enable":{
+                "enable":true,
+                "endpoint":"https://example.com/instance-enable"
+              }
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+        let named = cfg
+            .get_value(NOTIFY_WEBHOOK_SUB_SYS, "enable")
+            .expect("instance named 'enable' should be decoded");
+        assert_eq!(named.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(named.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/instance-enable");
+    }
+
+    #[test]
+    fn test_decode_server_config_reads_audit_targets() {
+        let input = r#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1"},
+          "logger":{
+            "webhook":{
+              "primary":{
+                "enable":true,
+                "endpoint":"https://example.com/audit-hook",
+                "queue_dir":"/tmp/audit-queue"
+              }
+            },
+            "mqtt":{
+              "default":{
+                "enable":true,
+                "topic":"audit-events"
+              },
+              "analytics":{
+                "enable":true,
+                "broker":"tcp://127.0.0.1:1883",
+                "topic":"audit-events"
+              }
+            }
+          }
+        }"#;
+
+        let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
+
+        let webhook = cfg
+            .get_value(AUDIT_WEBHOOK_SUB_SYS, "primary")
+            .expect("audit webhook target should be decoded");
+        assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/audit-hook");
+        assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/audit-queue");
+
+        let mqtt_default = cfg
+            .get_value(AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER)
+            .expect("audit mqtt default should be decoded");
+        assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string());
+        assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "audit-events");
+
+        let mqtt = cfg
+            .get_value(AUDIT_MQTT_SUB_SYS, "analytics")
+            .expect("audit mqtt target should be decoded");
+        assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
+    }
+
     #[test]
     fn test_encode_server_config_writes_external_object_shape() {
         let mut cfg = Config::new();
@@ -1388,6 +1895,174 @@ mod tests {
         assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
     }
 
+    #[test]
+    fn test_encode_server_config_writes_notify_object_shape() {
+        let mut cfg = Config::new();
+        let mut webhook_section = std::collections::HashMap::new();
+        webhook_section.insert(DEFAULT_DELIMITER.to_string(), notify::DEFAULT_NOTIFY_WEBHOOK_KVS.clone());
+        webhook_section.insert(
+            "primary".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::WEBHOOK_ENDPOINT.to_string(),
+                    value: "https://example.com/hook".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(),
+                    value: "/tmp/webhook-queue".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_section);
+
+        let mut mqtt_default = notify::DEFAULT_NOTIFY_MQTT_KVS.clone();
+        mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "events".to_string());
+        let mut mqtt_section = std::collections::HashMap::new();
+        mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default);
+        mqtt_section.insert(
+            "analytics".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::MQTT_BROKER.to_string(),
+                    value: "tcp://127.0.0.1:1883".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::MQTT_QUEUE_DIR.to_string(),
+                    value: "".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
+
+        let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
+        let v: Value = serde_json::from_slice(&out).expect("output should be json");
+        let notify = v
+            .get("notify")
+            .and_then(Value::as_object)
+            .expect("notify object should be present");
+        let webhook = notify
+            .get("webhook")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("primary"))
+            .and_then(Value::as_object)
+            .expect("webhook target should be encoded");
+        assert_eq!(
+            webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str),
+            Some("https://example.com/hook")
+        );
+        assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+
+        let mqtt_default = notify
+            .get("mqtt")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("default"))
+            .and_then(Value::as_object)
+            .expect("mqtt default should be encoded");
+        assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+        assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("events"));
+
+        let mqtt = notify
+            .get("mqtt")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("analytics"))
+            .and_then(Value::as_object)
+            .expect("mqtt target should be encoded");
+        assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
+        assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
+    }
+
+    #[test]
+    fn test_encode_server_config_writes_audit_object_shape() {
+        let mut cfg = Config::new();
+        let mut webhook_section = std::collections::HashMap::new();
+        webhook_section.insert(DEFAULT_DELIMITER.to_string(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
+        webhook_section.insert(
+            "primary".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::WEBHOOK_ENDPOINT.to_string(),
+                    value: "https://example.com/audit-hook".to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(),
+                    value: "/tmp/audit-queue".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), webhook_section);
+
+        let mut mqtt_default = audit::DEFAULT_AUDIT_MQTT_KVS.clone();
+        mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+        mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "audit-events".to_string());
+        let mut mqtt_section = std::collections::HashMap::new();
+        mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default);
+        mqtt_section.insert(
+            "analytics".to_string(),
+            crate::config::KVS(vec![
+                crate::config::KV {
+                    key: ENABLE_KEY.to_string(),
+                    value: EnableState::On.to_string(),
+                    hidden_if_empty: false,
+                },
+                crate::config::KV {
+                    key: rustfs_config::MQTT_BROKER.to_string(),
+                    value: "tcp://127.0.0.1:1883".to_string(),
+                    hidden_if_empty: false,
+                },
+            ]),
+        );
+        cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
+
+        let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
+        let v: Value = serde_json::from_slice(&out).expect("output should be json");
+        let logger = v
+            .get("logger")
+            .and_then(Value::as_object)
+            .expect("logger object should be present");
+        let webhook = logger
+            .get("webhook")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("primary"))
+            .and_then(Value::as_object)
+            .expect("audit webhook target should be encoded");
+        assert_eq!(
+            webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str),
+            Some("https://example.com/audit-hook")
+        );
+        assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+
+        let mqtt_default = logger
+            .get("mqtt")
+            .and_then(Value::as_object)
+            .and_then(|targets| targets.get("default"))
+            .and_then(Value::as_object)
+            .expect("audit mqtt default should be encoded");
+        assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
+        assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
+    }
+
     #[test]
     fn test_is_standard_object_server_config_detection() {
         let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
@@ -1441,6 +2116,113 @@ mod tests {
         assert!(configs_semantically_equal(&lhs, &rhs));
     }
 
+    #[test]
+    fn test_configs_semantically_equal_accounts_for_notify() {
+        let external = br#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
+          "notify":{
+            "webhook":{
+              "primary":{
+                "enable":true,
+                "endpoint":"https://example.com/hook"
+              }
+            }
+          }
+        }"#;
+        let legacy = br#"{
+          "storage_class":{"_":[
+            {"key":"standard","value":"EC:2"},
+            {"key":"rrs","value":"EC:1"},
+            {"key":"optimize","value":"availability"}
+          ]},
+          "notify_webhook":{
+            "_":[
+              {"key":"enable","value":"off"},
+              {"key":"endpoint","value":""},
+              {"key":"queue_limit","value":"100000"},
+              {"key":"queue_dir","value":"/opt/rustfs/events"},
+              {"key":"client_cert","value":""},
+              {"key":"client_key","value":""},
+              {"key":"comment","value":""},
+              {"key":"client_ca","value":""},
+              {"key":"skip_tls_verify","value":"off"}
+            ],
+            "primary":[
+              {"key":"enable","value":"on"},
+              {"key":"endpoint","value":"https://example.com/hook"}
+            ]
+          }
+        }"#;
+
+        let lhs = decode_server_config_blob(external).expect("decode external");
+        let rhs = decode_server_config_blob(legacy).expect("decode legacy");
+        assert!(configs_semantically_equal(&lhs, &rhs));
+    }
+
+    #[test]
+    fn test_configs_semantically_equal_detects_notify_changes() {
+        let lhs = decode_server_config_blob(
+            br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/a"}}}}"#,
+        )
+        .expect("decode lhs");
+        let rhs = decode_server_config_blob(
+            br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/b"}}}}"#,
+        )
+        .expect("decode rhs");
+
+        assert!(!configs_semantically_equal(&lhs, &rhs));
+    }
+
+    #[test]
+    fn test_configs_semantically_equal_accounts_for_audit() {
+        let external = br#"{
+          "version":"33",
+          "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
+          "logger":{
+            "webhook":{
+              "primary":{
+                "enable":true,
+                "endpoint":"https://example.com/audit-hook"
+              }
+            }
+          }
+        }"#;
+        let legacy = br#"{
+          "storage_class":{"_":[
+            {"key":"standard","value":"EC:2"},
+            {"key":"rrs","value":"EC:1"},
+            {"key":"optimize","value":"availability"}
+          ]},
+          "audit_webhook":{
+            "_":[
+              {"key":"enable","value":"off"},
+              {"key":"endpoint","value":""},
+              {"key":"auth_token","value":""},
+              {"key":"client_cert","value":""},
+              {"key":"client_key","value":""},
+              {"key":"client_ca","value":""},
+              {"key":"skip_tls_verify","value":"off"},
+              {"key":"batch_size","value":"1"},
+              {"key":"queue_limit","value":"100000"},
+              {"key":"queue_dir","value":"/opt/rustfs/events"},
+              {"key":"max_retry","value":"0"},
+              {"key":"retry_interval","value":"3s"},
+              {"key":"http_timeout","value":"5s"},
+              {"key":"comment","value":""}
+            ],
+            "primary":[
+              {"key":"enable","value":"on"},
+              {"key":"endpoint","value":"https://example.com/audit-hook"}
+            ]
+          }
+        }"#;
+
+        let lhs = decode_server_config_blob(external).expect("decode external");
+        let rhs = decode_server_config_blob(legacy).expect("decode legacy");
+        assert!(configs_semantically_equal(&lhs, &rhs));
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     #[serial]
     async fn test_read_config_with_metadata_succeeds_with_one_healthy_locker_in_two_node_dist_setup() {

crates/ecstore/src/config/notify.rs

@@ -16,7 +16,8 @@ use crate::config::{KV, KVS};
 use rustfs_config::{
     COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
     MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
-    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
+    WEBHOOK_SKIP_TLS_VERIFY,
 };
 use std::sync::LazyLock;
 
@@ -60,6 +61,16 @@ pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
             value: "".to_owned(),
             hidden_if_empty: false,
         },
+        KV {
+            key: WEBHOOK_CLIENT_CA.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        },
+        KV {
+            key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(),
+            value: EnableState::Off.to_string(),
+            hidden_if_empty: false,
+        },
         KV {
             key: COMMENT_KEY.to_owned(),
             value: "".to_owned(),

crates/filemeta/Cargo.toml

@@ -44,6 +44,7 @@ regex.workspace = true
 
 [dev-dependencies]
 criterion = { workspace = true }
+tempfile = { workspace = true }
 
 [[bench]]
 name = "xl_meta_bench"

crates/filemeta/src/filemeta.rs

@@ -1907,7 +1907,6 @@ mod test {
 
 #[tokio::test]
 async fn test_read_xl_meta_no_data() {
-    use tokio::fs;
     use tokio::fs::File;
     use tokio::io::AsyncWriteExt;
 
@@ -1926,13 +1925,15 @@ async fn test_read_xl_meta_no_data() {
 
     buff.resize(buff.len() + 100, 0);
 
-    let filepath = "./test_xl.meta";
+    // Use tempfile to avoid conflicts with parallel tests or previous runs
+    let dir = tempfile::tempdir().unwrap();
+    let filepath = dir.path().join("test_xl.meta");
 
-    let mut file = File::create(filepath).await.unwrap();
+    let mut file = File::create(&filepath).await.unwrap();
     // Write string data
     file.write_all(&buff).await.unwrap();
 
-    let mut f = File::open(filepath).await.unwrap();
+    let mut f = File::open(&filepath).await.unwrap();
 
     let stat = f.metadata().await.unwrap();
 
@@ -1941,7 +1942,5 @@ async fn test_read_xl_meta_no_data() {
     let mut newfm = FileMeta::default();
     newfm.unmarshal_msg(&data).unwrap();
 
-    fs::remove_file(filepath).await.unwrap();
-
     assert_eq!(fm, newfm)
 }

crates/notify/src/integration.rs

@@ -18,12 +18,14 @@ use crate::{
     Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
 };
 use hashbrown::HashMap;
-use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
+use rustfs_config::notify::{
+    DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
 use rustfs_ecstore::config::{Config, KVS};
 use rustfs_s3_common::EventName;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::store::{Key, Store};
-use rustfs_targets::target::EntityTarget;
+use rustfs_targets::target::QueuedPayload;
 use rustfs_targets::{StoreError, Target};
 use std::collections::VecDeque;
 use std::sync::Arc;
@@ -34,6 +36,21 @@ use tracing::{debug, error, info, warn};
 
 const MAX_RECENT_LIVE_EVENTS: usize = 1024;
 
+fn subsystem_target_type(target_type: &str) -> &str {
+    match target_type {
+        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_MQTT_SUB_SYS => "mqtt",
+        _ => target_type,
+    }
+}
+
+fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
+    TargetID {
+        id: target_name.to_lowercase(),
+        name: subsystem_target_type(target_type).to_string(),
+    }
+}
+
 #[derive(Clone)]
 pub struct LiveEventBatch {
     pub events: Vec<Arc<Event>>,
@@ -183,58 +200,84 @@ impl NotificationSystem {
         }
     }
 
+    /// Initializes targets and starts event streams for those with stores.
+    /// Returns a map of (target_id -> cancel_sender) for streams that were started.
+    async fn init_targets_and_start_streams(
+        &self,
+        targets: &[Box<dyn Target<Event> + Send + Sync>],
+    ) -> HashMap<TargetID, mpsc::Sender<()>> {
+        let mut cancellers = HashMap::new();
+        for target in targets {
+            let target_id = target.id();
+            info!("Initializing target: {}", target_id);
+
+            let has_store = target.store().is_some();
+
+            if let Err(e) = target.init().await {
+                warn!("Target {} Initialization failed: {}", target_id, e);
+                // For targets without a store, init failure is fatal — skip.
+                // For store-backed targets, still start the stream so queued events
+                // can be drained when connectivity recovers (send_from_store retries).
+                if !has_store {
+                    continue;
+                }
+                warn!(
+                    "Target {} has a store, starting stream despite init failure — \
+                     connectivity will be retried by send_from_store",
+                    target_id
+                );
+            } else {
+                debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
+            }
+
+            if !target.is_enabled() {
+                info!("Target {} is not enabled, event stream processing is skipped", target_id);
+                continue;
+            }
+
+            if let Some(store) = target.store() {
+                info!("Start event stream processing for target {}", target_id);
+
+                let store_clone = store.boxed_clone();
+                let target_arc = Arc::from(target.clone_dyn());
+
+                let cancel_tx = self.enhanced_start_event_stream(
+                    store_clone,
+                    target_arc,
+                    self.metrics.clone(),
+                    self.concurrency_limiter.clone(),
+                );
+
+                let target_id_clone = target_id.clone();
+                cancellers.insert(target_id, cancel_tx);
+                info!("Event stream processing for target {} is started successfully", target_id_clone);
+            } else {
+                info!("Target {} No storage is configured, event stream processing is skipped", target_id);
+            }
+        }
+        cancellers
+    }
+
     /// Initializes the notification system
     pub async fn init(&self) -> Result<(), NotificationError> {
         info!("Initialize notification system...");
 
-        let config = self.config.read().await;
+        let config = {
-        debug!("Initializing notification system with config: {:?}", *config);
+            let guard = self.config.read().await;
+            debug!("Initializing notification system with config: {:?}", *guard);
+            guard.clone()
+        };
+
         let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
 
         info!("{} notification targets were created", targets.len());
 
-        // Initiate event stream processing for each storage enabled target
+        // Initialize targets and start event streams
-        let mut cancellers = HashMap::new();
+        let cancellers = self.init_targets_and_start_streams(&targets).await;
-        for target in &targets {
-            let target_id = target.id();
-            info!("Initializing target: {}", target.id());
-            // Initialize the target
-            if let Err(e) = target.init().await {
-                warn!("Target {} Initialization failed:{}", target.id(), e);
-                continue;
-            }
-            debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled());
-            // Check if the target is enabled and has storage
-            if target.is_enabled() {
-                if let Some(store) = target.store() {
-                    info!("Start event stream processing for target {}", target.id());
 
-                    // The storage of the cloned target and the target itself
+        // Update canceller collection
-                    let store_clone = store.boxed_clone();
-                    let target_box = target.clone_dyn();
-                    let target_arc = Arc::from(target_box);
-
-                    // Add a reference to the monitoring metrics
-                    let metrics = self.metrics.clone();
-                    let semaphore = self.concurrency_limiter.clone();
-
-                    // Encapsulated enhanced version of start_event_stream
-                    let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
-
-                    // Start event stream processing and save cancel sender
-                    let target_id_clone = target_id.clone();
-                    cancellers.insert(target_id, cancel_tx);
-                    info!("Event stream processing for target {} is started successfully", target_id_clone);
-                } else {
-                    info!("Target {} No storage is configured, event stream processing is skipped", target_id);
-                }
-            } else {
-                info!("Target {} is not enabled, event stream processing is skipped", target_id);
-            }
-        }
-
-        // Update canceler collection
         *self.stream_cancellers.write().await = cancellers;
+
         // Initialize the bucket target
         self.notifier.init_bucket_targets(targets).await?;
         info!("Notification system initialized");
@@ -333,7 +376,7 @@ impl NotificationSystem {
         info!("Attempting to remove target: {}", target_id);
 
         let ttype = target_type.to_lowercase();
-        let tname = target_id.name.to_lowercase();
+        let tname = target_id.id.to_lowercase();
 
         self.update_config_and_reload(|config| {
             let mut changed = false;
@@ -405,11 +448,7 @@ impl NotificationSystem {
 
         let ttype = target_type.to_lowercase();
         let tname = target_name.to_lowercase();
-
+        let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
-        let target_id = TargetID {
-            id: tname.clone(),
-            name: ttype.clone(),
-        };
 
         // Deletion is prohibited if bucket rules refer to it
         if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
@@ -451,7 +490,7 @@ impl NotificationSystem {
     /// Enhanced event stream startup function, including monitoring and concurrency control
     fn enhanced_start_event_stream(
         &self,
-        store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
+        store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
         target: Arc<dyn Target<Event> + Send + Sync>,
         metrics: Arc<NotificationMetrics>,
         semaphore: Arc<Semaphore>,
@@ -476,14 +515,13 @@ impl NotificationSystem {
             let _ = cancel_tx.send(()).await;
         }
 
-        // Clear the target_list and ensure that reload is a replacement reconstruction (solve the target_list len unchanged/residual problem)
+        // Clear the target_list and ensure that reload is a replacement reconstruction
         self.notifier.remove_all_bucket_targets().await;
 
         // Update the config
         self.update_config(new_config.clone()).await;
 
-        // Create a new target from configuration
+        // Create new targets from configuration
-        // This function will now be responsible for merging env, creating and persisting the final configuration.
         let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
             .registry
             .create_targets_from_config(&new_config)
@@ -492,46 +530,8 @@ impl NotificationSystem {
 
         info!("{} notification targets were created from the new configuration", targets.len());
 
-        // Start new event stream processing for each storage enabled target
+        // Initialize targets and start event streams using shared helper
-        let mut new_cancellers = HashMap::new();
+        let new_cancellers = self.init_targets_and_start_streams(&targets).await;
-        for target in &targets {
-            let target_id = target.id();
-
-            // Initialize the target
-            if let Err(e) = target.init().await {
-                error!("Target {} Initialization failed:{}", target_id, e);
-                continue;
-            }
-            // Check if the target is enabled and has storage
-            if target.is_enabled() {
-                if let Some(store) = target.store() {
-                    info!("Start new event stream processing for target {}", target_id);
-
-                    // The storage of the cloned target and the target itself
-                    let store_clone = store.boxed_clone();
-                    // let target_box = target.clone_dyn();
-                    let target_arc = Arc::from(target.clone_dyn());
-
-                    // Encapsulated enhanced version of start_event_stream
-                    let cancel_tx = self.enhanced_start_event_stream(
-                        store_clone,
-                        target_arc,
-                        self.metrics.clone(),
-                        self.concurrency_limiter.clone(),
-                    );
-
-                    // Start event stream processing and save cancel sender
-                    // let cancel_tx = start_event_stream(store_clone, target_clone);
-                    let target_id_clone = target_id.clone();
-                    new_cancellers.insert(target_id, cancel_tx);
-                    info!("Event stream processing of target {} is restarted successfully", target_id_clone);
-                } else {
-                    info!("Target {} No storage is configured, event stream processing is skipped", target_id);
-                }
-            } else {
-                info!("Target {} disabled, event stream processing is skipped", target_id);
-            }
-        }
 
         // Update canceler collection
         *cancellers = new_cancellers;
@@ -665,4 +665,18 @@ mod tests {
         assert_eq!(batch.events.len(), 1);
         assert_eq!(batch.events[0].s3.object.key, "one");
     }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
+        assert_eq!(target_id.id, "primary");
+        assert_eq!(target_id.name, "webhook");
+    }
+
+    #[test]
+    fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
+        let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
+        assert_eq!(target_id.id, "analytics");
+        assert_eq!(target_id.name, "mqtt");
+    }
 }

crates/notify/src/notifier.rs

@@ -399,7 +399,7 @@ mod tests {
     use rustfs_targets::{
         TargetError,
         store::{Key, Store},
-        target::EntityTarget,
+        target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
     };
     use serde::{Serialize, de::DeserializeOwned};
     use std::sync::{
@@ -442,7 +442,7 @@ mod tests {
             Ok(())
         }
 
-        async fn send_from_store(&self, _key: Key) -> Result<(), TargetError> {
+        async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
             Ok(())
         }
 
@@ -450,7 +450,7 @@ mod tests {
             Ok(())
         }
 
-        fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
+        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
             None
         }
 

crates/notify/src/registry.rs

@@ -89,9 +89,6 @@ impl TargetRegistry {
         let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
         // A collection of asynchronous tasks for concurrently executing target creation
         let mut tasks = FuturesUnordered::new();
-        // let final_config = config.clone(); // Clone a configuration for aggregating the final result
-        // Record the defaults for each segment so that the segment can eventually be rebuilt
-        let mut section_defaults: HashMap<String, KVS> = HashMap::new();
         // 1. Traverse all registered plants and process them by target type
         for (target_type, factory) in &self.factories {
             tracing::Span::current().record("target_type", target_type.as_str());
@@ -105,9 +102,6 @@ impl TargetRegistry {
             let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
             debug!(?default_cfg, "Get the default configuration");
 
-            // Save defaults for eventual write back
-            section_defaults.insert(section_name.clone(), default_cfg.clone());
-
             // *** Optimization point 1: Get all legitimate fields of the current target type ***
             let valid_fields = factory.get_valid_fields();
             debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -215,110 +209,28 @@ impl TargetRegistry {
                 if enabled {
                     info!(instance_id = %id, "Target is enabled, ready to create a task");
                     // 5.3. Create asynchronous tasks for enabled instances
-                    let target_type_clone = target_type.clone();
                     let tid = id.clone();
                     let merged_config_arc = Arc::new(merged_config);
                     tasks.push(async move {
                         let result = factory.create_target(tid.clone(), &merged_config_arc).await;
-                        (target_type_clone, tid, result, Arc::clone(&merged_config_arc))
+                        (tid, result)
                     });
                 } else {
-                    info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
+                    info!(instance_id = %id, "Skip disabled target");
-                    // Remove disabled target from final configuration
-                    // final_config.0.entry(section_name.clone()).or_default().remove(&id);
                 }
             }
         }
 
         // 6. Concurrently execute all creation tasks and collect results
         let mut successful_targets = Vec::new();
-        let mut successful_configs = Vec::new();
+        while let Some((id, result)) = tasks.next().await {
-        while let Some((target_type, id, result, final_config)) = tasks.next().await {
             match result {
                 Ok(target) => {
-                    info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
+                    info!(instance_id = %id, "Create target successfully");
                     successful_targets.push(target);
-                    successful_configs.push((target_type, id, final_config));
                 }
                 Err(e) => {
-                    error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
+                    error!(instance_id = %id, error = %e, "Failed to create target");
-                }
-            }
-        }
-
-        // 7. Aggregate new configuration and write back to system configuration
-        if !successful_configs.is_empty() || !section_defaults.is_empty() {
-            info!(
-                "Prepare to update {} successfully created target configurations to the system configuration...",
-                successful_configs.len()
-            );
-
-            let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
-
-            for (target_type, id, kvs) in successful_configs {
-                let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
-                successes_by_section
-                    .entry(section_name)
-                    .or_default()
-                    .insert(id.to_lowercase(), (*kvs).clone());
-            }
-
-            let mut new_config = config.clone();
-            // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
-            let mut sections: HashSet<String> = HashSet::new();
-            sections.extend(section_defaults.keys().cloned());
-            sections.extend(successes_by_section.keys().cloned());
-
-            for section in sections {
-                let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
-                // Add default item
-                if let Some(default_kvs) = section_defaults.get(&section)
-                    && !default_kvs.is_empty()
-                {
-                    section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
-                }
-
-                // Add successful instance item
-                if let Some(instances) = successes_by_section.get(&section) {
-                    for (id, kvs) in instances {
-                        section_map.insert(id.clone(), kvs.clone());
-                    }
-                }
-
-                // Empty breaks are removed and non-empty breaks are replaced entirely.
-                if section_map.is_empty() {
-                    new_config.0.remove(&section);
-                } else {
-                    new_config.0.insert(section, section_map);
-                }
-            }
-
-            if &new_config == config {
-                info!("Notification target configuration unchanged, skip persisting server config");
-                info!(count = successful_targets.len(), "All target processing completed");
-                return Ok(successful_targets);
-            }
-
-            let store = match rustfs_ecstore::global::new_object_layer_fn() {
-                Some(s) => s,
-                None => {
-                    warn!(
-                        "Object store not available at notification init; skipping config persistence. \
-                         {} target(s) active in memory.",
-                        successful_targets.len()
-                    );
-                    info!(count = successful_targets.len(), "All target processing completed");
-                    return Ok(successful_targets);
-                }
-            };
-
-            match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
-                Ok(_) => {
-                    info!("The new configuration was saved to the system successfully.")
-                }
-                Err(e) => {
-                    error!("Failed to save the new configuration: {}", e);
-                    return Err(TargetError::SaveConfig(e.to_string()));
                 }
             }
         }

crates/notify/src/stream.rs

@@ -15,8 +15,8 @@
 use crate::{Event, integration::NotificationMetrics};
 use rustfs_targets::{
     StoreError, Target, TargetError,
-    store::{Key, Store},
+    store::{Key, Store, ensure_store_entry_raw_readable},
-    target::EntityTarget,
+    target::QueuedPayload,
 };
 use rustfs_utils::get_env_usize;
 use std::sync::Arc;
@@ -32,7 +32,7 @@ use tracing::{debug, error, info, warn};
 /// - `target`: The target to send events to
 /// - `cancel_rx`: Receiver to listen for cancellation signals
 pub async fn stream_events(
-    store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
+    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
     target: &dyn Target<Event>,
     mut cancel_rx: mpsc::Receiver<()>,
 ) {
@@ -119,7 +119,7 @@ pub async fn stream_events(
 /// # Returns
 /// A sender to signal cancellation of the event stream
 pub fn start_event_stream(
-    mut store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
+    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
     target: Arc<dyn Target<Event> + Send + Sync>,
 ) -> mpsc::Sender<()> {
     let (cancel_tx, cancel_rx) = mpsc::channel(1);
@@ -143,7 +143,7 @@ pub fn start_event_stream(
 /// # Returns
 /// A sender to signal cancellation of the event stream
 pub fn start_event_stream_with_batching(
-    mut store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
+    mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
     target: Arc<dyn Target<Event> + Send + Sync>,
     metrics: Arc<NotificationMetrics>,
     semaphore: Arc<Semaphore>,
@@ -170,7 +170,7 @@ pub fn start_event_stream_with_batching(
 /// # Notes
 /// This function processes events in batches to improve efficiency.
 pub async fn stream_events_with_batching(
-    store: &mut (dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send),
+    store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
     target: &dyn Target<Event>,
     mut cancel_rx: mpsc::Receiver<()>,
     metrics: Arc<NotificationMetrics>,
@@ -185,7 +185,6 @@ pub async fn stream_events_with_batching(
     const MAX_RETRIES: usize = 5;
     const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
 
-    let mut batch: Vec<EntityTarget<Event>> = Vec::with_capacity(batch_size);
     let mut batch_keys = Vec::with_capacity(batch_size);
     let mut last_flush = Instant::now();
 
@@ -201,8 +200,8 @@ pub async fn stream_events_with_batching(
         debug!("Found {} keys in store for target: {}", keys.len(), target.name());
         if keys.is_empty() {
             // If there is data in the batch and timeout, refresh the batch
-            if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
+            if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
-                process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
+                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
                 last_flush = Instant::now();
             }
 
@@ -218,41 +217,31 @@ pub async fn stream_events_with_batching(
                 info!("Cancellation received during processing for target: {}", target.name());
 
                 // Processing collected batches before exiting
-                if !batch.is_empty() {
+                if !batch_keys.is_empty() {
-                    process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
+                    process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
                 }
                 return;
             }
 
-            // Try to get events from storage
+            // Skip unreadable entries so a single corrupt file cannot stall the stream.
-            match store.get(&key) {
+            // ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to
-                Ok(event) => {
+            // remove the corrupt entry before returning Err, so no cleanup is needed here.
-                    // Add to batch
+            match ensure_store_entry_raw_readable(&*store, &key) {
-                    batch.push(event);
+                Ok(true) => {}         // entry is readable, proceed
-                    batch_keys.push(key);
+                Ok(false) => continue, // entry not found (already removed), skip
-                    metrics.increment_processing();
+                Err(err) => {
-
+                    warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err);
-                    // If the batch is full or enough time has passed since the last refresh, the batch will be processed
+                    continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable
-                    if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
-                        process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore)
-                            .await;
-                        last_flush = Instant::now();
-                    }
                 }
-                Err(e) => {
+            }
-                    error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e);
-                    // Consider deleting unreadable events to prevent infinite loops from trying to read
-                    match store.del(&key) {
-                        Ok(_) => {
-                            info!("Deleted corrupted event {} from store", key.to_string());
-                        }
-                        Err(del_err) => {
-                            error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err);
-                        }
-                    }
 
-                    metrics.increment_failed();
+            batch_keys.push(key);
-                }
+            metrics.increment_processing();
+
+            // If the batch is full or enough time has passed since the last refresh, the batch will be processed
+            if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
+                process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
+                last_flush = Instant::now();
             }
         }
 
@@ -273,7 +262,6 @@ pub async fn stream_events_with_batching(
 /// # Notes
 /// This function processes a batch of events, sending each event to the target with retry
 async fn process_batch(
-    batch: &mut Vec<EntityTarget<Event>>,
     batch_keys: &mut Vec<Key>,
     target: &dyn Target<Event>,
     max_retries: usize,
@@ -281,8 +269,8 @@ async fn process_batch(
     metrics: &Arc<NotificationMetrics>,
     semaphore: &Arc<Semaphore>,
 ) {
-    debug!("Processing batch of {} events for target: {}", batch.len(), target.name());
+    debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name());
-    if batch.is_empty() {
+    if batch_keys.is_empty() {
         return;
     }
 
@@ -296,44 +284,42 @@ async fn process_batch(
     };
 
     // Handle every event in the batch
-    for (_event, key) in batch.iter().zip(batch_keys.iter()) {
+    for key in batch_keys.iter() {
         let mut retry_count = 0;
         let mut success = false;
 
         // Retry logic
         while retry_count < max_retries && !success {
-            // After sending successfully, the event in the storage is deleted synchronously.
             match target.send_from_store(key.clone()).await {
                 Ok(_) => {
-                    info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
+                    debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
                     success = true;
                     metrics.increment_processed();
                 }
-                Err(e) => {
+                Err(e) => match &e {
-                    // Different retry strategies are adopted according to the error type
+                    TargetError::NotConnected => {
-                    match &e {
+                        warn!("Target {} not connected, retrying...", target.name());
-                        TargetError::NotConnected => {
+                        retry_count += 1;
-                            warn!("Target {} not connected, retrying...", target.name());
+                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
-                            retry_count += 1;
+                        let backoff = 1u32 << retry_count as u32;
-                            tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff
+                        tokio::time::sleep(base_delay * backoff + jitter).await;
-                        }
-                        TargetError::Timeout(_) => {
-                            warn!("Timeout for target {}, retrying...", target.name());
-                            retry_count += 1;
-                            tokio::time::sleep(base_delay * (1 << retry_count)).await;
-                        }
-                        _ => {
-                            // Permanent error, skip this event
-                            error!("Permanent error for target {}: {}", target.name(), e);
-                            metrics.increment_failed();
-                            break;
-                        }
                     }
-                }
+                    TargetError::Timeout(_) => {
+                        warn!("Timeout for target {}, retrying...", target.name());
+                        retry_count += 1;
+                        let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
+                        let backoff = 1u32 << retry_count as u32;
+                        tokio::time::sleep(base_delay * backoff + jitter).await;
+                    }
+                    _ => {
+                        error!("Permanent error for target {}: {}", target.name(), e);
+                        metrics.increment_failed();
+                        break;
+                    }
+                },
             }
         }
 
-        // Handle the situation where the maximum number of retry exhaustion is exhausted
         if retry_count >= max_retries && !success {
             warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
             metrics.increment_failed();
@@ -341,7 +327,6 @@ async fn process_batch(
     }
 
     // Clear processed batches
-    batch.clear();
     batch_keys.clear();
 
     // Release semaphore permission (via drop)

crates/targets/Cargo.toml

@@ -28,5 +28,12 @@ url = { workspace = true }
 urlencoding = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 
+[dev-dependencies]
+criterion = { workspace = true }
+
+[[bench]]
+name = "queue_store_benchmark"
+harness = false
+
 [lints]
 workspace = true

crates/targets/benches/queue_store_benchmark.rs

@@ -0,0 +1,94 @@
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use rustfs_targets::store::{QueueStore, Store};
+use serde::{Deserialize, Serialize};
+use std::hint::black_box;
+use std::sync::Arc;
+use uuid::Uuid;
+
+#[derive(Clone, Serialize, Deserialize)]
+struct BenchEvent {
+    bucket: String,
+    key: String,
+    metadata: Vec<(String, String)>,
+    payload: String,
+}
+
+fn bench_dir(prefix: &str) -> std::path::PathBuf {
+    std::env::temp_dir().join(format!("rustfs-targets-bench-{prefix}-{}", Uuid::new_v4()))
+}
+
+fn build_payload(payload_len: usize) -> Vec<u8> {
+    let event = BenchEvent {
+        bucket: "bench-bucket".to_string(),
+        key: format!("objects/{payload_len}/file.json"),
+        metadata: (0..8)
+            .map(|idx| (format!("x-amz-meta-{idx}"), "benchmark-value".repeat(4)))
+            .collect(),
+        payload: "abcdefghijklmnopqrstuvwxyz0123456789".repeat(payload_len / 36 + 1)[..payload_len].to_string(),
+    };
+    serde_json::to_vec(&event).unwrap()
+}
+
+fn queue_store_write_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("queue_store_put_raw");
+
+    for payload_size in [512usize, 8 * 1024, 64 * 1024] {
+        let payload = build_payload(payload_size);
+        group.throughput(Throughput::Bytes(payload.len() as u64));
+
+        for compress in [false, true] {
+            let dir = bench_dir(if compress { "put-compress" } else { "put-plain" });
+            let store = QueueStore::<BenchEvent>::new_with_compression(&dir, 100_000, ".bench", compress);
+            store.open().unwrap();
+
+            group.bench_with_input(
+                BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size),
+                &payload,
+                |b, payload| {
+                    b.iter(|| {
+                        let key = store.put_raw(payload).unwrap();
+                        store.del(&key).unwrap();
+                    });
+                },
+            );
+
+            let _ = store.delete();
+        }
+    }
+
+    group.finish();
+}
+
+fn queue_store_read_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("queue_store_get_raw");
+
+    for payload_size in [512usize, 8 * 1024, 64 * 1024] {
+        let payload = build_payload(payload_size);
+        group.throughput(Throughput::Bytes(payload.len() as u64));
+
+        for compress in [false, true] {
+            let dir = bench_dir(if compress { "get-compress" } else { "get-plain" });
+            let store = Arc::new(QueueStore::<BenchEvent>::new_with_compression(&dir, 100_000, ".bench", compress));
+            store.open().unwrap();
+            let key = store.put_raw(&payload).unwrap();
+
+            group.bench_with_input(
+                BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size),
+                &(Arc::clone(&store), key),
+                |b, (store, key)| {
+                    b.iter(|| {
+                        let raw = store.get_raw(key).unwrap();
+                        black_box(raw);
+                    });
+                },
+            );
+
+            let _ = store.delete();
+        }
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, queue_store_write_benchmark, queue_store_read_benchmark);
+criterion_main!(benches);

crates/targets/src/store.rs

@@ -13,20 +13,34 @@
 // limitations under the License.
 
 use crate::error::StoreError;
-use rustfs_config::DEFAULT_LIMIT;
 use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
+use rustfs_config::{DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, ENV_TARGET_STORE_COMPRESS, EnableState};
 use serde::{Serialize, de::DeserializeOwned};
 use snap::raw::{Decoder, Encoder};
-use std::sync::{Arc, RwLock};
 use std::{
     collections::HashMap,
     marker::PhantomData,
     path::PathBuf,
+    sync::{
+        Arc, RwLock,
+        atomic::{AtomicU64, Ordering},
+    },
     time::{SystemTime, UNIX_EPOCH},
 };
 use tracing::{debug, warn};
 use uuid::Uuid;
 
+fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
+    value
+        .and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
+        .unwrap_or(DEFAULT_TARGET_STORE_COMPRESS)
+}
+
+fn queue_store_compression_enabled() -> bool {
+    let value = std::env::var(ENV_TARGET_STORE_COMPRESS).ok();
+    resolve_queue_store_compression_from_env_value(value.as_deref())
+}
+
 /// Represents a key for an entry in the store
 #[derive(Debug, Clone)]
 pub struct Key {
@@ -63,21 +77,7 @@ impl Key {
 
 impl std::fmt::Display for Key {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let name_part = if self.item_count > 1 {
+        f.write_str(&self.to_key_string())
-            format!("{}:{}", self.item_count, self.name)
-        } else {
-            self.name.clone()
-        };
-
-        let mut file_name = name_part;
-        if !self.extension.is_empty() {
-            file_name.push_str(&self.extension);
-        }
-
-        if self.compress {
-            file_name.push_str(COMPRESS_EXT);
-        }
-        write!(f, "{file_name}")
     }
 }
 
@@ -123,6 +123,28 @@ pub fn parse_key(s: &str) -> Key {
     }
 }
 
+pub fn ensure_store_entry_raw_readable<T>(
+    store: &(dyn Store<T, Error = StoreError, Key = Key> + Send),
+    key: &Key,
+) -> Result<bool, StoreError>
+where
+    T: Send + Sync + 'static + Clone + Serialize,
+{
+    match store.get_raw(key) {
+        Ok(_) => Ok(true),
+        Err(StoreError::NotFound) => Ok(false),
+        Err(err) => {
+            match store.del(key) {
+                Ok(()) | Err(StoreError::NotFound) => {}
+                Err(del_err) => {
+                    return Err(StoreError::Internal(format!("Failed to remove unreadable store entry {key}: {del_err}")));
+                }
+            }
+            Err(err)
+        }
+    }
+}
+
 /// Trait for a store that can store and retrieve items of type T
 pub trait Store<T>: Send + Sync
 where
@@ -142,15 +164,24 @@ where
     /// Stores multiple items in a single batch
     fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error>;
 
+    /// Stores raw bytes in a single entry.
+    fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error>;
+
     /// Retrieves a single item by key
     fn get(&self, key: &Self::Key) -> Result<T, Self::Error>;
 
     /// Retrieves multiple items by key
     fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error>;
 
+    /// Retrieves the raw bytes stored for a key.
+    fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error>;
+
     /// Deletes an item by key
     fn del(&self, key: &Self::Key) -> Result<(), Self::Error>;
 
+    /// Deletes the underlying store directory and clears all in-memory state.
+    fn delete(&self) -> Result<(), Self::Error>;
+
     /// Lists all keys in the store
     fn list(&self) -> Vec<Self::Key>;
 
@@ -169,7 +200,10 @@ pub struct QueueStore<T> {
     entry_limit: u64,
     directory: PathBuf,
     file_ext: String,
+    compress: bool,
     entries: Arc<RwLock<HashMap<String, i64>>>, // key -> modtime as unix nano
+    pending_entries: Arc<AtomicU64>,
+    fs_guard: Arc<RwLock<()>>,
     _phantom: PhantomData<T>,
 }
 
@@ -179,35 +213,70 @@ impl<T> Clone for QueueStore<T> {
             entry_limit: self.entry_limit,
             directory: self.directory.clone(),
             file_ext: self.file_ext.clone(),
+            compress: self.compress,
             entries: Arc::clone(&self.entries),
+            pending_entries: Arc::clone(&self.pending_entries),
+            fs_guard: Arc::clone(&self.fs_guard),
             _phantom: PhantomData,
         }
     }
 }
 
+struct EntryReservation<'a> {
+    pending_entries: &'a AtomicU64,
+}
+
+impl Drop for EntryReservation<'_> {
+    fn drop(&mut self) {
+        self.pending_entries.fetch_sub(1, Ordering::SeqCst);
+    }
+}
+
 impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
     /// Creates a new QueueStore
     pub fn new(directory: impl Into<PathBuf>, limit: u64, ext: &str) -> Self {
+        Self::new_with_compression(directory, limit, ext, queue_store_compression_enabled())
+    }
+
+    /// Creates a new QueueStore with an explicit compression setting.
+    pub fn new_with_compression(directory: impl Into<PathBuf>, limit: u64, ext: &str, compress: bool) -> Self {
         let file_ext = if ext.is_empty() { DEFAULT_EXT } else { ext };
+        let entry_limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
 
         QueueStore {
             directory: directory.into(),
-            entry_limit: if limit == 0 { DEFAULT_LIMIT } else { limit },
+            entry_limit,
             file_ext: file_ext.to_string(),
-            entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
+            compress,
+            entries: Arc::new(RwLock::new(HashMap::with_capacity(entry_limit as usize))),
+            pending_entries: Arc::new(AtomicU64::new(0)),
+            fs_guard: Arc::new(RwLock::new(())),
             _phantom: PhantomData,
         }
     }
 
     /// Returns the full path for a key
     fn file_path(&self, key: &Key) -> PathBuf {
-        self.directory.join(key.to_string())
+        self.directory.join(key.to_key_string())
+    }
+
+    fn build_key(&self, item_count: usize) -> Key {
+        Key {
+            name: Uuid::new_v4().to_string(),
+            extension: self.file_ext.clone(),
+            item_count,
+            compress: self.compress,
+        }
     }
 
     /// Reads a file for the given key
     fn read_file(&self, key: &Key) -> Result<Vec<u8>, StoreError> {
+        let _fs_guard = self
+            .fs_guard
+            .read()
+            .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
         let path = self.file_path(key);
-        debug!("Reading file for key: {},path: {}", key.to_string(), path.display());
+        debug!("Reading file for key: {},path: {}", key, path.display());
         let data = std::fs::read(&path).map_err(|e| {
             if e.kind() == std::io::ErrorKind::NotFound {
                 StoreError::NotFound
@@ -220,41 +289,89 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
             return Err(StoreError::NotFound);
         }
 
-        if key.compress {
+        if !key.compress {
-            let mut decoder = Decoder::new();
+            return Ok(data);
-            decoder
+        }
-                .decompress_vec(&data)
+
-                .map_err(|e| StoreError::Compression(e.to_string()))
+        let mut decoder = Decoder::new();
-        } else {
+        decoder
-            Ok(data)
+            .decompress_vec(&data)
+            .map_err(|e| StoreError::Compression(e.to_string()))
+    }
+
+    fn reserve_entry_slot(&self) -> Result<EntryReservation<'_>, StoreError> {
+        loop {
+            let entries = self
+                .entries
+                .read()
+                .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
+            let entries_len = entries.len() as u64;
+            let pending = self.pending_entries.load(Ordering::SeqCst);
+
+            if entries_len + pending >= self.entry_limit {
+                return Err(StoreError::LimitExceeded);
+            }
+
+            if self
+                .pending_entries
+                .compare_exchange(pending, pending + 1, Ordering::SeqCst, Ordering::SeqCst)
+                .is_ok()
+            {
+                return Ok(EntryReservation {
+                    pending_entries: self.pending_entries.as_ref(),
+                });
+            }
         }
     }
 
-    /// Writes data to a file for the given key
+    /// Writes data to a file for the given key.
-    fn write_file(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
+    fn write_file(&self, key: &Key, data: &[u8]) -> Result<i64, StoreError> {
         let path = self.file_path(key);
         // Create directory if it doesn't exist
         if let Some(parent) = path.parent() {
             std::fs::create_dir_all(parent).map_err(StoreError::Io)?;
         }
 
-        let data = if key.compress {
+        if key.compress {
             let mut encoder = Encoder::new();
-            encoder
+            let compressed = encoder
                 .compress_vec(data)
-                .map_err(|e| StoreError::Compression(e.to_string()))?
+                .map_err(|e| StoreError::Compression(e.to_string()))?;
+            std::fs::write(&path, &compressed).map_err(StoreError::Io)?;
         } else {
-            data.to_vec()
+            std::fs::write(&path, data).map_err(StoreError::Io)?;
-        };
+        }
-
-        std::fs::write(&path, &data).map_err(StoreError::Io)?;
         let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
+        debug!("Wrote event to store: {}", key);
+        Ok(modified)
+    }
+
+    fn insert_entry(&self, key: &Key, modified: i64) -> Result<(), StoreError> {
         let mut entries = self
             .entries
             .write()
             .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
-        entries.insert(key.to_string(), modified);
+        entries.insert(key.to_key_string(), modified);
-        debug!("Wrote event to store: {}", key.to_string());
+        Ok(())
+    }
+
+    fn remove_file_if_present(&self, key: &Key) -> Result<(), StoreError> {
+        let path = self.file_path(key);
+        match std::fs::remove_file(&path) {
+            Ok(()) => Ok(()),
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(err) => Err(StoreError::Io(err)),
+        }
+    }
+
+    fn write_and_index(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
+        let modified = self.write_file(key, data)?;
+        if let Err(err) = self.insert_entry(key, modified) {
+            self.remove_file_if_present(key).map_err(|cleanup_err| {
+                StoreError::Internal(format!("Failed to index store entry {key}: {err}; cleanup failed: {cleanup_err}"))
+            })?;
+            return Err(err);
+        }
         Ok(())
     }
 }
@@ -267,15 +384,20 @@ where
     type Key = Key;
 
     fn open(&self) -> Result<(), Self::Error> {
+        let _fs_guard = self
+            .fs_guard
+            .write()
+            .map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
         std::fs::create_dir_all(&self.directory).map_err(StoreError::Io)?;
 
-        let entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
+        let dir_entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
-        // Get the write lock to update the internal state
         let mut entries_map = self
             .entries
             .write()
             .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
-        for entry in entries {
+        self.pending_entries.store(0, Ordering::SeqCst);
+        entries_map.clear();
+        for entry in dir_entries {
             let entry = entry.map_err(StoreError::Io)?;
             let metadata = entry.metadata().map_err(StoreError::Io)?;
             if metadata.is_file() {
@@ -292,71 +414,47 @@ where
     }
 
     fn put(&self, item: Arc<T>) -> Result<Self::Key, Self::Error> {
-        // Check storage limits
+        let _fs_guard = self
-        {
+            .fs_guard
-            let entries = self
+            .read()
-                .entries
+            .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
-                .read()
+        let _reservation = self.reserve_entry_slot()?;
-                .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
+        let key = self.build_key(1);
-
-            if entries.len() as u64 >= self.entry_limit {
-                return Err(StoreError::LimitExceeded);
-            }
-        }
-
-        let uuid = Uuid::new_v4();
-        let key = Key {
-            name: uuid.to_string(),
-            extension: self.file_ext.clone(),
-            item_count: 1,
-            compress: true,
-        };
-
         let data = serde_json::to_vec(&*item).map_err(|e| StoreError::Serialization(e.to_string()))?;
-        self.write_file(&key, &data)?;
+        self.write_and_index(&key, &data)?;
 
         Ok(key)
     }
 
     fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error> {
-        // Check storage limits
-        {
-            let entries = self
-                .entries
-                .read()
-                .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
-
-            if entries.len() as u64 >= self.entry_limit {
-                return Err(StoreError::LimitExceeded);
-            }
-        }
         if items.is_empty() {
-            // Or return an error, or a special key?
             return Err(StoreError::Internal("Cannot put_multiple with empty items list".to_string()));
         }
-        let uuid = Uuid::new_v4();
+        let _fs_guard = self
-        let key = Key {
+            .fs_guard
-            name: uuid.to_string(),
+            .read()
-            extension: self.file_ext.clone(),
+            .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
-            item_count: items.len(),
+        let _reservation = self.reserve_entry_slot()?;
-            compress: true,
+        let key = self.build_key(items.len());
-        };
 
-        // Serialize all items into a single Vec<u8>
-        // This current approach for get_multiple/put_multiple assumes items are concatenated JSON objects.
-        // This might be problematic for deserialization if not handled carefully.
-        // A better approach for multiple items might be to store them as a JSON array `Vec<T>`.
-        // For now, sticking to current logic of concatenating.
         let mut buffer = Vec::new();
         for item in items {
-            // If items are Vec<Event>, and Event is large, this could be inefficient.
+            serde_json::to_writer(&mut buffer, &item).map_err(|e| StoreError::Serialization(e.to_string()))?;
-            // The current get_multiple deserializes one by one.
-            let item_data = serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?;
-            buffer.extend_from_slice(&item_data);
-            // If using JSON array: buffer = serde_json::to_vec(&items)?
         }
 
-        self.write_file(&key, &buffer)?;
+        self.write_and_index(&key, &buffer)?;
+
+        Ok(key)
+    }
+
+    fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+        let _fs_guard = self
+            .fs_guard
+            .read()
+            .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
+        let _reservation = self.reserve_entry_slot()?;
+        let key = self.build_key(1);
+        self.write_and_index(&key, data)?;
 
         Ok(key)
     }
@@ -373,8 +471,8 @@ where
     }
 
     fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
-        debug!("Reading items from store for key: {}", key.to_string());
+        debug!("Reading items from store for key: {}", key);
-        let data = self.read_file(key)?;
+        let data = self.get_raw(key)?;
         if data.is_empty() {
             return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
         }
@@ -404,7 +502,7 @@ where
                         warn!(
                             "Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
                             key.item_count,
-                            key.to_string(),
+                            key,
                             items.len()
                         );
                         // Depending on strictness, this could be an error.
@@ -426,20 +524,24 @@ where
         Ok(items)
     }
 
+    fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+        self.read_file(key)
+    }
+
     fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+        let _fs_guard = self
+            .fs_guard
+            .read()
+            .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
         let path = self.file_path(key);
-        std::fs::remove_file(&path).map_err(|e| {
+        match std::fs::remove_file(&path) {
-            if e.kind() == std::io::ErrorKind::NotFound {
+            Ok(()) => {}
-                // If file not found, still try to remove from entries map in case of inconsistency
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
-                warn!(
+                // File already gone — still clean up the entries map to avoid stale keys.
-                    "File not found for key {} during del, but proceeding to remove from entries map.",
+                warn!("File not found for key {} during del, cleaning up entries map.", key);
-                    key.to_string()
-                );
-                StoreError::NotFound
-            } else {
-                StoreError::Io(e)
             }
-        })?;
+            Err(e) => return Err(StoreError::Io(e)),
+        }
 
         // Get the write lock to update the internal state
         let mut entries = self
@@ -447,15 +549,32 @@ where
             .write()
             .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
 
-        if entries.remove(&key.to_string()).is_none() {
+        if entries.remove(&key.to_key_string()).is_none() {
-            // Key was not in the map, could be an inconsistency or already deleted.
-            // This is not necessarily an error if the file deletion succeeded or was NotFound.
             debug!("Key {} not found in entries map during del, might have been already removed.", key);
         }
         debug!("Deleted event from store: {}", key.to_string());
         Ok(())
     }
 
+    fn delete(&self) -> Result<(), Self::Error> {
+        let _fs_guard = self
+            .fs_guard
+            .write()
+            .map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
+        let mut entries = self
+            .entries
+            .write()
+            .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
+        entries.clear();
+        self.pending_entries.store(0, Ordering::SeqCst);
+
+        match std::fs::remove_dir_all(&self.directory) {
+            Ok(()) => Ok(()),
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(err) => Err(StoreError::Io(err)),
+        }
+    }
+
     fn list(&self) -> Vec<Self::Key> {
         // Get the read lock to read the internal state
         let entries = match self.entries.read() {
@@ -492,3 +611,136 @@ where
         Box::new(self.clone()) as Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync>
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::{
+        sync::{Arc, Barrier},
+        thread,
+    };
+
+    fn temp_store_dir(name: &str) -> PathBuf {
+        std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
+    }
+
+    #[test]
+    fn resolve_queue_store_compression_defaults_to_true() {
+        assert!(resolve_queue_store_compression_from_env_value(None));
+    }
+
+    #[test]
+    fn resolve_queue_store_compression_respects_disabled_env_value() {
+        assert!(!resolve_queue_store_compression_from_env_value(Some("off")));
+        assert!(!resolve_queue_store_compression_from_env_value(Some("false")));
+    }
+
+    #[test]
+    fn put_uses_store_compression_setting_in_key() {
+        let dir = temp_store_dir("put-key");
+        let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
+        store.open().unwrap();
+
+        let key = store.put(Arc::new("payload".to_string())).unwrap();
+
+        assert!(!key.compress);
+        assert!(store.file_path(&key).exists());
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
+    #[test]
+    fn parse_key_round_trips_batch_and_compression_suffixes() {
+        let key = Key {
+            name: "event-id".to_string(),
+            extension: ".json".to_string(),
+            item_count: 3,
+            compress: true,
+        };
+
+        let parsed = parse_key(&key.to_key_string());
+
+        assert_eq!(parsed.name, key.name);
+        assert_eq!(parsed.extension, key.extension);
+        assert_eq!(parsed.item_count, key.item_count);
+        assert_eq!(parsed.compress, key.compress);
+    }
+
+    #[test]
+    fn put_raw_and_get_raw_round_trip_bytes() {
+        let dir = temp_store_dir("raw-roundtrip");
+        let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", true);
+        store.open().unwrap();
+
+        let payload = br#"{"kind":"notify","bucket":"demo","key":"alpha.txt"}"#;
+        let key = store.put_raw(payload).unwrap();
+        let raw = store.get_raw(&key).unwrap();
+
+        assert_eq!(raw, payload);
+
+        let _ = store.delete();
+    }
+
+    #[test]
+    fn delete_removes_directory_and_clears_entries() {
+        let dir = temp_store_dir("delete-store");
+        let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
+        store.open().unwrap();
+        let _ = store.put(Arc::new("payload".to_string())).unwrap();
+
+        store.delete().unwrap();
+
+        assert!(store.list().is_empty());
+        assert!(!dir.exists());
+    }
+
+    #[test]
+    fn put_enforces_entry_limit() {
+        let dir = temp_store_dir("limit");
+        let store = QueueStore::<String>::new_with_compression(&dir, 1, ".test", false);
+        store.open().unwrap();
+
+        let _ = store.put(Arc::new("first".to_string())).unwrap();
+        let err = store.put(Arc::new("second".to_string())).unwrap_err();
+
+        assert!(matches!(err, StoreError::LimitExceeded));
+
+        let _ = store.delete();
+    }
+
+    #[test]
+    fn concurrent_put_raw_respects_entry_limit() {
+        let dir = temp_store_dir("concurrent-limit");
+        let store = Arc::new(QueueStore::<String>::new_with_compression(&dir, 1, ".test", true));
+        store.open().unwrap();
+
+        let start = Arc::new(Barrier::new(4));
+        let mut handles = Vec::new();
+
+        for idx in 0..4 {
+            let store = Arc::clone(&store);
+            let start = Arc::clone(&start);
+            handles.push(thread::spawn(move || {
+                let payload = vec![b'x'; 32 * 1024 + idx];
+                start.wait();
+                store.put_raw(&payload)
+            }));
+        }
+
+        let mut successes = 0;
+        let mut limit_errors = 0;
+        for handle in handles {
+            match handle.join().unwrap() {
+                Ok(_) => successes += 1,
+                Err(StoreError::LimitExceeded) => limit_errors += 1,
+                Err(err) => panic!("unexpected error: {err}"),
+            }
+        }
+
+        assert_eq!(successes, 1);
+        assert_eq!(limit_errors, 3);
+        assert_eq!(store.len(), 1);
+
+        let _ = store.delete();
+    }
+}

crates/targets/src/target/mod.rs

@@ -21,6 +21,8 @@ use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use std::fmt::Formatter;
 use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tracing::warn;
 
 pub mod mqtt;
 pub mod webhook;
@@ -45,14 +47,43 @@ where
     /// Saves an event (either sends it immediately or stores it for later)
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
 
-    /// Sends an event from the store
+    /// Sends an event from the store using the queued raw body and metadata.
-    async fn send_from_store(&self, key: Key) -> Result<(), TargetError>;
+    async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError>;
+
+    /// Sends an event from the store.
+    async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
+        let store = self
+            .store()
+            .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
+
+        let raw = match store.get_raw(&key) {
+            Ok(raw) => raw,
+            Err(StoreError::NotFound) => return Ok(()),
+            Err(err) => return Err(TargetError::Storage(format!("Failed to read queued payload from store: {err}"))),
+        };
+
+        let queued = match QueuedPayload::decode(&raw) {
+            Ok(queued) => queued,
+            Err(err) => {
+                delete_stored_payload(store, &key).map_err(|delete_err| {
+                    TargetError::Storage(format!(
+                        "Failed to delete invalid queued payload {key} after decode error '{err}': {delete_err}"
+                    ))
+                })?;
+                warn!("Dropped invalid queued payload {key}: {err}");
+                return Ok(());
+            }
+        };
+
+        self.send_raw_from_store(key.clone(), queued.body, queued.meta).await?;
+        delete_stored_payload(store, &key)
+    }
 
     /// Closes the target and releases resources
     async fn close(&self) -> Result<(), TargetError>;
 
     /// Returns the store associated with the target (if any)
-    fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)>;
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)>;
 
     /// Returns the type of the target
     fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync>;
@@ -78,6 +109,106 @@ where
     pub data: E,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct QueuedPayloadMeta {
+    pub event_name: EventName,
+    pub bucket_name: String,
+    pub object_name: String,
+    pub content_type: String,
+    pub queued_at_unix_ms: u64,
+    pub payload_len: usize,
+}
+
+impl QueuedPayloadMeta {
+    pub fn new(
+        event_name: EventName,
+        bucket_name: String,
+        object_name: String,
+        content_type: impl Into<String>,
+        payload_len: usize,
+    ) -> Self {
+        Self {
+            event_name,
+            bucket_name,
+            object_name,
+            content_type: content_type.into(),
+            queued_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64,
+            payload_len,
+        }
+    }
+
+    pub fn best_effort_preview(&self, body: &[u8], limit: usize) -> String {
+        if limit == 0 || body.is_empty() {
+            return String::new();
+        }
+
+        let slice = &body[..body.len().min(limit)];
+        match std::str::from_utf8(slice) {
+            Ok(text) => {
+                if body.len() > limit {
+                    format!("{text}...")
+                } else {
+                    text.to_string()
+                }
+            }
+            Err(_) => format!("<{} bytes binary>", body.len()),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct QueuedPayload {
+    pub meta: QueuedPayloadMeta,
+    pub body: Vec<u8>,
+}
+
+impl QueuedPayload {
+    const MAGIC: [u8; 4] = *b"RQP1";
+
+    pub fn new(meta: QueuedPayloadMeta, body: Vec<u8>) -> Self {
+        Self { meta, body }
+    }
+
+    pub fn encode(&self) -> Result<Vec<u8>, TargetError> {
+        let meta = serde_json::to_vec(&self.meta)
+            .map_err(|err| TargetError::Serialization(format!("Failed to serialize queued payload metadata: {err}")))?;
+        let meta_len = u32::try_from(meta.len())
+            .map_err(|_| TargetError::Serialization("Queued payload metadata is too large".to_string()))?;
+
+        let mut out = Vec::with_capacity(Self::MAGIC.len() + 4 + meta.len() + self.body.len());
+        out.extend_from_slice(&Self::MAGIC);
+        out.extend_from_slice(&meta_len.to_le_bytes());
+        out.extend_from_slice(&meta);
+        out.extend_from_slice(&self.body);
+        Ok(out)
+    }
+
+    pub fn decode(raw: &[u8]) -> Result<Self, TargetError> {
+        if raw.len() < Self::MAGIC.len() + 4 {
+            return Err(TargetError::Serialization("Queued payload is too short".to_string()));
+        }
+        if raw[..Self::MAGIC.len()] != Self::MAGIC {
+            return Err(TargetError::Serialization("Queued payload magic mismatch".to_string()));
+        }
+
+        let mut meta_len_bytes = [0u8; 4];
+        meta_len_bytes.copy_from_slice(&raw[Self::MAGIC.len()..Self::MAGIC.len() + 4]);
+        let meta_len = u32::from_le_bytes(meta_len_bytes) as usize;
+        let meta_start = Self::MAGIC.len() + 4;
+        let meta_end = meta_start + meta_len;
+
+        if meta_end > raw.len() {
+            return Err(TargetError::Serialization("Queued payload metadata length exceeds input".to_string()));
+        }
+
+        let meta = serde_json::from_slice(&raw[meta_start..meta_end])
+            .map_err(|err| TargetError::Serialization(format!("Failed to deserialize queued payload metadata: {err}")))?;
+        let body = raw[meta_end..].to_vec();
+
+        Ok(Self { meta, body })
+    }
+}
+
 /// The `ChannelTargetType` enum represents the different types of channel Target
 /// used in the notification system.
 ///
@@ -187,3 +318,45 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
         .map(|s| s.into_owned())
         .map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}")))
 }
+
+pub(crate) fn delete_stored_payload(
+    store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
+    key: &Key,
+) -> Result<(), TargetError> {
+    match store.del(key) {
+        Ok(()) | Err(StoreError::NotFound) => Ok(()),
+        Err(err) => Err(TargetError::Storage(format!("Failed to delete event from store: {err}"))),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn queued_payload_round_trips_meta_and_body() {
+        let meta = QueuedPayloadMeta::new(
+            EventName::ObjectCreatedPut,
+            "bucket-a".to_string(),
+            "folder/object.txt".to_string(),
+            "application/json",
+            12,
+        );
+        let payload = QueuedPayload::new(meta.clone(), br#"{"ok":true}"#.to_vec());
+
+        let encoded = payload.encode().unwrap();
+        let decoded = QueuedPayload::decode(&encoded).unwrap();
+
+        assert_eq!(decoded.meta.event_name, meta.event_name);
+        assert_eq!(decoded.meta.bucket_name, meta.bucket_name);
+        assert_eq!(decoded.meta.object_name, meta.object_name);
+        assert_eq!(decoded.meta.content_type, meta.content_type);
+        assert_eq!(decoded.body, br#"{"ok":true}"#);
+    }
+
+    #[test]
+    fn queued_payload_decode_rejects_invalid_magic() {
+        let err = QueuedPayload::decode(b"bad-payload").unwrap_err();
+        assert!(err.to_string().contains("magic") || err.to_string().contains("short"));
+    }
+}

crates/targets/src/target/mqtt.rs

@@ -17,7 +17,7 @@ use crate::{
     arn::TargetID,
     error::TargetError,
     store::{Key, QueueStore, Store},
-    target::{ChannelTargetType, EntityTarget, TargetType},
+    target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
 };
 use async_trait::async_trait;
 use rumqttc::{AsyncClient, ConnectionError, EventLoop, MqttOptions, Outgoing, Packet, QoS, mqttbytes::Error as MqttBytesError};
@@ -25,6 +25,7 @@ use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::sync::Arc;
 use std::{
+    marker::PhantomData,
     path::PathBuf,
     sync::atomic::{AtomicBool, Ordering},
     time::Duration,
@@ -110,9 +111,10 @@ where
     id: TargetID,
     args: MQTTArgs,
     client: Arc<Mutex<Option<AsyncClient>>>,
-    store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
+    store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     connected: Arc<AtomicBool>,
     bg_task_manager: Arc<BgTaskManager>,
+    _phantom: PhantomData<E>,
 }
 
 impl<E> MQTTTarget<E>
@@ -135,7 +137,7 @@ where
                 TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
             };
 
-            let store = QueueStore::<EntityTarget<E>>::new(specific_queue_path, args.queue_limit, extension);
+            let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
             if let Err(e) = store.open() {
                 error!(
                     target_id = %target_id,
@@ -144,7 +146,7 @@ where
                 );
                 return Err(TargetError::Storage(format!("{e}")));
             }
-            Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
+            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
         } else {
             None
         };
@@ -157,13 +159,14 @@ where
         });
 
         info!(target_id = %target_id, "MQTT target created");
-        Ok(MQTTTarget {
+        Ok(MQTTTarget::<E> {
             id: target_id,
             args,
             client: Arc::new(Mutex::new(None)),
             store: queue_store,
             connected: Arc::new(AtomicBool::new(false)),
             bg_task_manager,
+            _phantom: PhantomData,
         })
     }
 
@@ -251,14 +254,7 @@ where
         }
     }
 
-    #[instrument(skip(self, event), fields(target_id = %self.id))]
+    fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-    async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
-        // Decode form-urlencoded object name
         let object_name = crate::target::decode_object_name(&event.object_name)?;
 
         let key = format!("{}/{}", event.bucket_name, object_name);
@@ -269,14 +265,35 @@ where
             records: vec![event.clone()],
         };
 
-        let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+        let meta = QueuedPayloadMeta::new(
+            event.event_name,
+            event.bucket_name.clone(),
+            event.object_name.clone(),
+            "application/json",
+            body.len(),
+        );
+        Ok(QueuedPayload::new(meta, body))
+    }
 
-        let data_string = String::from_utf8(data.clone())
+    #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
-            .map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
+    async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        debug!("Sending event to mqtt target: {}, event log: {}", self.id, data_string);
+        let client_guard = self.client.lock().await;
+        let client = client_guard
+            .as_ref()
+            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+
+        debug!(
+            target = %self.id,
+            bucket = %meta.bucket_name,
+            object = %meta.object_name,
+            event = %meta.event_name,
+            preview = %meta.best_effort_preview(&body, 256),
+            "Sending MQTT payload"
+        );
 
         client
-            .publish(&self.args.topic, self.args.qos, false, data)
+            .publish(&self.args.topic, self.args.qos, false, body)
             .await
             .map_err(|e| {
                 if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
@@ -293,13 +310,14 @@ where
     }
 
     pub fn clone_target(&self) -> Box<dyn Target<E> + Send + Sync> {
-        Box::new(MQTTTarget {
+        Box::new(MQTTTarget::<E> {
             id: self.id.clone(),
             args: self.args.clone(),
             client: self.client.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             connected: self.connected.clone(),
             bg_task_manager: self.bg_task_manager.clone(),
+            _phantom: PhantomData,
         })
     }
 }
@@ -494,11 +512,15 @@ where
 
     #[instrument(skip(self, event), fields(target_id = %self.id))]
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+        let queued = self.build_queued_payload(&event)?;
+
         if let Some(store) = &self.store {
             debug!(target_id = %self.id, "Event saved to store start");
-            // If store is configured, ONLY put the event into the store.
+            match store.put_raw(
-            // Do NOT send it directly here.
+                &queued
-            match store.put(event.clone()) {
+                    .encode()
+                    .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
+            ) {
                 Ok(_) => {
                     debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
                     Ok(())
@@ -516,7 +538,7 @@ where
             if !self.connected.load(Ordering::SeqCst) {
                 warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
                 // Call the struct's init method, not the trait's default
-                match MQTTTarget::init(self).await {
+                match MQTTTarget::<E>::init(self).await {
                     Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
                     Err(e) => {
                         error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -528,13 +550,13 @@ where
                     return Err(TargetError::NotConnected);
                 }
             }
-            self.send(&event).await
+            self.send_body(queued.body, &queued.meta).await
         }
     }
 
-    #[instrument(skip(self), fields(target_id = %self.id))]
+    #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
-    async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
+    async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
-        debug!(target_id = %self.id, ?key, "Attempting to send event from store with key.");
+        debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
 
         if !self.is_enabled() {
             return Err(TargetError::Disabled);
@@ -542,7 +564,7 @@ where
 
         if !self.connected.load(Ordering::SeqCst) {
             warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
-            match MQTTTarget::init(self).await {
+            match MQTTTarget::<E>::init(self).await {
                 Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
                 Err(e) => {
                     error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -555,33 +577,8 @@ where
             }
         }
 
-        let store = self
-            .store
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
-
-        let event = match store.get(&key) {
-            Ok(event) => {
-                debug!(target_id = %self.id, ?key, "Retrieved event from store for sending.");
-                event
-            }
-            Err(StoreError::NotFound) => {
-                // Assuming NotFound takes the key
-                debug!(target_id = %self.id, ?key, "Event not found in store for sending.");
-                return Ok(());
-            }
-            Err(e) => {
-                error!(
-                    target_id = %self.id,
-                    error = %e,
-                    "Failed to get event from store"
-                );
-                return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
-            }
-        };
-
         debug!(target_id = %self.id, ?key, "Sending event from store.");
-        if let Err(e) = self.send(&event).await {
+        if let Err(e) = self.send_body(body, &meta).await {
             if matches!(e, TargetError::NotConnected) {
                 warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
                 return Err(TargetError::NotConnected);
@@ -589,22 +586,7 @@ where
             error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
             return Err(e);
         }
-        debug!(target_id = %self.id, ?key, "Event sent from store successfully. deleting from store. ");
+        debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
-
-        match store.del(&key) {
-            Ok(_) => {
-                debug!(target_id = %self.id, ?key, "Event deleted from store after successful send.")
-            }
-            Err(StoreError::NotFound) => {
-                debug!(target_id = %self.id, ?key, "Event already deleted from store.");
-            }
-            Err(e) => {
-                error!(target_id = %self.id, error = %e, "Failed to delete event from store after send.");
-                return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
-            }
-        }
-
-        debug!(target_id = %self.id, ?key, "Event deleted from store.");
         Ok(())
     }
 
@@ -637,7 +619,7 @@ where
         Ok(())
     }
 
-    fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
         self.store.as_deref()
     }
 
@@ -651,7 +633,7 @@ where
             return Ok(());
         }
         // Call the internal init logic
-        MQTTTarget::init(self).await
+        MQTTTarget::<E>::init(self).await
     }
 
     fn is_enabled(&self) -> bool {

crates/targets/src/target/webhook.rs

@@ -17,7 +17,7 @@ use crate::{
     arn::TargetID,
     error::TargetError,
     store::{Key, QueueStore, Store},
-    target::{ChannelTargetType, EntityTarget, TargetType},
+    target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
 };
 use async_trait::async_trait;
 use reqwest::{Client, StatusCode, Url};
@@ -26,6 +26,7 @@ use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
 use serde::Serialize;
 use serde::de::DeserializeOwned;
 use std::{
+    marker::PhantomData,
     path::PathBuf,
     sync::{
         Arc,
@@ -33,7 +34,6 @@ use std::{
     },
     time::Duration,
 };
-use tokio::net::lookup_host;
 use tokio::sync::mpsc;
 use tracing::{debug, error, info, instrument, warn};
 
@@ -105,10 +105,10 @@ where
     args: WebhookArgs,
     http_client: Arc<Client>,
     // Add Send + Sync constraints to ensure thread safety
-    store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
+    store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     initialized: AtomicBool,
-    addr: String,
     cancel_sender: mpsc::Sender<()>,
+    _phantom: PhantomData<E>,
 }
 
 impl<E> WebhookTarget<E>
@@ -117,14 +117,14 @@ where
 {
     /// Clones the WebhookTarget, creating a new instance with the same configuration
     pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
-        Box::new(WebhookTarget {
+        Box::new(WebhookTarget::<E> {
             id: self.id.clone(),
             args: self.args.clone(),
             http_client: Arc::clone(&self.http_client),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
-            addr: self.addr.clone(),
             cancel_sender: self.cancel_sender.clone(),
+            _phantom: PhantomData,
         })
     }
 
@@ -149,7 +149,7 @@ where
                 TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
             };
 
-            let store = QueueStore::<EntityTarget<E>>::new(queue_dir, args.queue_limit, extension);
+            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
 
             if let Err(e) = store.open() {
                 error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
@@ -157,32 +157,22 @@ where
             }
 
             // Make sure that the Store trait implemented by QueueStore matches the expected error type
-            Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
+            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
         } else {
             None
         };
 
-        // resolved address
-        let addr = {
-            let host = args.endpoint.host_str().unwrap_or("localhost");
-            let port = args
-                .endpoint
-                .port()
-                .unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 });
-            format!("{host}:{port}")
-        };
-
         // Create a cancel channel
         let (cancel_sender, _) = mpsc::channel(1);
         info!(target_id = %target_id.id, "Webhook target created");
-        Ok(WebhookTarget {
+        Ok(WebhookTarget::<E> {
             id: target_id,
             args,
             http_client,
             store: queue_store,
             initialized: AtomicBool::new(false),
-            addr,
             cancel_sender,
+            _phantom: PhantomData,
         })
     }
 
@@ -226,53 +216,80 @@ where
             .map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
     }
 
-    async fn init(&self) -> Result<(), TargetError> {
+    async fn init_inner(&self) -> Result<(), TargetError> {
-        // Use CAS operations to ensure thread-safe initialization
+        if self.initialized.load(Ordering::SeqCst) {
-        if !self.initialized.load(Ordering::SeqCst) {
+            return Ok(());
-            // Check the connection
+        }
-            match self.is_active().await {
+
-                Ok(true) => {
+        // HTTP HEAD probe: verifies the full request path (proxy, TLS, firewall)
-                    info!("Webhook target {} is active", self.id);
+        // unlike TCP connect which can't detect proxy issues.
-                }
+        let probe_timeout = Duration::from_secs(5);
-                Ok(false) => {
+        match tokio::time::timeout(probe_timeout, self.http_client.head(self.args.endpoint.as_str()).send()).await {
-                    return Err(TargetError::NotConnected);
+            Ok(Ok(resp)) => {
-                }
+                let status = resp.status();
-                Err(e) => {
+                if status.is_success() || status == StatusCode::NOT_FOUND {
-                    error!("Failed to check if Webhook target {} is active: {}", self.id, e);
+                    // NOT_FOUND is acceptable for HEAD probes — the endpoint may not
-                    return Err(e);
+                    // exist as a HEAD route, but the server is reachable.
+                    debug!("Webhook target {} HEAD probe returned {}", self.id, status);
+                } else if status == StatusCode::METHOD_NOT_ALLOWED {
+                    // Server is reachable but doesn't support HEAD — still valid.
+                    debug!("Webhook target {} HEAD probe: METHOD_NOT_ALLOWED (reachable)", self.id);
+                } else {
+                    warn!("Webhook target {} HEAD probe returned {}", self.id, status);
                 }
             }
-            self.initialized.store(true, Ordering::SeqCst);
+            Ok(Err(e)) => {
-            info!("Webhook target {} initialized", self.id);
+                // Connection-level error (DNS, TLS, refused, timeout)
+                return Err(if e.is_timeout() || e.is_connect() {
+                    TargetError::NotConnected
+                } else {
+                    TargetError::Network(format!("Webhook HEAD probe failed: {e}"))
+                });
+            }
+            Err(_) => {
+                return Err(TargetError::Timeout("Webhook HEAD probe timed out".to_string()));
+            }
         }
+
+        self.initialized.store(true, Ordering::SeqCst);
+        info!("Webhook target {} initialized", self.id);
         Ok(())
     }
 
-    async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
+    fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
-        info!("Webhook Sending event to webhook target: {}", self.id);
-        // Decode form-urlencoded object name
         let object_name = crate::target::decode_object_name(&event.object_name)?;
-
         let key = format!("{}/{}", event.bucket_name, object_name);
-
         let log = TargetLog {
             event_name: event.event_name,
             key,
             records: vec![event.data.clone()],
         };
+        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+        let meta = QueuedPayloadMeta::new(
+            event.event_name,
+            event.bucket_name.clone(),
+            event.object_name.clone(),
+            "application/json",
+            body.len(),
+        );
+        Ok(QueuedPayload::new(meta, body))
+    }
 
-        let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+    async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
+        info!("Webhook sending queued payload to target: {}", self.id);
+        debug!(
+            target = %self.id,
+            bucket = %meta.bucket_name,
+            object = %meta.object_name,
+            event = %meta.event_name,
+            preview = %meta.best_effort_preview(&body, 256),
+            "Sending webhook payload"
+        );
 
-        // Vec<u8> Convert to String
-        let data_string = String::from_utf8(data.clone())
-            .map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
-        debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string);
-
-        // build request
         let mut req_builder = self
             .http_client
             .post(self.args.endpoint.as_str())
-            .header("Content-Type", "application/json");
+            .header("Content-Type", meta.content_type.as_str());
 
         if !self.args.auth_token.is_empty() {
             // Split auth_token string to check if the authentication type is included
@@ -293,7 +310,7 @@ where
         }
 
         // Send a request
-        let resp = req_builder.body(data).send().await.map_err(|e| {
+        let resp = req_builder.body(body).send().await.map_err(|e| {
             if e.is_timeout() || e.is_connect() {
                 TargetError::NotConnected
             } else {
@@ -329,34 +346,39 @@ where
     }
 
     async fn is_active(&self) -> Result<bool, TargetError> {
-        let socket_addr = lookup_host(&self.addr)
+        match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(self.args.endpoint.as_str()).send()).await {
-            .await
+            Ok(Ok(resp)) => {
-            .map_err(|e| TargetError::Network(format!("Failed to resolve host: {e}")))?
+                let status = resp.status();
-            .next()
+                if status.is_server_error() {
-            .ok_or_else(|| TargetError::Network("No address found".to_string()))?;
+                    debug!("Webhook {} server error: {}", self.id, status);
-        debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id);
+                    Ok(false)
-        match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
-            Ok(Ok(_)) => {
-                debug!("Connection to {} is active", self.addr);
-                Ok(true)
-            }
-            Ok(Err(e)) => {
-                debug!("Connection to {} failed: {}", self.addr, e);
-                if e.kind() == std::io::ErrorKind::ConnectionRefused {
-                    Err(TargetError::NotConnected)
                 } else {
-                    Err(TargetError::Network(format!("Connection failed: {e}")))
+                    debug!("Webhook {} is reachable (status: {})", self.id, status);
+                    Ok(true)
                 }
             }
-            Err(_) => Err(TargetError::Timeout("Connection timed out".to_string())),
+            Ok(Err(e)) => {
+                debug!("Webhook {} request failed: {}", self.id, e);
+                if e.is_timeout() || e.is_connect() {
+                    Err(TargetError::NotConnected)
+                } else {
+                    Err(TargetError::Network(format!("Webhook health check failed: {e}")))
+                }
+            }
+            Err(_) => Err(TargetError::Timeout("Webhook health check timed out".to_string())),
         }
     }
 
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+        let queued = self.build_queued_payload(&event)?;
+
         if let Some(store) = &self.store {
-            // Call the store method directly, no longer need to acquire the lock
             store
-                .put(event)
+                .put_raw(
+                    &queued
+                        .encode()
+                        .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
+                )
                 .map_err(|e| TargetError::Storage(format!("Failed to save event to store: {e}")))?;
             debug!("Event saved to store for target: {}", self.id);
             Ok(())
@@ -368,12 +390,12 @@ where
                     return Err(TargetError::NotConnected);
                 }
             }
-            self.send(&event).await
+            self.send_body(queued.body, &queued.meta).await
         }
     }
 
-    async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
+    async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
-        debug!("Sending event from store for target: {}", self.id);
+        debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
         match self.init().await {
             Ok(_) => {
                 debug!("Event sent to store for target: {}", self.name());
@@ -384,37 +406,13 @@ where
             }
         }
 
-        let store = self
+        if let Err(e) = self.send_body(body, &meta).await {
-            .store
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
-
-        // Get events directly from the store, no longer need to acquire locks
-        let event = match store.get(&key) {
-            Ok(event) => event,
-            Err(StoreError::NotFound) => return Ok(()),
-            Err(e) => {
-                return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
-            }
-        };
-
-        if let Err(e) = self.send(&event).await {
             if let TargetError::NotConnected = e {
                 return Err(TargetError::NotConnected);
             }
             return Err(e);
         }
 
-        // Use the immutable reference of the store to delete the event content corresponding to the key
-        debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string());
-        match store.del(&key) {
-            Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()),
-            Err(e) => {
-                error!("Failed to delete event from store: {}", e);
-                return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
-            }
-        }
-
         debug!("Event sent from store and deleted for target: {}", self.id);
         Ok(())
     }
@@ -426,7 +424,7 @@ where
         Ok(())
     }
 
-    fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
         // Returns the reference to the internal store
         self.store.as_deref()
     }
@@ -436,14 +434,11 @@ where
     }
 
     async fn init(&self) -> Result<(), TargetError> {
-        // If the target is disabled, return to success directly
         if !self.is_enabled() {
             debug!("Webhook target {} is disabled, skipping initialization", self.id);
             return Ok(());
         }
-
+        self.init_inner().await
-        // Use existing initialization logic
-        WebhookTarget::init(self).await
     }
 
     fn is_enabled(&self) -> bool {

rustfs/src/admin/handlers/audit.rs

@@ -0,0 +1,818 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::admin::router::{AdminOperation, Operation, S3Router};
+use crate::auth::{check_key_valid, get_session_token};
+use crate::server::ADMIN_PREFIX;
+use futures::stream::{FuturesUnordered, StreamExt};
+use hashbrown::HashSet as HbHashSet;
+use http::{HeaderMap, StatusCode};
+use hyper::Method;
+use matchit::Params;
+use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
+use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
+use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
+use rustfs_ecstore::config::Config;
+use rustfs_targets::check_mqtt_broker_available;
+use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
+use std::io::{Error, ErrorKind};
+use std::path::Path;
+use std::sync::Arc;
+use tokio::sync::Semaphore;
+use tokio::time::{Duration, sleep, timeout};
+use tracing::{Span, warn};
+use url::Url;
+
+pub fn register_audit_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/list").as_str(),
+        AdminOperation(&ListAuditTargets {}),
+    )?;
+
+    r.insert(
+        Method::PUT,
+        format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}").as_str(),
+        AdminOperation(&AuditTargetConfig {}),
+    )?;
+
+    r.insert(
+        Method::DELETE,
+        format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}/reset").as_str(),
+        AdminOperation(&RemoveAuditTarget {}),
+    )?;
+
+    Ok(())
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KeyValue {
+    pub key: String,
+    pub value: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct AuditTargetBody {
+    pub key_values: Vec<KeyValue>,
+}
+
+#[derive(Serialize, Debug)]
+struct AuditEndpoint {
+    account_id: String,
+    service: String,
+    status: String,
+    source: AuditEndpointSource,
+}
+
+#[derive(Serialize, Debug)]
+struct AuditEndpointsResponse {
+    audit_endpoints: Vec<AuditEndpoint>,
+}
+
+type EndpointKey = (String, String);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "lowercase")]
+enum AuditEndpointSource {
+    Config,
+    Env,
+    Mixed,
+    Runtime,
+}
+
+fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
+    (account_id.to_lowercase(), service.to_string())
+}
+
+async fn check_permissions(req: &S3Request<Body>) -> S3Result<()> {
+    let Some(input_cred) = &req.credentials else {
+        return Err(s3_error!(InvalidRequest, "credentials not found"));
+    };
+    check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+    Ok(())
+}
+
+fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
+    let mut header = HeaderMap::new();
+    header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
+    if let Some(v) = request_id {
+        header.insert("x-request-id", v.clone());
+    }
+    S3Response::with_headers((status, body), header)
+}
+
+async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, Error>>,
+{
+    let mut attempts = 0;
+    let mut delay = base_delay;
+    let mut last_err = None;
+
+    while attempts < max_attempts {
+        match operation().await {
+            Ok(result) => return Ok(result),
+            Err(e) => {
+                last_err = Some(e);
+                attempts += 1;
+                if attempts < max_attempts {
+                    sleep(delay).await;
+                    delay = delay.saturating_mul(2);
+                }
+            }
+        }
+    }
+    Err(last_err.unwrap_or_else(|| Error::other("retry_with_backoff: unknown error")))
+}
+
+async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
+    if !queue_dir.is_empty() {
+        if !Path::new(queue_dir).is_absolute() {
+            return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path"));
+        }
+        retry_with_backoff(
+            || async { tokio::fs::metadata(queue_dir).await.map(|_| ()) },
+            3,
+            Duration::from_millis(100),
+        )
+        .await
+        .map_err(|e| match e.kind() {
+            ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"),
+            ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"),
+            _ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e),
+        })?;
+    }
+    Ok(())
+}
+
+fn config_enable_is_on(value: &str) -> bool {
+    matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
+}
+
+fn has_any_audit_targets(config: &Config) -> bool {
+    for subsystem in [AUDIT_WEBHOOK_SUB_SYS, AUDIT_MQTT_SUB_SYS] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+        if targets.keys().any(|key| key != DEFAULT_DELIMITER) {
+            return true;
+        }
+    }
+    false
+}
+
+fn collect_configured_audit_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
+    let mut endpoints = Vec::new();
+    for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+
+        for (target_name, kvs) in targets {
+            if target_name == DEFAULT_DELIMITER {
+                continue;
+            }
+            let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
+            if enabled {
+                endpoints.push((target_name.clone(), service.to_string()));
+            }
+        }
+    }
+    endpoints
+}
+
+fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
+    let mut endpoints = HbHashSet::new();
+    for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+
+        for target_name in targets.keys() {
+            if target_name == DEFAULT_DELIMITER {
+                continue;
+            }
+            endpoints.insert(normalized_endpoint_key(target_name, service));
+        }
+    }
+    endpoints
+}
+
+fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
+    let mut endpoints = HbHashSet::new();
+
+    for (service, valid_keys) in [("webhook", AUDIT_WEBHOOK_KEYS), ("mqtt", AUDIT_MQTT_KEYS)] {
+        let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase();
+
+        for (key, _value) in std::env::vars() {
+            let Some(rest) = key.strip_prefix(&env_prefix) else {
+                continue;
+            };
+
+            let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
+            let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
+            let field_name_part = parts.next();
+
+            let (field_name, instance_id) = match field_name_part {
+                Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
+                None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
+            };
+
+            if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() {
+                continue;
+            }
+
+            if valid_keys.contains(&field_name.as_str()) {
+                endpoints.insert(normalized_endpoint_key(&instance_id, service));
+            }
+        }
+    }
+
+    endpoints
+}
+
+fn classify_audit_endpoint_source(
+    config_targets: &HbHashSet<EndpointKey>,
+    env_targets: &HbHashSet<EndpointKey>,
+    key: &EndpointKey,
+) -> AuditEndpointSource {
+    match (config_targets.contains(key), env_targets.contains(key)) {
+        (true, true) => AuditEndpointSource::Mixed,
+        (true, false) => AuditEndpointSource::Config,
+        (false, true) => AuditEndpointSource::Env,
+        (false, false) => AuditEndpointSource::Runtime,
+    }
+}
+
+fn audit_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> AuditEndpointSource {
+    let config_targets = collect_config_entry_keys(config);
+    let env_targets = collect_env_endpoint_keys();
+    let service = match target_type {
+        AUDIT_WEBHOOK_SUB_SYS => "webhook",
+        AUDIT_MQTT_SUB_SYS => "mqtt",
+        _ => "",
+    };
+
+    let key = normalized_endpoint_key(target_name, service);
+    classify_audit_endpoint_source(&config_targets, &env_targets, &key)
+}
+
+fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+    match audit_endpoint_source(config, target_type, target_name) {
+        AuditEndpointSource::Env => Some(format!(
+            "audit target '{}' is managed by environment variables and cannot be modified from the console",
+            target_name
+        )),
+        AuditEndpointSource::Mixed => Some(format!(
+            "audit target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
+            target_name
+        )),
+        AuditEndpointSource::Config | AuditEndpointSource::Runtime => None,
+    }
+}
+
+fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
+    let mut audit_endpoints = Vec::new();
+    let mut seen = HashSet::new();
+    let configured_keys = collect_configured_audit_endpoint_keys(config);
+    let config_targets = collect_config_entry_keys(config);
+    let env_targets = collect_env_endpoint_keys();
+    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
+    for ((account_id, service), status) in runtime_statuses {
+        let normalized = normalized_endpoint_key(&account_id, &service);
+        normalized_runtime_statuses
+            .entry(normalized)
+            .or_insert((account_id, service, status));
+    }
+
+    for key in configured_keys {
+        let normalized = normalized_endpoint_key(&key.0, &key.1);
+        if !seen.insert(normalized.clone()) {
+            continue;
+        }
+        let status = normalized_runtime_statuses
+            .remove(&normalized)
+            .map(|(_, _, status)| status)
+            .unwrap_or_else(|| "offline".to_string());
+        let source = classify_audit_endpoint_source(&config_targets, &env_targets, &normalized);
+        audit_endpoints.push(AuditEndpoint {
+            account_id: key.0,
+            service: key.1,
+            status,
+            source,
+        });
+    }
+
+    for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
+        if seen.insert(normalized.clone()) {
+            audit_endpoints.push(AuditEndpoint {
+                account_id,
+                service,
+                status,
+                source: classify_audit_endpoint_source(&config_targets, &env_targets, &normalized),
+            });
+        }
+    }
+
+    for key in &env_targets {
+        if !seen.insert(key.clone()) {
+            continue;
+        }
+
+        audit_endpoints.push(AuditEndpoint {
+            account_id: key.0.clone(),
+            service: key.1.clone(),
+            status: "offline".to_string(),
+            source: classify_audit_endpoint_source(&config_targets, &env_targets, key),
+        });
+    }
+
+    audit_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
+    audit_endpoints
+}
+
+fn collect_validated_key_values(
+    key_values: &[KeyValue],
+    allowed_keys: &HashSet<&str>,
+    target_type: &str,
+) -> S3Result<HashMap<String, String>> {
+    let mut kv_map = HashMap::new();
+    let mut seen = HashSet::new();
+
+    for kv in key_values {
+        if !allowed_keys.contains(kv.key.as_str()) {
+            return Err(s3_error!(
+                InvalidArgument,
+                "key '{}' not allowed for audit target type '{}'",
+                kv.key,
+                target_type
+            ));
+        }
+
+        if !seen.insert(kv.key.as_str()) {
+            return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
+        }
+
+        kv_map.insert(kv.key.clone(), kv.value.clone());
+    }
+
+    Ok(kv_map)
+}
+
+fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
+    let target_type = params
+        .get("target_type")
+        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
+    if target_type != AUDIT_WEBHOOK_SUB_SYS && target_type != AUDIT_MQTT_SUB_SYS {
+        return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
+    }
+    let target_name = params
+        .get("target_name")
+        .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
+    Ok((target_type, target_name))
+}
+
+async fn load_server_config_from_store() -> S3Result<Config> {
+    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+        return Ok(Config::new());
+    };
+
+    rustfs_ecstore::config::com::read_config_without_migrate(store)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
+}
+
+async fn apply_audit_runtime_config(config: Config) -> S3Result<()> {
+    let has_targets = has_any_audit_targets(&config);
+
+    if let Some(system) = audit_system() {
+        match system.get_state().await {
+            AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
+                if has_targets {
+                    system
+                        .reload_config(config)
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
+                } else {
+                    system
+                        .close()
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
+                }
+            }
+            AuditSystemState::Stopped | AuditSystemState::Stopping => {
+                if has_targets {
+                    system
+                        .start(config)
+                        .await
+                        .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+                }
+            }
+        }
+    } else if has_targets {
+        start_global_audit_system(config)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+    }
+
+    Ok(())
+}
+
+async fn update_audit_config_and_reload<F>(mut modifier: F) -> S3Result<()>
+where
+    F: FnMut(&mut Config) -> bool,
+{
+    let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
+        return Err(s3_error!(InternalError, "server storage not initialized"));
+    };
+
+    let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
+
+    if !modifier(&mut config) {
+        return Ok(());
+    }
+
+    rustfs_ecstore::config::com::save_server_config(store, &config)
+        .await
+        .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+
+    apply_audit_runtime_config(config).await
+}
+
+pub struct AuditTargetConfig {}
+
+#[async_trait::async_trait]
+impl Operation for AuditTargetConfig {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let span = Span::current();
+        let _enter = span.enter();
+        let (target_type, target_name) = extract_target_params(&params)?;
+
+        check_permissions(&req).await?;
+        let config_snapshot = load_server_config_from_store().await?;
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
+
+        let mut input = req.input;
+        let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| {
+            warn!("failed to read request body: {:?}", e);
+            s3_error!(InvalidRequest, "failed to read request body")
+        })?;
+
+        let audit_body: AuditTargetBody = serde_json::from_slice(&body_bytes)
+            .map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
+
+        let allowed_keys: HashSet<&str> = match target_type {
+            AUDIT_WEBHOOK_SUB_SYS => AUDIT_WEBHOOK_KEYS.iter().cloned().collect(),
+            AUDIT_MQTT_SUB_SYS => AUDIT_MQTT_KEYS.iter().cloned().collect(),
+            _ => unreachable!(),
+        };
+
+        let kv_map = collect_validated_key_values(&audit_body.key_values, &allowed_keys, target_type)?;
+
+        if target_type == AUDIT_WEBHOOK_SUB_SYS {
+            let endpoint = kv_map
+                .get("endpoint")
+                .map(String::as_str)
+                .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
+            let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
+            match parsed_endpoint.scheme() {
+                "http" | "https" => {}
+                other => {
+                    return Err(s3_error!(
+                        InvalidArgument,
+                        "unsupported endpoint scheme: {} (only http and https are allowed)",
+                        other
+                    ));
+                }
+            }
+            if let Some(queue_dir) = kv_map.get("queue_dir") {
+                validate_queue_dir(queue_dir.as_str()).await?;
+            }
+            if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") {
+                return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair"));
+            }
+        } else if target_type == AUDIT_MQTT_SUB_SYS {
+            let endpoint = kv_map
+                .get(rustfs_config::MQTT_BROKER)
+                .map(String::as_str)
+                .ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?;
+            let topic = kv_map
+                .get(rustfs_config::MQTT_TOPIC)
+                .map(String::as_str)
+                .ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?;
+            let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str);
+            let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str);
+            check_mqtt_broker_available(endpoint, topic, username, password)
+                .await
+                .map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?;
+
+            if let Some(queue_dir) = kv_map.get("queue_dir") {
+                validate_queue_dir(queue_dir.as_str()).await?;
+                if let Some(qos) = kv_map.get("qos") {
+                    match qos.parse::<u8>() {
+                        Ok(1) | Ok(2) => {}
+                        Ok(0) => return Err(s3_error!(InvalidArgument, "qos should be 1 or 2 if queue_dir is set")),
+                        _ => return Err(s3_error!(InvalidArgument, "qos must be an integer 0, 1, or 2")),
+                    }
+                }
+            }
+        }
+
+        let mut kvs = rustfs_ecstore::config::KVS::new();
+        for (key, value) in kv_map {
+            kvs.insert(key, value);
+        }
+        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+
+        update_audit_config_and_reload(|config| {
+            config
+                .0
+                .entry(target_type.to_lowercase())
+                .or_default()
+                .insert(target_name.to_lowercase(), kvs.clone());
+            true
+        })
+        .await?;
+
+        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+    }
+}
+
+pub struct ListAuditTargets {}
+
+#[async_trait::async_trait]
+impl Operation for ListAuditTargets {
+    async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let span = Span::current();
+        let _enter = span.enter();
+        check_permissions(&req).await?;
+
+        let mut runtime_statuses = HashMap::new();
+        if let Some(system) = audit_system() {
+            let targets = system.get_target_values().await;
+            let semaphore = Arc::new(Semaphore::new(10));
+            let mut futures = FuturesUnordered::new();
+
+            for target in targets {
+                let sem = Arc::clone(&semaphore);
+                futures.push(async move {
+                    let _permit = sem.acquire().await;
+                    let status = match timeout(Duration::from_secs(3), target.is_active()).await {
+                        Ok(Ok(true)) => "online",
+                        _ => "offline",
+                    };
+                    ((target.id().id.clone(), target.id().name.to_string()), status.to_string())
+                });
+            }
+
+            while let Some((key, status)) = futures.next().await {
+                runtime_statuses.insert(key, status);
+            }
+        }
+
+        let config = load_server_config_from_store().await?;
+        let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
+        let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints })
+            .map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?;
+
+        Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
+    }
+}
+
+pub struct RemoveAuditTarget {}
+
+#[async_trait::async_trait]
+impl Operation for RemoveAuditTarget {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let span = Span::current();
+        let _enter = span.enter();
+        let (target_type, target_name) = extract_target_params(&params)?;
+
+        check_permissions(&req).await?;
+        let config_snapshot = load_server_config_from_store().await?;
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
+
+        update_audit_config_and_reload(|config| {
+            let mut changed = false;
+            if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
+                if targets.remove(&target_name.to_lowercase()).is_some() {
+                    changed = true;
+                }
+                if targets.is_empty() {
+                    config.0.remove(&target_type.to_lowercase());
+                }
+            }
+            changed
+        })
+        .await?;
+
+        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_ecstore::config::{KV, KVS};
+    use std::collections::{HashMap, HashSet};
+    use temp_env::{with_var, with_vars};
+
+    fn enabled_kvs(value: &str) -> KVS {
+        KVS(vec![KV {
+            key: ENABLE_KEY.to_string(),
+            value: value.to_string(),
+            hidden_if_empty: false,
+        }])
+    }
+
+    #[test]
+    fn merge_audit_endpoints_marks_config_env_and_mixed_sources() {
+        let config = Config(HashMap::from([(
+            AUDIT_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([
+                ("mixed-target".to_string(), enabled_kvs("on")),
+                ("config-target".to_string(), enabled_kvs("on")),
+            ]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
+            ],
+            || {
+                let runtime = HashMap::from([
+                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                ]);
+                let merged = merge_audit_endpoints(&config, runtime);
+
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "mixed-target")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, AuditEndpointSource::Mixed);
+
+                let env_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "env-only")
+                    .expect("env-only target should be present");
+                assert_eq!(env_only.source, AuditEndpointSource::Env);
+
+                let config_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "config-target")
+                    .expect("config target should be present");
+                assert_eq!(config_only.source, AuditEndpointSource::Config);
+            },
+        );
+    }
+
+    #[test]
+    fn audit_target_mutation_block_reason_rejects_env_managed_target() {
+        with_vars(
+            [
+                ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")),
+            ],
+            || {
+                let config = Config(HashMap::new());
+                let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+                assert!(reason.is_some());
+                assert!(reason.unwrap().contains("managed by environment variables"));
+            },
+        );
+    }
+
+    #[test]
+    fn audit_target_mutation_block_reason_rejects_mixed_target() {
+        with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
+            let config = Config(HashMap::from([(
+                AUDIT_WEBHOOK_SUB_SYS.to_string(),
+                HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
+            )]));
+            let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+            assert!(reason.is_some());
+            assert!(reason.unwrap().contains("both persisted config and environment variables"));
+        });
+    }
+
+    #[test]
+    fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() {
+        let config = Config(HashMap::from([(
+            AUDIT_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_AUDIT_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
+            ],
+            || {
+                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "mixed-disabled")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, AuditEndpointSource::Mixed);
+                assert_eq!(mixed.status, "offline");
+            },
+        );
+    }
+
+    #[test]
+    fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() {
+        let config = Config(HashMap::new());
+
+        with_vars(
+            [
+                ("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
+            ],
+            || {
+                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let env_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "env-only")
+                    .expect("env-only target should be present");
+                assert_eq!(env_only.source, AuditEndpointSource::Env);
+                assert_eq!(env_only.status, "offline");
+            },
+        );
+    }
+
+    #[test]
+    fn collect_validated_key_values_rejects_duplicate_keys() {
+        let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
+        let key_values = vec![
+            KeyValue {
+                key: "endpoint".to_string(),
+                value: "https://example.com/one".to_string(),
+            },
+            KeyValue {
+                key: "endpoint".to_string(),
+                value: "https://example.com/two".to_string(),
+            },
+        ];
+
+        let err = collect_validated_key_values(&key_values, &allowed_keys, AUDIT_WEBHOOK_SUB_SYS).unwrap_err();
+        assert!(err.to_string().contains("duplicate key"));
+    }
+
+    #[test]
+    fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() {
+        let config = Config(HashMap::from([(
+            AUDIT_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")),
+                ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
+            ],
+            || {
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
+                let merged = merge_audit_endpoints(&config, runtime);
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, AuditEndpointSource::Mixed);
+            },
+        );
+    }
+
+    #[test]
+    fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
+        let config = Config(HashMap::from([(
+            AUDIT_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
+        )]));
+
+        assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
+    }
+}

rustfs/src/admin/handlers/event.rs

@@ -16,21 +16,23 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::ADMIN_PREFIX;
 use futures::stream::{FuturesUnordered, StreamExt};
+use hashbrown::HashSet as HbHashSet;
 use http::{HeaderMap, StatusCode};
 use hyper::Method;
 use matchit::Params;
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::notify::{
-use rustfs_config::{ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
+    NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+};
+use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
+use rustfs_ecstore::config::Config;
 use rustfs_targets::check_mqtt_broker_available;
 use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::io::{Error, ErrorKind};
-use std::net::SocketAddr;
 use std::path::Path;
 use std::sync::Arc;
-use tokio::net::lookup_host;
 use tokio::sync::Semaphore;
 use tokio::time::{Duration, sleep, timeout};
 use tracing::{Span, info, warn};
@@ -80,6 +82,7 @@ struct NotificationEndpoint {
     account_id: String,
     service: String,
     status: String,
+    source: NotificationEndpointSource,
 }
 
 #[derive(Serialize, Debug)]
@@ -87,6 +90,21 @@ struct NotificationEndpointsResponse {
     notification_endpoints: Vec<NotificationEndpoint>,
 }
 
+type EndpointKey = (String, String);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "lowercase")]
+enum NotificationEndpointSource {
+    Config,
+    Env,
+    Mixed,
+    Runtime,
+}
+
+fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
+    (account_id.to_lowercase(), service.to_string())
+}
+
 // --- Helper Functions ---
 
 async fn check_permissions(req: &S3Request<Body>) -> S3Result<()> {
@@ -155,6 +173,215 @@ async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
     Ok(())
 }
 
+fn config_enable_is_on(value: &str) -> bool {
+    matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
+}
+
+fn collect_configured_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
+    let mut endpoints = Vec::new();
+    for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+
+        for (target_name, kvs) in targets {
+            if target_name == DEFAULT_DELIMITER {
+                continue;
+            }
+            let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
+            if enabled {
+                endpoints.push((target_name.clone(), service.to_string()));
+            }
+        }
+    }
+    endpoints
+}
+
+fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
+    let mut endpoints = HbHashSet::new();
+    for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+
+        for target_name in targets.keys() {
+            if target_name == DEFAULT_DELIMITER {
+                continue;
+            }
+            endpoints.insert(normalized_endpoint_key(target_name, service));
+        }
+    }
+    endpoints
+}
+
+fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
+    let mut endpoints = HbHashSet::new();
+
+    for (service, valid_keys) in [("webhook", NOTIFY_WEBHOOK_KEYS), ("mqtt", NOTIFY_MQTT_KEYS)] {
+        let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase();
+
+        for (key, _value) in std::env::vars() {
+            let Some(rest) = key.strip_prefix(&env_prefix) else {
+                continue;
+            };
+
+            let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
+            let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
+            let field_name_part = parts.next();
+
+            let (field_name, instance_id) = match field_name_part {
+                Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
+                None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
+            };
+
+            if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() {
+                continue;
+            }
+
+            if valid_keys.contains(&field_name.as_str()) {
+                endpoints.insert(normalized_endpoint_key(&instance_id, service));
+            }
+        }
+    }
+
+    endpoints
+}
+
+fn classify_notification_endpoint_source(
+    config_targets: &HbHashSet<EndpointKey>,
+    env_targets: &HbHashSet<EndpointKey>,
+    key: &EndpointKey,
+) -> NotificationEndpointSource {
+    match (config_targets.contains(key), env_targets.contains(key)) {
+        (true, true) => NotificationEndpointSource::Mixed,
+        (true, false) => NotificationEndpointSource::Config,
+        (false, true) => NotificationEndpointSource::Env,
+        (false, false) => NotificationEndpointSource::Runtime,
+    }
+}
+
+fn notification_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> NotificationEndpointSource {
+    let config_targets = collect_config_entry_keys(config);
+    let env_targets = collect_env_endpoint_keys();
+    let service = match target_type {
+        NOTIFY_WEBHOOK_SUB_SYS => "webhook",
+        NOTIFY_MQTT_SUB_SYS => "mqtt",
+        _ => "",
+    };
+
+    let key = normalized_endpoint_key(target_name, service);
+    classify_notification_endpoint_source(&config_targets, &env_targets, &key)
+}
+
+fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+    match notification_endpoint_source(config, target_type, target_name) {
+        NotificationEndpointSource::Env => Some(format!(
+            "target '{}' is managed by environment variables and cannot be modified from the console",
+            target_name
+        )),
+        NotificationEndpointSource::Mixed => Some(format!(
+            "target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
+            target_name
+        )),
+        NotificationEndpointSource::Config | NotificationEndpointSource::Runtime => None,
+    }
+}
+
+fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
+    let mut notification_endpoints = Vec::new();
+    let mut seen = HashSet::new();
+    let configured_keys = collect_configured_endpoint_keys(config);
+    let config_targets = collect_config_entry_keys(config);
+    let env_targets = collect_env_endpoint_keys();
+    let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
+    for ((account_id, service), status) in runtime_statuses {
+        let normalized = normalized_endpoint_key(&account_id, &service);
+        normalized_runtime_statuses
+            .entry(normalized)
+            .or_insert((account_id, service, status));
+    }
+
+    for key in configured_keys {
+        let normalized = normalized_endpoint_key(&key.0, &key.1);
+        if !seen.insert(normalized.clone()) {
+            continue;
+        }
+        let status = normalized_runtime_statuses
+            .remove(&normalized)
+            .map(|(_, _, status)| status)
+            .unwrap_or_else(|| "offline".to_string());
+        let source = classify_notification_endpoint_source(&config_targets, &env_targets, &normalized);
+        notification_endpoints.push(NotificationEndpoint {
+            account_id: key.0,
+            service: key.1,
+            status,
+            source,
+        });
+    }
+
+    for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
+        if seen.insert(normalized.clone()) {
+            notification_endpoints.push(NotificationEndpoint {
+                account_id,
+                service,
+                status,
+                source: classify_notification_endpoint_source(&config_targets, &env_targets, &normalized),
+            });
+        }
+    }
+
+    for key in &env_targets {
+        if !seen.insert(key.clone()) {
+            continue;
+        }
+
+        notification_endpoints.push(NotificationEndpoint {
+            account_id: key.0.clone(),
+            service: key.1.clone(),
+            status: "offline".to_string(),
+            source: classify_notification_endpoint_source(&config_targets, &env_targets, key),
+        });
+    }
+
+    notification_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
+    notification_endpoints
+}
+
+fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
+    target_statuses
+        .into_iter()
+        .filter_map(|(target_id, status)| (status == "online").then(|| target_id.to_arn(region).to_string()))
+        .collect()
+}
+
+fn collect_validated_key_values(
+    key_values: &[KeyValue],
+    allowed_keys: &HashSet<&str>,
+    target_type: &str,
+) -> S3Result<HashMap<String, String>> {
+    let mut kv_map = HashMap::new();
+    let mut seen = HashSet::new();
+
+    for kv in key_values {
+        if !allowed_keys.contains(kv.key.as_str()) {
+            return Err(s3_error!(
+                InvalidArgument,
+                "key '{}' not allowed for target type '{}'",
+                kv.key,
+                target_type
+            ));
+        }
+
+        if !seen.insert(kv.key.as_str()) {
+            return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
+        }
+
+        kv_map.insert(kv.key.clone(), kv.value.clone());
+    }
+
+    Ok(kv_map)
+}
+
 // --- Operations ---
 
 pub struct NotificationTarget {}
@@ -167,6 +394,10 @@ impl Operation for NotificationTarget {
 
         check_permissions(&req).await?;
         let ns = get_notification_system()?;
+        let config_snapshot = ns.config.read().await.clone();
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
 
         let mut input = req.input;
         let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| {
@@ -183,37 +414,27 @@ impl Operation for NotificationTarget {
             _ => unreachable!(),
         };
 
-        let kv_map: HashMap<&str, &str> = notification_body
+        let kv_map = collect_validated_key_values(&notification_body.key_values, &allowed_keys, target_type)?;
-            .key_values
-            .iter()
-            .map(|kv| (kv.key.as_str(), kv.value.as_str()))
-            .collect();
-
-        // Validate keys
-        for key in kv_map.keys() {
-            if !allowed_keys.contains(key) {
-                return Err(s3_error!(InvalidArgument, "key '{}' not allowed for target type '{}'", key, target_type));
-            }
-        }
 
         // Type-specific validation
         if target_type == NOTIFY_WEBHOOK_SUB_SYS {
             let endpoint = kv_map
                 .get("endpoint")
+                .map(String::as_str)
                 .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
-            let url = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
+            let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
-            let host = url
+            match parsed_endpoint.scheme() {
-                .host_str()
+                "http" | "https" => {}
-                .ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing host"))?;
+                other => {
-            let port = url
+                    return Err(s3_error!(
-                .port_or_known_default()
+                        InvalidArgument,
-                .ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing port"))?;
+                        "unsupported endpoint scheme: {} (only http and https are allowed)",
-            let addr = format!("{host}:{port}");
+                        other
-            if addr.parse::<SocketAddr>().is_err() && lookup_host(&addr).await.is_err() {
+                    ));
-                return Err(s3_error!(InvalidArgument, "invalid or unresolvable endpoint address"));
+                }
             }
             if let Some(queue_dir) = kv_map.get("queue_dir") {
-                validate_queue_dir(queue_dir).await?;
+                validate_queue_dir(queue_dir.as_str()).await?;
             }
             if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") {
                 return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair"));
@@ -221,18 +442,20 @@ impl Operation for NotificationTarget {
         } else if target_type == NOTIFY_MQTT_SUB_SYS {
             let endpoint = kv_map
                 .get(rustfs_config::MQTT_BROKER)
+                .map(String::as_str)
                 .ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?;
             let topic = kv_map
                 .get(rustfs_config::MQTT_TOPIC)
+                .map(String::as_str)
                 .ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?;
-            let username = kv_map.get(rustfs_config::MQTT_USERNAME).copied();
+            let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str);
-            let password = kv_map.get(rustfs_config::MQTT_PASSWORD).copied();
+            let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str);
             check_mqtt_broker_available(endpoint, topic, username, password)
                 .await
                 .map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?;
 
             if let Some(queue_dir) = kv_map.get("queue_dir") {
-                validate_queue_dir(queue_dir).await?;
+                validate_queue_dir(queue_dir.as_str()).await?;
                 if let Some(qos) = kv_map.get("qos") {
                     match qos.parse::<u8>() {
                         Ok(1) | Ok(2) => {}
@@ -243,24 +466,14 @@ impl Operation for NotificationTarget {
             }
         }
 
-        let mut kvs_vec: Vec<_> = notification_body
+        let mut kvs = rustfs_ecstore::config::KVS::new();
-            .key_values
+        for (key, value) in kv_map {
-            .into_iter()
+            kvs.insert(key, value);
-            .map(|kv| rustfs_ecstore::config::KV {
+        }
-                key: kv.key,
+        kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
-                value: kv.value,
-                hidden_if_empty: false,
-            })
-            .collect();
-
-        kvs_vec.push(rustfs_ecstore::config::KV {
-            key: ENABLE_KEY.to_string(),
-            value: EnableState::On.to_string(),
-            hidden_if_empty: false,
-        });
 
         info!("Setting target config for type '{}', name '{}'", target_type, target_name);
-        ns.set_target_config(target_type, target_name, rustfs_ecstore::config::KVS(kvs_vec))
+        ns.set_target_config(target_type, target_name, kvs)
             .await
             .map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?;
 
@@ -278,8 +491,6 @@ impl Operation for ListNotificationTargets {
         let ns = get_notification_system()?;
 
         let targets = ns.get_target_values().await;
-        let target_count = targets.len();
-
         let semaphore = Arc::new(Semaphore::new(10));
         let mut futures = FuturesUnordered::new();
 
@@ -291,18 +502,16 @@ impl Operation for ListNotificationTargets {
                     Ok(Ok(true)) => "online",
                     _ => "offline",
                 };
-                NotificationEndpoint {
+                ((target.id().id.clone(), target.id().name.to_string()), status.to_string())
-                    account_id: target.id().id.clone(),
-                    service: target.id().name.to_string(),
-                    status: status.to_string(),
-                }
             });
         }
 
-        let mut notification_endpoints = Vec::with_capacity(target_count);
+        let mut runtime_statuses = HashMap::new();
-        while let Some(endpoint) = futures.next().await {
+        while let Some((key, status)) = futures.next().await {
-            notification_endpoints.push(endpoint);
+            runtime_statuses.insert(key, status);
         }
+        let config = ns.config.read().await.clone();
+        let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
 
         let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
             .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
@@ -320,16 +529,32 @@ impl Operation for ListTargetsArns {
         check_permissions(&req).await?;
         let ns = get_notification_system()?;
 
-        let active_targets = ns.get_active_targets().await;
+        let targets = ns.get_target_values().await;
         let region = req
             .region
             .clone()
             .ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?;
+        let semaphore = Arc::new(Semaphore::new(10));
+        let mut futures = FuturesUnordered::new();
 
-        let data_target_arn_list: Vec<_> = active_targets
+        for target in targets {
-            .iter()
+            let sem = Arc::clone(&semaphore);
-            .map(|id| id.to_arn(region.as_str()).to_string())
+            futures.push(async move {
-            .collect();
+                let _permit = sem.acquire().await;
+                let status = match timeout(Duration::from_secs(3), target.is_active()).await {
+                    Ok(Ok(true)) => "online",
+                    _ => "offline",
+                };
+                (target.id(), status.to_string())
+            });
+        }
+
+        let mut target_statuses = Vec::new();
+        while let Some(target_status) = futures.next().await {
+            target_statuses.push(target_status);
+        }
+
+        let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
 
         let data = serde_json::to_vec(&data_target_arn_list)
             .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
@@ -348,6 +573,10 @@ impl Operation for RemoveNotificationTarget {
 
         check_permissions(&req).await?;
         let ns = get_notification_system()?;
+        let config_snapshot = ns.config.read().await.clone();
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+            return Err(s3_error!(InvalidRequest, "{reason}"));
+        }
 
         info!("Removing target config for type '{}', name '{}'", target_type, target_name);
         ns.remove_target_config(target_type, target_name)
@@ -372,3 +601,284 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &
     let target_name = extract_param(params, "target_name")?;
     Ok((target_type, target_name))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_ecstore::config::{KV, KVS};
+    use rustfs_targets::arn::TargetID;
+    use std::collections::{HashMap, HashSet};
+    use temp_env::{with_var, with_vars};
+
+    fn enabled_kvs(value: &str) -> KVS {
+        KVS(vec![KV {
+            key: ENABLE_KEY.to_string(),
+            value: value.to_string(),
+            hidden_if_empty: false,
+        }])
+    }
+
+    #[test]
+    fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
+        let mut cfg_map = HashMap::new();
+        cfg_map.insert(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("webhook-a".to_string(), enabled_kvs("on"))]),
+        );
+        cfg_map.insert(
+            NOTIFY_MQTT_SUB_SYS.to_string(),
+            HashMap::from([("mqtt-a".to_string(), enabled_kvs("on"))]),
+        );
+        let config = Config(cfg_map);
+
+        let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
+        let merged = merge_notification_endpoints(&config, runtime);
+
+        let mqtt = merged
+            .iter()
+            .find(|entry| entry.account_id == "mqtt-a" && entry.service == "mqtt")
+            .expect("mqtt-a should be present");
+        assert_eq!(mqtt.status, "offline");
+        assert_eq!(mqtt.source, NotificationEndpointSource::Config);
+
+        let webhook = merged
+            .iter()
+            .find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
+            .expect("webhook-a should be present");
+        assert_eq!(webhook.status, "online");
+        assert_eq!(webhook.source, NotificationEndpointSource::Config);
+    }
+
+    #[test]
+    fn merge_notification_endpoints_skips_disabled_and_default_entries() {
+        let mut webhook_targets = HashMap::new();
+        webhook_targets.insert(DEFAULT_DELIMITER.to_string(), enabled_kvs("on"));
+        webhook_targets.insert("webhook-disabled".to_string(), enabled_kvs("off"));
+        webhook_targets.insert("webhook-enabled".to_string(), enabled_kvs("on"));
+        let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)]));
+
+        let runtime = HashMap::from([
+            (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
+            (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
+        ]);
+        let merged = merge_notification_endpoints(&config, runtime);
+
+        let env_only = merged
+            .iter()
+            .find(|entry| entry.account_id == "env-only" && entry.service == "mqtt")
+            .expect("env-only should be present");
+        assert_eq!(env_only.status, "offline");
+        assert_eq!(env_only.source, NotificationEndpointSource::Runtime);
+
+        let enabled = merged
+            .iter()
+            .find(|entry| entry.account_id == "webhook-enabled" && entry.service == "webhook")
+            .expect("webhook-enabled should be present");
+        assert_eq!(enabled.status, "online");
+        assert_eq!(enabled.source, NotificationEndpointSource::Config);
+    }
+
+    #[test]
+    fn merge_notification_endpoints_marks_env_and_mixed_sources() {
+        let config = Config(HashMap::from([
+            (
+                NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+                HashMap::from([("mixed-target".to_string(), enabled_kvs("on"))]),
+            ),
+            (
+                NOTIFY_MQTT_SUB_SYS.to_string(),
+                HashMap::from([("config-target".to_string(), enabled_kvs("on"))]),
+            ),
+        ]));
+
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
+            ],
+            || {
+                let runtime = HashMap::from([
+                    (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
+                    (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
+                ]);
+                let merged = merge_notification_endpoints(&config, runtime);
+
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "mixed-target")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
+
+                let env_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "env-only")
+                    .expect("env-only target should be present");
+                assert_eq!(env_only.source, NotificationEndpointSource::Env);
+
+                let config_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "config-target")
+                    .expect("config target should be present");
+                assert_eq!(config_only.source, NotificationEndpointSource::Config);
+            },
+        );
+    }
+
+    #[test]
+    fn target_mutation_block_reason_rejects_env_managed_target() {
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", Some("on")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")),
+            ],
+            || {
+                let config = Config(HashMap::new());
+                let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+                assert!(reason.is_some());
+                assert!(reason.unwrap().contains("managed by environment variables"));
+            },
+        );
+    }
+
+    #[test]
+    fn target_mutation_block_reason_rejects_mixed_target() {
+        with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
+            let config = Config(HashMap::from([(
+                NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+                HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
+            )]));
+            let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+            assert!(reason.is_some());
+            assert!(reason.unwrap().contains("both persisted config and environment variables"));
+        });
+    }
+
+    #[test]
+    fn target_mutation_block_reason_allows_config_only_target() {
+        let target_name = "config-only-target";
+        let config = Config(HashMap::from([(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
+        )]));
+        assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
+    }
+
+    #[test]
+    fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() {
+        let config = Config(HashMap::from([(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
+            ],
+            || {
+                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "mixed-disabled")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
+                assert_eq!(mixed.status, "offline");
+            },
+        );
+    }
+
+    #[test]
+    fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() {
+        let config = Config(HashMap::new());
+
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
+            ],
+            || {
+                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let env_only = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "env-only")
+                    .expect("env-only target should be present");
+                assert_eq!(env_only.source, NotificationEndpointSource::Env);
+                assert_eq!(env_only.status, "offline");
+            },
+        );
+    }
+
+    #[test]
+    fn collect_validated_key_values_rejects_duplicate_keys() {
+        let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
+        let key_values = vec![
+            KeyValue {
+                key: "endpoint".to_string(),
+                value: "https://example.com/one".to_string(),
+            },
+            KeyValue {
+                key: "endpoint".to_string(),
+                value: "https://example.com/two".to_string(),
+            },
+        ];
+
+        let err = collect_validated_key_values(&key_values, &allowed_keys, NOTIFY_WEBHOOK_SUB_SYS).unwrap_err();
+        assert!(err.to_string().contains("duplicate key"));
+    }
+
+    #[test]
+    fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() {
+        let config = Config(HashMap::from([(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
+            ],
+            || {
+                let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
+                let merged = merge_notification_endpoints(&config, runtime);
+                let mixed = merged
+                    .iter()
+                    .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
+                    .expect("mixed target should be present");
+                assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
+            },
+        );
+    }
+
+    #[test]
+    fn collect_online_target_arns_filters_offline_targets() {
+        let arns = collect_online_target_arns(
+            "us-east-1",
+            vec![
+                (TargetID::new("webhook-a".to_string(), "webhook".to_string()), "online".to_string()),
+                (TargetID::new("mqtt-a".to_string(), "mqtt".to_string()), "offline".to_string()),
+            ],
+        );
+
+        assert_eq!(arns, vec!["arn:rustfs:sqs:us-east-1:webhook-a:webhook".to_string()]);
+    }
+
+    #[test]
+    fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
+        let config = Config(HashMap::from([(
+            NOTIFY_WEBHOOK_SUB_SYS.to_string(),
+            HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
+        )]));
+
+        with_vars(
+            [
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", None::<&str>),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
+            ],
+            || {
+                assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
+            },
+        );
+    }
+}

rustfs/src/admin/handlers/mod.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 pub mod account_info;
+pub mod audit;
 pub mod bucket_meta;
 pub mod event;
 pub mod group;
@@ -51,6 +52,7 @@ mod tests {
     fn test_handler_struct_creation() {
         // Test that handler structs can be created
         let _account_handler = account_info::AccountInfoHandler {};
+        let _list_audit_targets = audit::ListAuditTargets {};
         let _service_handler = system::ServiceHandle {};
         let _server_info_handler = system::ServerInfoHandler {};
         let _inspect_data_handler = system::InspectDataHandler {};

rustfs/src/admin/mod.rs

@@ -25,8 +25,8 @@ mod console_test;
 mod route_registration_test;
 
 use handlers::{
-    bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
+    audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts,
-    tier, user,
+    system, tier, user,
 };
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
@@ -55,6 +55,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
 
     quota::register_quota_route(&mut r)?;
     bucket_meta::register_bucket_meta_route(&mut r)?;
+    audit::register_audit_target_route(&mut r)?;
 
     replication::register_replication_route(&mut r)?;
     site_replication::register_site_replication_route(&mut r)?;

rustfs/src/admin/route_registration_test.rs

@@ -14,8 +14,8 @@
 
 use crate::admin::{
     handlers::{
-        bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
+        audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts,
-        tier, user,
+        system, tier, user,
     },
     router::{AdminOperation, S3Router},
 };
@@ -50,6 +50,7 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
     tier::register_tier_route(router).expect("register tier route");
     quota::register_quota_route(router).expect("register quota route");
     bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route");
+    audit::register_audit_target_route(router).expect("register audit target route");
     replication::register_replication_route(router).expect("register replication route");
     site_replication::register_site_replication_route(router).expect("register site replication route");
     profile_admin::register_profiling_route(router).expect("register profile route");
@@ -60,7 +61,6 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
 #[test]
 fn test_register_routes_cover_representative_admin_paths() {
     let mut router: S3Router<AdminOperation> = S3Router::new(false);
-
     register_admin_routes(&mut router);
     assert_route(&router, Method::GET, HEALTH_PREFIX);
     assert_route(&router, Method::HEAD, HEALTH_PREFIX);
@@ -91,6 +91,9 @@ fn test_register_routes_cover_representative_admin_paths() {
     assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach"));
     assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities"));
     assert_route(&router, Method::GET, &admin_path("/v3/target/list"));
+    assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
+    assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit"));
+    assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset"));
     assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));
 
     assert_route(&router, Method::POST, &admin_path("/v3/service"));
@@ -165,7 +168,6 @@ fn test_register_routes_cover_representative_admin_paths() {
 #[test]
 fn test_admin_alias_paths_match_existing_admin_routes() {
     let mut router: S3Router<AdminOperation> = S3Router::new(false);
-
     register_admin_routes(&mut router);
 
     for (method, path) in [

rustfs/src/server/audit.rs

@@ -14,13 +14,27 @@
 
 use crate::app::context::resolve_server_config;
 use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState};
-use rustfs_config::DEFAULT_DELIMITER;
 use tracing::{info, warn};
 
 fn server_config_from_context() -> Option<rustfs_ecstore::config::Config> {
     resolve_server_config()
 }
 
+fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool {
+    for subsystem in [
+        rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
+        rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
+    ] {
+        let Some(targets) = config.0.get(subsystem) else {
+            continue;
+        };
+        if targets.keys().any(|key| key != rustfs_config::DEFAULT_DELIMITER) {
+            return true;
+        }
+    }
+    false
+}
+
 /// Start the audit system.
 /// This function checks if the audit subsystem is configured in the global server configuration.
 /// If configured, it initializes and starts the audit system.
@@ -55,10 +69,8 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> {
         "The global server configuration is loaded"
     );
     // 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
-    let mqtt_config = server_config.get_value(rustfs_config::audit::AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER);
+    let has_targets = has_any_audit_targets(&server_config);
-    let webhook_config = server_config.get_value(rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER);
+    if !has_targets {
-
-    if mqtt_config.is_none() && webhook_config.is_none() {
         info!(
             target: "rustfs::main::start_audit_system",
             "Audit subsystem (MQTT/Webhook) is not configured, and audit system initialization is skipped."
@@ -68,9 +80,7 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> {
 
     info!(
         target: "rustfs::main::start_audit_system",
-        "Audit subsystem configuration detected (MQTT: {}, Webhook: {}) and started initializing the audit system.",
+        "Audit subsystem configuration detected and started initializing the audit system."
-        mqtt_config.is_some(),
-        webhook_config.is_some()
     );
     // 3. Initialize and start the audit system
     let system = init_audit_system();

scripts/run.sh

@@ -71,7 +71,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name
 export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version
 export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production
 export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error
-#export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
+export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
 export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory
 export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily"
 export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep