diff --git a/Cargo.lock b/Cargo.lock index e51a91424..a74a30c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7703,6 +7703,7 @@ dependencies = [ "rustfs-targets", "serde", "serde_json", + "temp-env", "thiserror 2.0.18", "tokio", "tracing", @@ -7897,6 +7898,7 @@ dependencies = [ "rustfs-utils", "s3s", "serde", + "tempfile", "thiserror 2.0.18", "time", "tokio", @@ -8396,6 +8398,7 @@ name = "rustfs-targets" version = "0.0.5" dependencies = [ "async-trait", + "criterion", "reqwest 0.13.2", "rumqttc", "rustfs-config", diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index ec904e1d6..0e0b03c09 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -44,6 +44,8 @@ tracing = { workspace = true, features = ["std", "attributes"] } url = { workspace = true } rumqttc = { workspace = true } +[dev-dependencies] +temp-env = { workspace = true } [lints] workspace = true diff --git a/crates/audit/src/registry.rs b/crates/audit/src/registry.rs index c6eccae57..f15bdba87 100644 --- a/crates/audit/src/registry.rs +++ b/crates/audit/src/registry.rs @@ -109,9 +109,6 @@ impl AuditRegistry { let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect(); // A collection of asynchronous tasks for concurrently executing target creation let mut tasks = FuturesUnordered::new(); - // let final_config = config.clone(); // Clone a configuration for aggregating the final result - // Record the defaults for each segment so that the segment can eventually be rebuilt - let mut section_defaults: HashMap = HashMap::new(); // 1. Traverse all registered plants and process them by target type for (target_type, factory) in &self.factories { tracing::Span::current().record("target_type", target_type.as_str()); @@ -125,9 +122,6 @@ impl AuditRegistry { let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default(); debug!(?default_cfg, "Get the default configuration"); - // Save defaults for eventual write back - section_defaults.insert(section_name.clone(), default_cfg.clone()); - // *** Optimization point 1: Get all legitimate fields of the current target type *** let valid_fields = factory.get_valid_fields(); debug!(?valid_fields, "Get the legitimate configuration fields"); @@ -235,103 +229,28 @@ impl AuditRegistry { if enabled { info!(instance_id = %id, "Target is enabled, ready to create a task"); // 5.3. Create asynchronous tasks for enabled instances - let target_type_clone = target_type.clone(); let tid = id.clone(); let merged_config_arc = Arc::new(merged_config); tasks.push(async move { let result = factory.create_target(tid.clone(), &merged_config_arc).await; - (target_type_clone, tid, result, Arc::clone(&merged_config_arc)) + (tid, result) }); } else { - info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration"); - // Remove disabled target from final configuration - // final_config.0.entry(section_name.clone()).or_default().remove(&id); + info!(instance_id = %id, "Skip disabled target"); } } } // 6. Concurrently execute all creation tasks and collect results let mut successful_targets = Vec::new(); - let mut successful_configs = Vec::new(); - while let Some((target_type, id, result, final_config)) = tasks.next().await { + while let Some((id, result)) = tasks.next().await { match result { Ok(target) => { - info!(target_type = %target_type, instance_id = %id, "Create a target successfully"); + info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully"); successful_targets.push(target); - successful_configs.push((target_type, id, final_config)); } Err(e) => { - error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target"); - } - } - } - - // 7. Aggregate new configuration and write back to system configuration - if !successful_configs.is_empty() || !section_defaults.is_empty() { - info!( - "Prepare to update {} successfully created target configurations to the system configuration...", - successful_configs.len() - ); - - let mut successes_by_section: HashMap> = HashMap::new(); - - for (target_type, id, kvs) in successful_configs { - let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase(); - successes_by_section - .entry(section_name) - .or_default() - .insert(id.to_lowercase(), (*kvs).clone()); - } - - let mut new_config = config.clone(); - // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist - let mut sections: HashSet = HashSet::new(); - sections.extend(section_defaults.keys().cloned()); - sections.extend(successes_by_section.keys().cloned()); - - for section in sections { - let mut section_map: std::collections::HashMap = std::collections::HashMap::new(); - // Add default item - if let Some(default_kvs) = section_defaults.get(§ion) - && !default_kvs.is_empty() - { - section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone()); - } - - // Add successful instance item - if let Some(instances) = successes_by_section.get(§ion) { - for (id, kvs) in instances { - section_map.insert(id.clone(), kvs.clone()); - } - } - - // Empty breaks are removed and non-empty breaks are replaced entirely. - if section_map.is_empty() { - new_config.0.remove(§ion); - } else { - new_config.0.insert(section, section_map); - } - } - - if &new_config == config { - info!("Audit target configuration unchanged, skip persisting server config"); - info!(count = successful_targets.len(), "All target processing completed"); - return Ok(successful_targets); - } - - let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { - return Err(AuditError::StorageNotAvailable( - "Failed to save target configuration: server storage not initialized".to_string(), - )); - }; - - match rustfs_ecstore::config::com::save_server_config(store, &new_config).await { - Ok(_) => { - info!("The new configuration was saved to the system successfully.") - } - Err(e) => { - error!("Failed to save the new configuration: {}", e); - return Err(AuditError::SaveConfig(Box::new(e))); + error!(instance_id = %id, error = %e, "Failed to create a target"); } } } @@ -371,6 +290,11 @@ impl AuditRegistry { self.targets.get(id).map(|t| t.as_ref()) } + /// Lists cloned target values for runtime inspection without exposing mutable registry access. + pub fn list_target_values(&self) -> Vec + Send + Sync>> { + self.targets.values().map(|target| target.clone_dyn()).collect() + } + /// Lists all target IDs /// /// # Returns diff --git a/crates/audit/src/system.rs b/crates/audit/src/system.rs index d9116b659..d47729722 100644 --- a/crates/audit/src/system.rs +++ b/crates/audit/src/system.rs @@ -13,14 +13,15 @@ // limitations under the License. use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability}; +use hashbrown::HashMap; use rustfs_ecstore::config::Config; use rustfs_targets::{ StoreError, Target, TargetError, store::{Key, Store}, - target::EntityTarget, + target::{EntityTarget, QueuedPayload}, }; use std::sync::Arc; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, RwLock, mpsc}; use tracing::{error, info, warn}; /// State of the audit system @@ -39,6 +40,8 @@ pub struct AuditSystem { registry: Arc>, state: Arc>, config: Arc>>, + /// Cancellation senders for active audit stream tasks (target_id -> cancel tx) + stream_cancellers: Arc>>>, } impl Default for AuditSystem { @@ -54,6 +57,7 @@ impl AuditSystem { registry: Arc::new(Mutex::new(AuditRegistry::new())), state: Arc::new(RwLock::new(AuditSystemState::Stopped)), config: Arc::new(RwLock::new(None)), + stream_cancellers: Arc::new(RwLock::new(HashMap::new())), } } @@ -110,27 +114,7 @@ impl AuditSystem { // Initialize all targets for target in targets { - let target_id = target.id().to_string(); - if let Err(e) = target.init().await { - error!(target_id = %target_id, error = %e, "Failed to initialize audit target"); - } else { - // After successful initialization, if enabled and there is a store, start the send from storage task - if target.is_enabled() { - if let Some(store) = target.store() { - info!(target_id = %target_id, "Start audit stream processing for target"); - let store_clone: Box, Error = StoreError, Key = Key> + Send> = - store.boxed_clone(); - let target_arc: Arc + Send + Sync> = Arc::from(target.clone_dyn()); - self.start_audit_stream_with_batching(store_clone, target_arc); - info!(target_id = %target_id, "Audit stream processing started"); - } else { - info!(target_id = %target_id, "No store configured, skip audit stream processing"); - } - } else { - info!(target_id = %target_id, "Target disabled, skip audit stream processing"); - } - registry.add_target(target_id, target); - } + self.init_and_register_target(target, &mut registry).await; } // Update state to running @@ -214,6 +198,9 @@ impl AuditSystem { info!("Stopping audit system"); + // Stop all stream tasks first + self.stop_all_streams().await; + // Close all targets let mut registry = self.registry.lock().await; if let Err(e) = registry.close_all().await { @@ -258,52 +245,49 @@ impl AuditSystem { let state = self.state.read().await; match *state { - AuditSystemState::Running => { - // Continue with dispatch - info!("Dispatching audit log entry"); - } + AuditSystemState::Running => {} AuditSystemState::Paused => { - // Skip dispatch when paused return Ok(()); } _ => { - // Don't dispatch when not running return Err(AuditError::NotInitialized("Audit system is not running".to_string())); } } drop(state); - let registry = self.registry.lock().await; - let target_keys = registry.list_targets(); + // Collect cloned targets under lock, then dispatch without holding it + let targets: Vec<(String, Box + Send + Sync>)> = { + let registry = self.registry.lock().await; + let target_keys = registry.list_targets(); - if target_keys.is_empty() { - warn!("No audit targets configured for dispatch"); - return Ok(()); - } + if target_keys.is_empty() { + warn!("No audit targets configured for dispatch"); + return Ok(()); + } - // Dispatch to all targets concurrently + target_keys + .into_iter() + .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn()))) + .collect() + }; + + // Dispatch to all targets concurrently (no lock held) let mut tasks = Vec::new(); - for target_key in target_keys { - if let Some(target) = registry.get_target(&target_key) { - let entry_clone = Arc::clone(&entry); - let target_key_clone = target_key.clone(); + for (target_key, target) in targets { + let entity_target = EntityTarget { + object_name: entry.api.name.clone().unwrap_or_default(), + bucket_name: entry.api.bucket.clone().unwrap_or_default(), + event_name: entry.event, + data: (*entry).clone(), + }; - // Create EntityTarget for the audit log entry - let entity_target = EntityTarget { - object_name: entry.api.name.clone().unwrap_or_default(), - bucket_name: entry.api.bucket.clone().unwrap_or_default(), - event_name: entry.event, // Default, should be derived from entry - data: (*entry_clone).clone(), - }; + let task = async move { + let result = target.save(Arc::new(entity_target)).await; + (target_key, result) + }; - let task = async move { - let result = target.save(Arc::new(entity_target)).await; - (target_key_clone, result) - }; - - tasks.push(task); - } + tasks.push(task); } // Execute all dispatch tasks @@ -359,39 +343,45 @@ impl AuditSystem { } drop(state); - let registry = self.registry.lock().await; - let target_keys = registry.list_targets(); + // Collect targets under lock, then dispatch without holding it + let targets: Vec<(String, Box + Send + Sync>)> = { + let registry = self.registry.lock().await; + let target_keys = registry.list_targets(); - if target_keys.is_empty() { - warn!("No audit targets configured for batch dispatch"); - return Ok(()); - } + if target_keys.is_empty() { + warn!("No audit targets configured for batch dispatch"); + return Ok(()); + } + + target_keys + .into_iter() + .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn()))) + .collect() + }; let mut tasks = Vec::new(); - for target_key in target_keys { - if let Some(target) = registry.get_target(&target_key) { - let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect(); - let target_key_clone = target_key.clone(); + for (target_key, target) in targets { + let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect(); + let target_key_clone = target_key.clone(); - let task = async move { - let mut success_count = 0; - let mut errors = Vec::new(); - for entry in entries_clone { - let entity_target = EntityTarget { - object_name: entry.api.name.clone().unwrap_or_default(), - bucket_name: entry.api.bucket.clone().unwrap_or_default(), - event_name: entry.event, - data: (*entry).clone(), - }; - match target.save(Arc::new(entity_target)).await { - Ok(_) => success_count += 1, - Err(e) => errors.push(e), - } + let task = async move { + let mut success_count = 0; + let mut errors = Vec::new(); + for entry in entries_clone { + let entity_target = EntityTarget { + object_name: entry.api.name.clone().unwrap_or_default(), + bucket_name: entry.api.bucket.clone().unwrap_or_default(), + event_name: entry.event, + data: (*entry).clone(), + }; + match target.save(Arc::new(entity_target)).await { + Ok(_) => success_count += 1, + Err(e) => errors.push(e), } - (target_key_clone, success_count, errors) - }; - tasks.push(task); - } + } + (target_key_clone, success_count, errors) + }; + tasks.push(task); } let results = futures::future::join_all(tasks).await; @@ -417,6 +407,60 @@ impl AuditSystem { Ok(()) } + /// Stops all active audit stream tasks by sending cancellation signals. + async fn stop_all_streams(&self) { + let mut cancellers = self.stream_cancellers.write().await; + for (target_id, cancel_tx) in cancellers.drain() { + info!(target_id = %target_id, "Stopping audit stream"); + let _ = cancel_tx.send(()).await; + } + } + + /// Initializes a single target: runs init(), starts stream if store is present, + /// and adds it to the registry. For store-backed targets, registration and stream + /// startup proceed even if init() fails so queued entries can be drained later. + async fn init_and_register_target( + &self, + target: Box + Send + Sync>, + registry: &mut AuditRegistry, + ) -> Option { + let target_id = target.id().to_string(); + let has_store = target.store().is_some(); + + if let Err(e) = target.init().await { + error!(target_id = %target_id, error = %e, "Failed to initialize audit target"); + // Non-store targets: init failure is fatal. + if !has_store { + return None; + } + // Store-backed targets: still register and start the stream so queued + // entries can be drained when connectivity recovers. + warn!( + target_id = %target_id, + "Proceeding with store-backed audit target despite init failure" + ); + } + + if target.is_enabled() { + if let Some(store) = target.store() { + info!(target_id = %target_id, "Start audit stream processing for target"); + let store_clone: Box + Send> = store.boxed_clone(); + let target_arc: Arc + Send + Sync> = Arc::from(target.clone_dyn()); + let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc); + + self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx); + info!(target_id = %target_id, "Audit stream processing started"); + } else { + info!(target_id = %target_id, "No store configured, skip audit stream processing"); + } + } else { + info!(target_id = %target_id, "Target disabled, skip audit stream processing"); + } + + registry.add_target(target_id.clone(), target); + Some(target_id) + } + /// Starts the audit stream processing for a target with batching and retry logic /// /// # Arguments @@ -427,9 +471,10 @@ impl AuditSystem { /// and attempts to send them to the specified target. It implements retry logic with exponential backoff fn start_audit_stream_with_batching( &self, - store: Box, Error = StoreError, Key = Key> + Send>, + store: Box + Send>, target: Arc + Send + Sync>, - ) { + ) -> mpsc::Sender<()> { + let (cancel_tx, mut cancel_rx) = mpsc::channel(1); let state = self.state.clone(); tokio::spawn(async move { @@ -442,6 +487,12 @@ impl AuditSystem { const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); loop { + // Check for cancellation signal + if cancel_rx.try_recv().is_ok() { + info!("Audit stream cancelled for target: {}", target.id()); + break; + } + match *state.read().await { AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {} _ => { @@ -452,11 +503,22 @@ impl AuditSystem { let keys: Vec = store.list(); if keys.is_empty() { - sleep(Duration::from_millis(500)).await; + tokio::select! { + _ = sleep(Duration::from_millis(500)) => {}, + _ = cancel_rx.recv() => { + info!("Audit stream cancelled during idle for target: {}", target.id()); + return; + } + } continue; } for key in keys { + if cancel_rx.try_recv().is_ok() { + info!("Audit stream cancelled during processing for target: {}", target.id()); + return; + } + let mut retries = 0usize; let mut success = false; @@ -497,6 +559,8 @@ impl AuditSystem { sleep(Duration::from_millis(100)).await; } }); + + cancel_tx } /// Enables a specific target @@ -594,6 +658,12 @@ impl AuditSystem { registry.list_targets() } + /// Returns cloned target values for read-only runtime inspection. + pub async fn get_target_values(&self) -> Vec + Send + Sync>> { + let registry = self.registry.lock().await; + registry.list_target_values() + } + /// Gets information about a specific target /// /// # Arguments @@ -616,9 +686,11 @@ impl AuditSystem { pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> { info!("Reloading audit system configuration"); - // Record config reload observability::record_config_reload(); + // Stop all existing stream tasks first + self.stop_all_streams().await; + // Store new configuration { let mut config_guard = self.config.write().await; @@ -636,28 +708,9 @@ impl AuditSystem { Ok(targets) => { info!(target_count = targets.len(), "Reloaded audit targets successfully"); - // Initialize all new targets for target in targets { - let target_id = target.id().to_string(); - if let Err(e) = target.init().await { - error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target"); - } else { - // Same starts the storage stream after a heavy load - if target.is_enabled() { - if let Some(store) = target.store() { - info!(target_id = %target_id, "Start audit stream processing for target (reload)"); - let store_clone: Box, Error = StoreError, Key = Key> + Send> = - store.boxed_clone(); - let target_arc: Arc + Send + Sync> = Arc::from(target.clone_dyn()); - self.start_audit_stream_with_batching(store_clone, target_arc); - info!(target_id = %target_id, "Audit stream processing started (reload)"); - } else { - info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)"); - } - } else { - info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)"); - } - registry.add_target(target.id().to_string(), target); + if let Some(target_id) = self.init_and_register_target(target, &mut registry).await { + info!(target_id = %target_id, "Target initialized (reload)"); } } diff --git a/crates/audit/tests/integration_test.rs b/crates/audit/tests/integration_test.rs index f2ef342e1..08f5f51e2 100644 --- a/crates/audit/tests/integration_test.rs +++ b/crates/audit/tests/integration_test.rs @@ -15,6 +15,7 @@ use rustfs_audit::*; use rustfs_ecstore::config::{Config, KVS}; use std::collections::HashMap; +use temp_env::with_vars; #[tokio::test] async fn test_audit_system_creation() { @@ -35,34 +36,42 @@ async fn test_config_parsing_webhook() { let mut config = Config(HashMap::new()); let mut audit_webhook_section = HashMap::new(); - // Create default configuration let mut default_kvs = KVS::new(); - default_kvs.insert("enable".to_string(), "on".to_string()); - default_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string()); - + default_kvs.insert("enable".to_string(), "off".to_string()); + default_kvs.insert("endpoint".to_string(), "".to_string()); audit_webhook_section.insert("_".to_string(), default_kvs); + let mut instance_kvs = KVS::new(); + instance_kvs.insert("enable".to_string(), "on".to_string()); + instance_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string()); + audit_webhook_section.insert("primary".to_string(), instance_kvs); config.0.insert("audit_webhook".to_string(), audit_webhook_section); let registry = AuditRegistry::new(); - // This should not fail even if server storage is not initialized - // as it's an integration test let result = registry.create_audit_targets_from_config(&config).await; + assert!(result.is_ok(), "audit target creation should not require server storage"); +} - // We expect this to fail due to server storage not being initialized - // but the parsing should work correctly - match result { - Err(AuditError::StorageNotAvailable(_)) => { - // This is expected in test environment - } - Err(e) => { - // Other errors might indicate parsing issues - println!("Unexpected error: {e}"); - } - Ok(_) => { - // Unexpected success in test environment without server storage - } - } +#[test] +fn test_env_only_audit_target_does_not_require_server_storage() { + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("http://localhost:3020/webhook")), + ], + || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to create tokio runtime"); + runtime.block_on(async { + let config = Config(HashMap::new()); + let registry = AuditRegistry::new(); + let result = registry.create_audit_targets_from_config(&config).await; + assert!(result.is_ok(), "env-only audit target creation should not require server storage"); + }); + }, + ) } #[test] diff --git a/crates/config/src/constants/targets.rs b/crates/config/src/constants/targets.rs index d6d3bb50c..ce341ae44 100644 --- a/crates/config/src/constants/targets.rs +++ b/crates/config/src/constants/targets.rs @@ -34,3 +34,10 @@ pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval"; pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval"; pub const MQTT_QUEUE_DIR: &str = "queue_dir"; pub const MQTT_QUEUE_LIMIT: &str = "queue_limit"; + +/// Environment variable controlling whether target queue files are Snappy-compressed. +/// Applies to both notify and audit target queue stores. +pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS"; + +/// Queue-store compression is enabled by default to reduce disk footprint. +pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true; diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index f4ed795a9..658b799de 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -313,6 +313,31 @@ async fn list_target_arns(env: &RustFSTestEnvironment) -> Result, Bo Ok(serde_json::from_slice(&body)?) } +async fn delete_webhook_target(env: &RustFSTestEnvironment, target_name: &str) -> Result<(), Box> { + let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}/reset", env.url); + let response = signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await?; + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if status != StatusCode::OK { + return Err(format!("failed to delete webhook target {target_name}: {status} {body}").into()); + } + + Ok(()) +} + +fn notification_target_is_listed(targets: &serde_json::Value, target_name: &str) -> bool { + targets["notification_endpoints"] + .as_array() + .into_iter() + .flatten() + .any(|entry| { + entry["account_id"].as_str() == Some(target_name) + && entry["service"] + .as_str() + .is_some_and(|service| service == "webhook" || service.starts_with("webhook-")) + }) +} + async fn wait_for_target_visibility( env: &RustFSTestEnvironment, target_name: &str, @@ -324,18 +349,7 @@ async fn wait_for_target_visibility( last_targets = list_notification_targets(env).await?; last_arns = list_target_arns(env).await?; - let listed = last_targets["notification_endpoints"] - .as_array() - .into_iter() - .flatten() - .any(|entry| { - entry["account_id"].as_str() == Some(target_name) - && entry["service"] - .as_str() - .is_some_and(|service| service == "webhook" || service.starts_with("webhook-")) - }); - - if listed { + if notification_target_is_listed(&last_targets, target_name) { return Ok((last_targets, last_arns)); } @@ -345,10 +359,51 @@ async fn wait_for_target_visibility( Err(format!("target {target_name} did not become visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into()) } +async fn wait_for_target_absence( + env: &RustFSTestEnvironment, + target_name: &str, +) -> Result<(serde_json::Value, Vec), Box> { + let mut last_targets = serde_json::Value::Null; + let mut last_arns = Vec::new(); + + for _ in 0..20 { + last_targets = list_notification_targets(env).await?; + last_arns = list_target_arns(env).await?; + + let listed = notification_target_is_listed(&last_targets, target_name); + let arn_listed = last_arns.iter().any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))); + if !listed && !arn_listed { + return Ok((last_targets, last_arns)); + } + + tokio::time::sleep(Duration::from_millis(250)).await; + } + + Err(format!("target {target_name} remained visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into()) +} + +async fn restart_rustfs_server(env: &mut RustFSTestEnvironment) -> Result<(), Box> { + env.stop_server(); + env.start_rustfs_server_without_cleanup(vec![]).await +} + async fn read_persisted_server_config(env: &RustFSTestEnvironment) -> String { let path = format!("{}/.rustfs.sys/config/config.json", env.temp_dir); match tokio::fs::read_to_string(&path).await { Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::IsADirectory => { + let mut entries = Vec::new(); + match tokio::fs::read_dir(&path).await { + Ok(mut dir) => { + while let Ok(Some(entry)) = dir.next_entry().await { + entries.push(entry.file_name().to_string_lossy().to_string()); + } + entries.sort(); + format!("persisted config stored as object directory at {path}; entries={entries:?}") + } + Err(dir_err) => format!("persisted config directory exists at {path} but could not be listed: {dir_err}"), + } + } Err(err) => format!("failed to read persisted config at {path}: {err}"), } } @@ -400,6 +455,66 @@ async fn read_listen_notification_event( } } +#[tokio::test] +#[serial] +async fn test_notification_target_persists_across_restart_and_delete() -> Result<(), Box> { + init_logging(); + + let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?; + + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + + let target_name = "restart-target"; + configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?; + + let (visible_targets, visible_arns) = wait_for_target_visibility(&env, target_name).await?; + assert!(notification_target_is_listed(&visible_targets, target_name)); + assert!( + visible_arns + .iter() + .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))), + "target ARN missing after initial configure: {visible_arns:?}" + ); + + restart_rustfs_server(&mut env).await?; + + let (targets_after_restart, arns_after_restart) = wait_for_target_visibility(&env, target_name).await?; + assert!(notification_target_is_listed(&targets_after_restart, target_name)); + assert!( + arns_after_restart + .iter() + .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))), + "target ARN missing after restart: {arns_after_restart:?}" + ); + + delete_webhook_target(&env, target_name).await?; + let (targets_after_delete, arns_after_delete) = wait_for_target_absence(&env, target_name).await?; + assert!(!notification_target_is_listed(&targets_after_delete, target_name)); + assert!( + !arns_after_delete + .iter() + .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))), + "target ARN still visible after delete: {arns_after_delete:?}" + ); + + restart_rustfs_server(&mut env).await?; + + let (targets_after_delete_restart, arns_after_delete_restart) = wait_for_target_absence(&env, target_name).await?; + assert!(!notification_target_is_listed(&targets_after_delete_restart, target_name)); + assert!( + !arns_after_delete_restart + .iter() + .any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))), + "target ARN still visible after delete + restart: {arns_after_delete_restart:?}" + ); + + webhook_handle.abort(); + let _ = webhook_handle.await; + + Ok(()) +} + #[tokio::test] #[serial] async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box> { diff --git a/crates/ecstore/src/config/audit.rs b/crates/ecstore/src/config/audit.rs index f0c864030..5574daeb4 100644 --- a/crates/ecstore/src/config/audit.rs +++ b/crates/ecstore/src/config/audit.rs @@ -16,8 +16,8 @@ use crate::config::{KV, KVS}; use rustfs_config::{ COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, - WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, - WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, + WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, + WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY, }; use std::sync::LazyLock; @@ -51,6 +51,16 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { value: "".to_owned(), hidden_if_empty: false, }, + KV { + key: WEBHOOK_CLIENT_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, KV { key: WEBHOOK_BATCH_SIZE.to_owned(), value: "1".to_owned(), @@ -81,6 +91,11 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { value: "5s".to_owned(), hidden_if_empty: false, }, + KV { + key: COMMENT_KEY.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, ]) }); diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 5885969b0..12a534cbd 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, oidc, storageclass}; +use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, storageclass}; use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::global::is_first_cluster_node_local; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use http::HeaderMap; +use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; +use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC}; use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION}; use rustfs_utils::path::SLASH_SEPARATOR; @@ -261,6 +263,159 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map) -> bool applied } +fn parse_notify_scalar_value(key: &str, value: &Value) -> Option { + match value { + Value::String(v) => Some(v.trim().to_string()), + Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v { + EnableState::On.to_string() + } else { + EnableState::Off.to_string() + }), + Value::Bool(v) => Some(v.to_string()), + Value::Number(v) => Some(v.to_string()), + Value::Null => None, + _ => None, + } +} + +fn decode_notify_instance_object(instance: &Map, valid_keys: &[&str]) -> KVS { + let mut kvs = KVS::new(); + + for (key, value) in instance { + if !valid_keys.contains(&key.as_str()) || key == COMMENT_KEY { + continue; + } + + if let Some(parsed) = parse_notify_scalar_value(key, value) { + kvs.insert(key.clone(), parsed); + } + } + + kvs +} + +fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option { + match value { + Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)), + Value::Array(_) => serde_json::from_value::(value.clone()).ok(), + _ => None, + } +} + +fn is_notify_instance_shorthand(section: &Map, valid_keys: &[&str]) -> bool { + section + .iter() + .any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some()) +} + +fn apply_external_notify_section( + cfg: &mut Config, + notify_obj: &Map, + external_key: &str, + subsystem_key: &str, + default_kvs: &KVS, + valid_keys: &[&str], +) -> bool { + let Some(Value::Object(section_obj)) = notify_obj.get(external_key).or_else(|| notify_obj.get(subsystem_key)) else { + return false; + }; + + if section_obj.is_empty() { + return false; + } + + let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default(); + let mut applied = false; + + if is_notify_instance_shorthand(section_obj, valid_keys) { + let kvs = decode_notify_instance_object(section_obj, valid_keys); + if !kvs.is_empty() { + let mut merged = default_kvs.clone(); + merged.extend(kvs); + subsystem.insert(DEFAULT_DELIMITER.to_string(), merged); + applied = true; + } + return applied; + } + + for (raw_instance, value) in section_obj { + let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else { + continue; + }; + if kvs.is_empty() { + continue; + } + + let instance_key = if raw_instance == "default" { + DEFAULT_DELIMITER.to_string() + } else { + raw_instance.to_string() + }; + + if instance_key == DEFAULT_DELIMITER { + let mut merged = default_kvs.clone(); + merged.extend(kvs); + kvs = merged; + } + + subsystem.insert(instance_key, kvs); + applied = true; + } + + applied +} + +fn apply_external_notify_map(cfg: &mut Config, root: &Map) -> bool { + let Some(Value::Object(notify_obj)) = root.get("notify") else { + return false; + }; + + let mut applied = false; + applied |= apply_external_notify_section( + cfg, + notify_obj, + "webhook", + NOTIFY_WEBHOOK_SUB_SYS, + ¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS, + NOTIFY_WEBHOOK_KEYS, + ); + applied |= apply_external_notify_section( + cfg, + notify_obj, + "mqtt", + NOTIFY_MQTT_SUB_SYS, + ¬ify::DEFAULT_NOTIFY_MQTT_KVS, + NOTIFY_MQTT_KEYS, + ); + applied +} + +fn apply_external_audit_map(cfg: &mut Config, root: &Map) -> bool { + let audit_root = root.get("audit").or_else(|| root.get("logger")).and_then(Value::as_object); + let Some(audit_obj) = audit_root else { + return false; + }; + + let mut applied = false; + applied |= apply_external_notify_section( + cfg, + audit_obj, + "webhook", + AUDIT_WEBHOOK_SUB_SYS, + &audit::DEFAULT_AUDIT_WEBHOOK_KVS, + AUDIT_WEBHOOK_KEYS, + ); + applied |= apply_external_notify_section( + cfg, + audit_obj, + "mqtt", + AUDIT_MQTT_SUB_SYS, + &audit::DEFAULT_AUDIT_MQTT_KVS, + AUDIT_MQTT_KEYS, + ); + applied +} + fn apply_external_storage_class_map(cfg: &mut Config, root: &Map) -> bool { let sc = root.get("storageclass").or_else(|| root.get("storage_class")); let Some(Value::Object(sc_obj)) = sc else { @@ -305,8 +460,10 @@ fn decode_server_config_blob(data: &[u8]) -> Result { let mut cfg = Config::new(); let has_storage = apply_external_storage_class_map(&mut cfg, &root); let has_oidc = apply_external_oidc_map(&mut cfg, &root); + let has_notify = apply_external_notify_map(&mut cfg, &root); + let has_audit = apply_external_audit_map(&mut cfg, &root); let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential"); - if !has_storage && !has_oidc && !has_header { + if !has_storage && !has_oidc && !has_notify && !has_audit && !has_header { return Err(Error::other("unrecognized external server config shape")); } Ok(cfg) @@ -449,6 +606,139 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map { oidc_obj } +fn is_notify_bool_key(key: &str) -> bool { + key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY +} + +fn encode_notify_scalar_value(key: &str, value: &str) -> Value { + if is_notify_bool_key(key) { + if let Ok(state) = value.parse::() { + return Value::Bool(state.is_enabled()); + } + if let Ok(boolean) = value.parse::() { + return Value::Bool(boolean); + } + } + + Value::String(value.to_string()) +} + +fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool { + default_kvs + .0 + .iter() + .find(|kv| kv.key == key) + .map(|kv| kv.hidden_if_empty) + .unwrap_or(false) +} + +fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map { + let mut instance = Map::new(); + + for key in valid_keys { + if *key == COMMENT_KEY { + continue; + } + + let baseline_value = baseline.lookup(key).unwrap_or_default(); + let effective_value = kvs.lookup(key).unwrap_or_else(|| baseline_value.clone()); + + if effective_value == baseline_value { + continue; + } + + if effective_value.trim().is_empty() && baseline_value.trim().is_empty() { + continue; + } + + if is_hidden_if_empty(default_kvs, key) && effective_value.trim().is_empty() && baseline_value.trim().is_empty() { + continue; + } + + instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value)); + } + + instance +} + +fn merged_notify_default_kvs(subsystem: &HashMap, default_kvs: &KVS) -> KVS { + let mut merged = default_kvs.clone(); + if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) { + merged.extend(kvs.clone()); + } + merged +} + +fn build_notify_subsystem_object( + cfg: &Config, + subsystem_key: &str, + default_kvs: &KVS, + valid_keys: &[&str], +) -> Map { + let Some(subsystem) = cfg.0.get(subsystem_key) else { + return Map::new(); + }; + + let effective_default = merged_notify_default_kvs(subsystem, default_kvs); + let mut subsystem_obj = Map::new(); + + if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) { + let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs); + if !default_obj.is_empty() { + subsystem_obj.insert("default".to_string(), Value::Object(default_obj)); + } + } + + let mut instances = subsystem + .iter() + .filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER) + .collect::>(); + instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs)); + + for (instance_key, kvs) in instances { + let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs); + if !instance_obj.is_empty() { + subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj)); + } + } + + subsystem_obj +} + +fn build_notify_object(cfg: &Config) -> Map { + let mut notify_obj = Map::new(); + + let webhook_obj = + build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, ¬ify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS); + if !webhook_obj.is_empty() { + notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj)); + } + + let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, ¬ify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS); + if !mqtt_obj.is_empty() { + notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj)); + } + + notify_obj +} + +fn build_audit_object(cfg: &Config) -> Map { + let mut audit_obj = Map::new(); + + let webhook_obj = + build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS); + if !webhook_obj.is_empty() { + audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj)); + } + + let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS); + if !mqtt_obj.is_empty() { + audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj)); + } + + audit_obj +} + fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result> { let mut root = seed.and_then(parse_object_seed).unwrap_or_default(); @@ -478,6 +768,73 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result v, + _ => Map::new(), + }; + let rendered_notify = build_notify_object(cfg); + match rendered_notify.get("webhook") { + Some(Value::Object(v)) => { + notify_obj.insert("webhook".to_string(), Value::Object(v.clone())); + notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS); + } + _ => { + notify_obj.remove("webhook"); + notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS); + } + } + match rendered_notify.get("mqtt") { + Some(Value::Object(v)) => { + notify_obj.insert("mqtt".to_string(), Value::Object(v.clone())); + notify_obj.remove(NOTIFY_MQTT_SUB_SYS); + } + _ => { + notify_obj.remove("mqtt"); + notify_obj.remove(NOTIFY_MQTT_SUB_SYS); + } + } + if notify_obj.is_empty() { + root.remove("notify"); + } else { + root.insert("notify".to_string(), Value::Object(notify_obj)); + } + root.remove(NOTIFY_WEBHOOK_SUB_SYS); + root.remove(NOTIFY_MQTT_SUB_SYS); + + let mut logger_obj = match root.remove("logger") { + Some(Value::Object(v)) => v, + _ => Map::new(), + }; + let rendered_audit = build_audit_object(cfg); + match rendered_audit.get("webhook") { + Some(Value::Object(v)) => { + logger_obj.insert("webhook".to_string(), Value::Object(v.clone())); + logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS); + } + _ => { + logger_obj.remove("webhook"); + logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS); + } + } + match rendered_audit.get("mqtt") { + Some(Value::Object(v)) => { + logger_obj.insert("mqtt".to_string(), Value::Object(v.clone())); + logger_obj.remove(AUDIT_MQTT_SUB_SYS); + } + _ => { + logger_obj.remove("mqtt"); + logger_obj.remove(AUDIT_MQTT_SUB_SYS); + } + } + if logger_obj.is_empty() { + root.remove("logger"); + } else { + root.insert("logger".to_string(), Value::Object(logger_obj)); + } + root.remove("audit"); + root.remove(AUDIT_WEBHOOK_SUB_SYS); + root.remove(AUDIT_MQTT_SUB_SYS); + Ok(serde_json::to_vec(&Value::Object(root))?) } @@ -496,6 +853,8 @@ fn is_standard_object_server_config(data: &[u8]) -> bool { fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool { build_storageclass_object(lhs) == build_storageclass_object(rhs) && build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs) + && build_notify_object(lhs) == build_notify_object(rhs) + && build_audit_object(lhs) == build_audit_object(rhs) } fn is_object_not_found(err: &Error) -> bool { @@ -712,7 +1071,7 @@ mod tests { configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config, read_config_with_metadata, storage_class_kvs_mut, }; - use crate::config::{Config, oidc}; + use crate::config::{Config, audit, notify, oidc}; use crate::disk::endpoint::Endpoint; use crate::endpoints::SetupType; use crate::error::{Error, Result}; @@ -725,6 +1084,8 @@ mod tests { ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions, }; use http::HeaderMap; + use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; + use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; use rustfs_filemeta::FileInfo; @@ -1332,6 +1693,152 @@ mod tests { ); } + #[test] + fn test_decode_server_config_reads_notify_targets() { + let input = r#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1"}, + "notify":{ + "webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://example.com/hook", + "queue_dir":"/tmp/webhook-queue" + } + }, + "mqtt":{ + "default":{ + "enable":true, + "topic":"events" + }, + "analytics":{ + "enable":true, + "broker":"tcp://127.0.0.1:1883", + "topic":"events", + "queue_dir":"" + } + } + } + }"#; + + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + + let webhook = cfg + .get_value(NOTIFY_WEBHOOK_SUB_SYS, "primary") + .expect("webhook target should be decoded"); + assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/hook"); + assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/webhook-queue"); + + let mqtt_default = cfg + .get_value(NOTIFY_MQTT_SUB_SYS, DEFAULT_DELIMITER) + .expect("mqtt default should be decoded"); + assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "events"); + assert_eq!( + mqtt_default.get(rustfs_config::MQTT_QUEUE_DIR), + notify::DEFAULT_NOTIFY_MQTT_KVS.get(rustfs_config::MQTT_QUEUE_DIR) + ); + + let mqtt = cfg + .get_value(NOTIFY_MQTT_SUB_SYS, "analytics") + .expect("mqtt target should be decoded"); + assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883"); + assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), ""); + } + + #[test] + fn test_decode_server_config_reads_notify_shorthand_default() { + let input = r#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1"}, + "notify":{ + "webhook":{ + "enable":true, + "endpoint":"https://example.com/shorthand" + } + } + }"#; + + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + let webhook_default = cfg + .get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER) + .expect("default webhook config should be decoded"); + assert_eq!(webhook_default.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(webhook_default.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/shorthand"); + } + + #[test] + fn test_decode_server_config_keeps_instance_named_like_field() { + let input = r#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1"}, + "notify":{ + "webhook":{ + "enable":{ + "enable":true, + "endpoint":"https://example.com/instance-enable" + } + } + } + }"#; + + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + let named = cfg + .get_value(NOTIFY_WEBHOOK_SUB_SYS, "enable") + .expect("instance named 'enable' should be decoded"); + assert_eq!(named.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(named.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/instance-enable"); + } + + #[test] + fn test_decode_server_config_reads_audit_targets() { + let input = r#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1"}, + "logger":{ + "webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://example.com/audit-hook", + "queue_dir":"/tmp/audit-queue" + } + }, + "mqtt":{ + "default":{ + "enable":true, + "topic":"audit-events" + }, + "analytics":{ + "enable":true, + "broker":"tcp://127.0.0.1:1883", + "topic":"audit-events" + } + } + } + }"#; + + let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed"); + + let webhook = cfg + .get_value(AUDIT_WEBHOOK_SUB_SYS, "primary") + .expect("audit webhook target should be decoded"); + assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/audit-hook"); + assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/audit-queue"); + + let mqtt_default = cfg + .get_value(AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER) + .expect("audit mqtt default should be decoded"); + assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string()); + assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "audit-events"); + + let mqtt = cfg + .get_value(AUDIT_MQTT_SUB_SYS, "analytics") + .expect("audit mqtt target should be decoded"); + assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883"); + } + #[test] fn test_encode_server_config_writes_external_object_shape() { let mut cfg = Config::new(); @@ -1388,6 +1895,174 @@ mod tests { assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); } + #[test] + fn test_encode_server_config_writes_notify_object_shape() { + let mut cfg = Config::new(); + let mut webhook_section = std::collections::HashMap::new(); + webhook_section.insert(DEFAULT_DELIMITER.to_string(), notify::DEFAULT_NOTIFY_WEBHOOK_KVS.clone()); + webhook_section.insert( + "primary".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/hook".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(), + value: "/tmp/webhook-queue".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_section); + + let mut mqtt_default = notify::DEFAULT_NOTIFY_MQTT_KVS.clone(); + mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "events".to_string()); + let mut mqtt_section = std::collections::HashMap::new(); + mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default); + mqtt_section.insert( + "analytics".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::MQTT_BROKER.to_string(), + value: "tcp://127.0.0.1:1883".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::MQTT_QUEUE_DIR.to_string(), + value: "".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section); + + let out = encode_server_config_blob(&cfg, None).expect("encode should succeed"); + let v: Value = serde_json::from_slice(&out).expect("output should be json"); + let notify = v + .get("notify") + .and_then(Value::as_object) + .expect("notify object should be present"); + let webhook = notify + .get("webhook") + .and_then(Value::as_object) + .and_then(|targets| targets.get("primary")) + .and_then(Value::as_object) + .expect("webhook target should be encoded"); + assert_eq!( + webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str), + Some("https://example.com/hook") + ); + assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); + + let mqtt_default = notify + .get("mqtt") + .and_then(Value::as_object) + .and_then(|targets| targets.get("default")) + .and_then(Value::as_object) + .expect("mqtt default should be encoded"); + assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); + assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("events")); + + let mqtt = notify + .get("mqtt") + .and_then(Value::as_object) + .and_then(|targets| targets.get("analytics")) + .and_then(Value::as_object) + .expect("mqtt target should be encoded"); + assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883")); + assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some("")); + } + + #[test] + fn test_encode_server_config_writes_audit_object_shape() { + let mut cfg = Config::new(); + let mut webhook_section = std::collections::HashMap::new(); + webhook_section.insert(DEFAULT_DELIMITER.to_string(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone()); + webhook_section.insert( + "primary".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/audit-hook".to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(), + value: "/tmp/audit-queue".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), webhook_section); + + let mut mqtt_default = audit::DEFAULT_AUDIT_MQTT_KVS.clone(); + mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "audit-events".to_string()); + let mut mqtt_section = std::collections::HashMap::new(); + mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default); + mqtt_section.insert( + "analytics".to_string(), + crate::config::KVS(vec![ + crate::config::KV { + key: ENABLE_KEY.to_string(), + value: EnableState::On.to_string(), + hidden_if_empty: false, + }, + crate::config::KV { + key: rustfs_config::MQTT_BROKER.to_string(), + value: "tcp://127.0.0.1:1883".to_string(), + hidden_if_empty: false, + }, + ]), + ); + cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section); + + let out = encode_server_config_blob(&cfg, None).expect("encode should succeed"); + let v: Value = serde_json::from_slice(&out).expect("output should be json"); + let logger = v + .get("logger") + .and_then(Value::as_object) + .expect("logger object should be present"); + let webhook = logger + .get("webhook") + .and_then(Value::as_object) + .and_then(|targets| targets.get("primary")) + .and_then(Value::as_object) + .expect("audit webhook target should be encoded"); + assert_eq!( + webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str), + Some("https://example.com/audit-hook") + ); + assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); + + let mqtt_default = logger + .get("mqtt") + .and_then(Value::as_object) + .and_then(|targets| targets.get("default")) + .and_then(Value::as_object) + .expect("audit mqtt default should be encoded"); + assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true)); + assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events")); + } + #[test] fn test_is_standard_object_server_config_detection() { let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#; @@ -1441,6 +2116,113 @@ mod tests { assert!(configs_semantically_equal(&lhs, &rhs)); } + #[test] + fn test_configs_semantically_equal_accounts_for_notify() { + let external = br#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"}, + "notify":{ + "webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://example.com/hook" + } + } + } + }"#; + let legacy = br#"{ + "storage_class":{"_":[ + {"key":"standard","value":"EC:2"}, + {"key":"rrs","value":"EC:1"}, + {"key":"optimize","value":"availability"} + ]}, + "notify_webhook":{ + "_":[ + {"key":"enable","value":"off"}, + {"key":"endpoint","value":""}, + {"key":"queue_limit","value":"100000"}, + {"key":"queue_dir","value":"/opt/rustfs/events"}, + {"key":"client_cert","value":""}, + {"key":"client_key","value":""}, + {"key":"comment","value":""}, + {"key":"client_ca","value":""}, + {"key":"skip_tls_verify","value":"off"} + ], + "primary":[ + {"key":"enable","value":"on"}, + {"key":"endpoint","value":"https://example.com/hook"} + ] + } + }"#; + + let lhs = decode_server_config_blob(external).expect("decode external"); + let rhs = decode_server_config_blob(legacy).expect("decode legacy"); + assert!(configs_semantically_equal(&lhs, &rhs)); + } + + #[test] + fn test_configs_semantically_equal_detects_notify_changes() { + let lhs = decode_server_config_blob( + br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/a"}}}}"#, + ) + .expect("decode lhs"); + let rhs = decode_server_config_blob( + br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/b"}}}}"#, + ) + .expect("decode rhs"); + + assert!(!configs_semantically_equal(&lhs, &rhs)); + } + + #[test] + fn test_configs_semantically_equal_accounts_for_audit() { + let external = br#"{ + "version":"33", + "storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"}, + "logger":{ + "webhook":{ + "primary":{ + "enable":true, + "endpoint":"https://example.com/audit-hook" + } + } + } + }"#; + let legacy = br#"{ + "storage_class":{"_":[ + {"key":"standard","value":"EC:2"}, + {"key":"rrs","value":"EC:1"}, + {"key":"optimize","value":"availability"} + ]}, + "audit_webhook":{ + "_":[ + {"key":"enable","value":"off"}, + {"key":"endpoint","value":""}, + {"key":"auth_token","value":""}, + {"key":"client_cert","value":""}, + {"key":"client_key","value":""}, + {"key":"client_ca","value":""}, + {"key":"skip_tls_verify","value":"off"}, + {"key":"batch_size","value":"1"}, + {"key":"queue_limit","value":"100000"}, + {"key":"queue_dir","value":"/opt/rustfs/events"}, + {"key":"max_retry","value":"0"}, + {"key":"retry_interval","value":"3s"}, + {"key":"http_timeout","value":"5s"}, + {"key":"comment","value":""} + ], + "primary":[ + {"key":"enable","value":"on"}, + {"key":"endpoint","value":"https://example.com/audit-hook"} + ] + } + }"#; + + let lhs = decode_server_config_blob(external).expect("decode external"); + let rhs = decode_server_config_blob(legacy).expect("decode legacy"); + assert!(configs_semantically_equal(&lhs, &rhs)); + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn test_read_config_with_metadata_succeeds_with_one_healthy_locker_in_two_node_dist_setup() { diff --git a/crates/ecstore/src/config/notify.rs b/crates/ecstore/src/config/notify.rs index c9ebf3ba6..5f3bb9442 100644 --- a/crates/ecstore/src/config/notify.rs +++ b/crates/ecstore/src/config/notify.rs @@ -16,7 +16,8 @@ use crate::config::{KV, KVS}; use rustfs_config::{ COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, - WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, + WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, + WEBHOOK_SKIP_TLS_VERIFY, }; use std::sync::LazyLock; @@ -60,6 +61,16 @@ pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock = LazyLock::new(|| { value: "".to_owned(), hidden_if_empty: false, }, + KV { + key: WEBHOOK_CLIENT_CA.to_owned(), + value: "".to_owned(), + hidden_if_empty: false, + }, + KV { + key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(), + value: EnableState::Off.to_string(), + hidden_if_empty: false, + }, KV { key: COMMENT_KEY.to_owned(), value: "".to_owned(), diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index b2d9feff5..3ac6c96d6 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -44,6 +44,7 @@ regex.workspace = true [dev-dependencies] criterion = { workspace = true } +tempfile = { workspace = true } [[bench]] name = "xl_meta_bench" diff --git a/crates/filemeta/src/filemeta.rs b/crates/filemeta/src/filemeta.rs index 2ed9d8e1f..33eee875a 100644 --- a/crates/filemeta/src/filemeta.rs +++ b/crates/filemeta/src/filemeta.rs @@ -1907,7 +1907,6 @@ mod test { #[tokio::test] async fn test_read_xl_meta_no_data() { - use tokio::fs; use tokio::fs::File; use tokio::io::AsyncWriteExt; @@ -1926,13 +1925,15 @@ async fn test_read_xl_meta_no_data() { buff.resize(buff.len() + 100, 0); - let filepath = "./test_xl.meta"; + // Use tempfile to avoid conflicts with parallel tests or previous runs + let dir = tempfile::tempdir().unwrap(); + let filepath = dir.path().join("test_xl.meta"); - let mut file = File::create(filepath).await.unwrap(); + let mut file = File::create(&filepath).await.unwrap(); // Write string data file.write_all(&buff).await.unwrap(); - let mut f = File::open(filepath).await.unwrap(); + let mut f = File::open(&filepath).await.unwrap(); let stat = f.metadata().await.unwrap(); @@ -1941,7 +1942,5 @@ async fn test_read_xl_meta_no_data() { let mut newfm = FileMeta::default(); newfm.unmarshal_msg(&data).unwrap(); - fs::remove_file(filepath).await.unwrap(); - assert_eq!(fm, newfm) } diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 89657f4c4..4ac5087e7 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -18,12 +18,14 @@ use crate::{ Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream, }; use hashbrown::HashMap; -use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY}; +use rustfs_config::notify::{ + DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, +}; use rustfs_ecstore::config::{Config, KVS}; use rustfs_s3_common::EventName; use rustfs_targets::arn::TargetID; use rustfs_targets::store::{Key, Store}; -use rustfs_targets::target::EntityTarget; +use rustfs_targets::target::QueuedPayload; use rustfs_targets::{StoreError, Target}; use std::collections::VecDeque; use std::sync::Arc; @@ -34,6 +36,21 @@ use tracing::{debug, error, info, warn}; const MAX_RECENT_LIVE_EVENTS: usize = 1024; +fn subsystem_target_type(target_type: &str) -> &str { + match target_type { + NOTIFY_WEBHOOK_SUB_SYS => "webhook", + NOTIFY_MQTT_SUB_SYS => "mqtt", + _ => target_type, + } +} + +fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID { + TargetID { + id: target_name.to_lowercase(), + name: subsystem_target_type(target_type).to_string(), + } +} + #[derive(Clone)] pub struct LiveEventBatch { pub events: Vec>, @@ -183,58 +200,84 @@ impl NotificationSystem { } } + /// Initializes targets and starts event streams for those with stores. + /// Returns a map of (target_id -> cancel_sender) for streams that were started. + async fn init_targets_and_start_streams( + &self, + targets: &[Box + Send + Sync>], + ) -> HashMap> { + let mut cancellers = HashMap::new(); + for target in targets { + let target_id = target.id(); + info!("Initializing target: {}", target_id); + + let has_store = target.store().is_some(); + + if let Err(e) = target.init().await { + warn!("Target {} Initialization failed: {}", target_id, e); + // For targets without a store, init failure is fatal — skip. + // For store-backed targets, still start the stream so queued events + // can be drained when connectivity recovers (send_from_store retries). + if !has_store { + continue; + } + warn!( + "Target {} has a store, starting stream despite init failure — \ + connectivity will be retried by send_from_store", + target_id + ); + } else { + debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled()); + } + + if !target.is_enabled() { + info!("Target {} is not enabled, event stream processing is skipped", target_id); + continue; + } + + if let Some(store) = target.store() { + info!("Start event stream processing for target {}", target_id); + + let store_clone = store.boxed_clone(); + let target_arc = Arc::from(target.clone_dyn()); + + let cancel_tx = self.enhanced_start_event_stream( + store_clone, + target_arc, + self.metrics.clone(), + self.concurrency_limiter.clone(), + ); + + let target_id_clone = target_id.clone(); + cancellers.insert(target_id, cancel_tx); + info!("Event stream processing for target {} is started successfully", target_id_clone); + } else { + info!("Target {} No storage is configured, event stream processing is skipped", target_id); + } + } + cancellers + } + /// Initializes the notification system pub async fn init(&self) -> Result<(), NotificationError> { info!("Initialize notification system..."); - let config = self.config.read().await; - debug!("Initializing notification system with config: {:?}", *config); + let config = { + let guard = self.config.read().await; + debug!("Initializing notification system with config: {:?}", *guard); + guard.clone() + }; + let targets: Vec + Send + Sync>> = self.registry.create_targets_from_config(&config).await?; info!("{} notification targets were created", targets.len()); - // Initiate event stream processing for each storage enabled target - let mut cancellers = HashMap::new(); - for target in &targets { - let target_id = target.id(); - info!("Initializing target: {}", target.id()); - // Initialize the target - if let Err(e) = target.init().await { - warn!("Target {} Initialization failed:{}", target.id(), e); - continue; - } - debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled()); - // Check if the target is enabled and has storage - if target.is_enabled() { - if let Some(store) = target.store() { - info!("Start event stream processing for target {}", target.id()); + // Initialize targets and start event streams + let cancellers = self.init_targets_and_start_streams(&targets).await; - // The storage of the cloned target and the target itself - let store_clone = store.boxed_clone(); - let target_box = target.clone_dyn(); - let target_arc = Arc::from(target_box); - - // Add a reference to the monitoring metrics - let metrics = self.metrics.clone(); - let semaphore = self.concurrency_limiter.clone(); - - // Encapsulated enhanced version of start_event_stream - let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore); - - // Start event stream processing and save cancel sender - let target_id_clone = target_id.clone(); - cancellers.insert(target_id, cancel_tx); - info!("Event stream processing for target {} is started successfully", target_id_clone); - } else { - info!("Target {} No storage is configured, event stream processing is skipped", target_id); - } - } else { - info!("Target {} is not enabled, event stream processing is skipped", target_id); - } - } - - // Update canceler collection + // Update canceller collection *self.stream_cancellers.write().await = cancellers; + // Initialize the bucket target self.notifier.init_bucket_targets(targets).await?; info!("Notification system initialized"); @@ -333,7 +376,7 @@ impl NotificationSystem { info!("Attempting to remove target: {}", target_id); let ttype = target_type.to_lowercase(); - let tname = target_id.name.to_lowercase(); + let tname = target_id.id.to_lowercase(); self.update_config_and_reload(|config| { let mut changed = false; @@ -405,11 +448,7 @@ impl NotificationSystem { let ttype = target_type.to_lowercase(); let tname = target_name.to_lowercase(); - - let target_id = TargetID { - id: tname.clone(), - name: ttype.clone(), - }; + let target_id = runtime_target_id_for_subsystem(&ttype, &tname); // Deletion is prohibited if bucket rules refer to it if self.notifier.is_target_bound_to_any_bucket(&target_id).await { @@ -451,7 +490,7 @@ impl NotificationSystem { /// Enhanced event stream startup function, including monitoring and concurrency control fn enhanced_start_event_stream( &self, - store: Box, Error = StoreError, Key = Key> + Send>, + store: Box + Send>, target: Arc + Send + Sync>, metrics: Arc, semaphore: Arc, @@ -476,14 +515,13 @@ impl NotificationSystem { let _ = cancel_tx.send(()).await; } - // Clear the target_list and ensure that reload is a replacement reconstruction (solve the target_list len unchanged/residual problem) + // Clear the target_list and ensure that reload is a replacement reconstruction self.notifier.remove_all_bucket_targets().await; // Update the config self.update_config(new_config.clone()).await; - // Create a new target from configuration - // This function will now be responsible for merging env, creating and persisting the final configuration. + // Create new targets from configuration let targets: Vec + Send + Sync>> = self .registry .create_targets_from_config(&new_config) @@ -492,46 +530,8 @@ impl NotificationSystem { info!("{} notification targets were created from the new configuration", targets.len()); - // Start new event stream processing for each storage enabled target - let mut new_cancellers = HashMap::new(); - for target in &targets { - let target_id = target.id(); - - // Initialize the target - if let Err(e) = target.init().await { - error!("Target {} Initialization failed:{}", target_id, e); - continue; - } - // Check if the target is enabled and has storage - if target.is_enabled() { - if let Some(store) = target.store() { - info!("Start new event stream processing for target {}", target_id); - - // The storage of the cloned target and the target itself - let store_clone = store.boxed_clone(); - // let target_box = target.clone_dyn(); - let target_arc = Arc::from(target.clone_dyn()); - - // Encapsulated enhanced version of start_event_stream - let cancel_tx = self.enhanced_start_event_stream( - store_clone, - target_arc, - self.metrics.clone(), - self.concurrency_limiter.clone(), - ); - - // Start event stream processing and save cancel sender - // let cancel_tx = start_event_stream(store_clone, target_clone); - let target_id_clone = target_id.clone(); - new_cancellers.insert(target_id, cancel_tx); - info!("Event stream processing of target {} is restarted successfully", target_id_clone); - } else { - info!("Target {} No storage is configured, event stream processing is skipped", target_id); - } - } else { - info!("Target {} disabled, event stream processing is skipped", target_id); - } - } + // Initialize targets and start event streams using shared helper + let new_cancellers = self.init_targets_and_start_streams(&targets).await; // Update canceler collection *cancellers = new_cancellers; @@ -665,4 +665,18 @@ mod tests { assert_eq!(batch.events.len(), 1); assert_eq!(batch.events[0].s3.object.key, "one"); } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "webhook"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics"); + assert_eq!(target_id.id, "analytics"); + assert_eq!(target_id.name, "mqtt"); + } } diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index 57c1febb2..4d6fe40c3 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -399,7 +399,7 @@ mod tests { use rustfs_targets::{ TargetError, store::{Key, Store}, - target::EntityTarget, + target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}, }; use serde::{Serialize, de::DeserializeOwned}; use std::sync::{ @@ -442,7 +442,7 @@ mod tests { Ok(()) } - async fn send_from_store(&self, _key: Key) -> Result<(), TargetError> { + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { Ok(()) } @@ -450,7 +450,7 @@ mod tests { Ok(()) } - fn store(&self) -> Option<&(dyn Store, Error = StoreError, Key = Key> + Send + Sync)> { + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { None } diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index 74ccbdfb9..6b5e3e515 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -89,9 +89,6 @@ impl TargetRegistry { let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect(); // A collection of asynchronous tasks for concurrently executing target creation let mut tasks = FuturesUnordered::new(); - // let final_config = config.clone(); // Clone a configuration for aggregating the final result - // Record the defaults for each segment so that the segment can eventually be rebuilt - let mut section_defaults: HashMap = HashMap::new(); // 1. Traverse all registered plants and process them by target type for (target_type, factory) in &self.factories { tracing::Span::current().record("target_type", target_type.as_str()); @@ -105,9 +102,6 @@ impl TargetRegistry { let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default(); debug!(?default_cfg, "Get the default configuration"); - // Save defaults for eventual write back - section_defaults.insert(section_name.clone(), default_cfg.clone()); - // *** Optimization point 1: Get all legitimate fields of the current target type *** let valid_fields = factory.get_valid_fields(); debug!(?valid_fields, "Get the legitimate configuration fields"); @@ -215,110 +209,28 @@ impl TargetRegistry { if enabled { info!(instance_id = %id, "Target is enabled, ready to create a task"); // 5.3. Create asynchronous tasks for enabled instances - let target_type_clone = target_type.clone(); let tid = id.clone(); let merged_config_arc = Arc::new(merged_config); tasks.push(async move { let result = factory.create_target(tid.clone(), &merged_config_arc).await; - (target_type_clone, tid, result, Arc::clone(&merged_config_arc)) + (tid, result) }); } else { - info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration"); - // Remove disabled target from final configuration - // final_config.0.entry(section_name.clone()).or_default().remove(&id); + info!(instance_id = %id, "Skip disabled target"); } } } // 6. Concurrently execute all creation tasks and collect results let mut successful_targets = Vec::new(); - let mut successful_configs = Vec::new(); - while let Some((target_type, id, result, final_config)) = tasks.next().await { + while let Some((id, result)) = tasks.next().await { match result { Ok(target) => { - info!(target_type = %target_type, instance_id = %id, "Create a target successfully"); + info!(instance_id = %id, "Create target successfully"); successful_targets.push(target); - successful_configs.push((target_type, id, final_config)); } Err(e) => { - error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target"); - } - } - } - - // 7. Aggregate new configuration and write back to system configuration - if !successful_configs.is_empty() || !section_defaults.is_empty() { - info!( - "Prepare to update {} successfully created target configurations to the system configuration...", - successful_configs.len() - ); - - let mut successes_by_section: HashMap> = HashMap::new(); - - for (target_type, id, kvs) in successful_configs { - let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase(); - successes_by_section - .entry(section_name) - .or_default() - .insert(id.to_lowercase(), (*kvs).clone()); - } - - let mut new_config = config.clone(); - // Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist - let mut sections: HashSet = HashSet::new(); - sections.extend(section_defaults.keys().cloned()); - sections.extend(successes_by_section.keys().cloned()); - - for section in sections { - let mut section_map: std::collections::HashMap = std::collections::HashMap::new(); - // Add default item - if let Some(default_kvs) = section_defaults.get(§ion) - && !default_kvs.is_empty() - { - section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone()); - } - - // Add successful instance item - if let Some(instances) = successes_by_section.get(§ion) { - for (id, kvs) in instances { - section_map.insert(id.clone(), kvs.clone()); - } - } - - // Empty breaks are removed and non-empty breaks are replaced entirely. - if section_map.is_empty() { - new_config.0.remove(§ion); - } else { - new_config.0.insert(section, section_map); - } - } - - if &new_config == config { - info!("Notification target configuration unchanged, skip persisting server config"); - info!(count = successful_targets.len(), "All target processing completed"); - return Ok(successful_targets); - } - - let store = match rustfs_ecstore::global::new_object_layer_fn() { - Some(s) => s, - None => { - warn!( - "Object store not available at notification init; skipping config persistence. \ - {} target(s) active in memory.", - successful_targets.len() - ); - info!(count = successful_targets.len(), "All target processing completed"); - return Ok(successful_targets); - } - }; - - match rustfs_ecstore::config::com::save_server_config(store, &new_config).await { - Ok(_) => { - info!("The new configuration was saved to the system successfully.") - } - Err(e) => { - error!("Failed to save the new configuration: {}", e); - return Err(TargetError::SaveConfig(e.to_string())); + error!(instance_id = %id, error = %e, "Failed to create target"); } } } diff --git a/crates/notify/src/stream.rs b/crates/notify/src/stream.rs index bbb784a87..597d824e1 100644 --- a/crates/notify/src/stream.rs +++ b/crates/notify/src/stream.rs @@ -15,8 +15,8 @@ use crate::{Event, integration::NotificationMetrics}; use rustfs_targets::{ StoreError, Target, TargetError, - store::{Key, Store}, - target::EntityTarget, + store::{Key, Store, ensure_store_entry_raw_readable}, + target::QueuedPayload, }; use rustfs_utils::get_env_usize; use std::sync::Arc; @@ -32,7 +32,7 @@ use tracing::{debug, error, info, warn}; /// - `target`: The target to send events to /// - `cancel_rx`: Receiver to listen for cancellation signals pub async fn stream_events( - store: &mut (dyn Store + Send), + store: &mut (dyn Store + Send), target: &dyn Target, mut cancel_rx: mpsc::Receiver<()>, ) { @@ -119,7 +119,7 @@ pub async fn stream_events( /// # Returns /// A sender to signal cancellation of the event stream pub fn start_event_stream( - mut store: Box + Send>, + mut store: Box + Send>, target: Arc + Send + Sync>, ) -> mpsc::Sender<()> { let (cancel_tx, cancel_rx) = mpsc::channel(1); @@ -143,7 +143,7 @@ pub fn start_event_stream( /// # Returns /// A sender to signal cancellation of the event stream pub fn start_event_stream_with_batching( - mut store: Box, Error = StoreError, Key = Key> + Send>, + mut store: Box + Send>, target: Arc + Send + Sync>, metrics: Arc, semaphore: Arc, @@ -170,7 +170,7 @@ pub fn start_event_stream_with_batching( /// # Notes /// This function processes events in batches to improve efficiency. pub async fn stream_events_with_batching( - store: &mut (dyn Store, Error = StoreError, Key = Key> + Send), + store: &mut (dyn Store + Send), target: &dyn Target, mut cancel_rx: mpsc::Receiver<()>, metrics: Arc, @@ -185,7 +185,6 @@ pub async fn stream_events_with_batching( const MAX_RETRIES: usize = 5; const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); - let mut batch: Vec> = Vec::with_capacity(batch_size); let mut batch_keys = Vec::with_capacity(batch_size); let mut last_flush = Instant::now(); @@ -201,8 +200,8 @@ pub async fn stream_events_with_batching( debug!("Found {} keys in store for target: {}", keys.len(), target.name()); if keys.is_empty() { // If there is data in the batch and timeout, refresh the batch - if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT { - process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; + if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT { + process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; last_flush = Instant::now(); } @@ -218,41 +217,31 @@ pub async fn stream_events_with_batching( info!("Cancellation received during processing for target: {}", target.name()); // Processing collected batches before exiting - if !batch.is_empty() { - process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; + if !batch_keys.is_empty() { + process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; } return; } - // Try to get events from storage - match store.get(&key) { - Ok(event) => { - // Add to batch - batch.push(event); - batch_keys.push(key); - metrics.increment_processing(); - - // If the batch is full or enough time has passed since the last refresh, the batch will be processed - if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT { - process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore) - .await; - last_flush = Instant::now(); - } + // Skip unreadable entries so a single corrupt file cannot stall the stream. + // ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to + // remove the corrupt entry before returning Err, so no cleanup is needed here. + match ensure_store_entry_raw_readable(&*store, &key) { + Ok(true) => {} // entry is readable, proceed + Ok(false) => continue, // entry not found (already removed), skip + Err(err) => { + warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err); + continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable } - Err(e) => { - error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e); - // Consider deleting unreadable events to prevent infinite loops from trying to read - match store.del(&key) { - Ok(_) => { - info!("Deleted corrupted event {} from store", key.to_string()); - } - Err(del_err) => { - error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err); - } - } + } - metrics.increment_failed(); - } + batch_keys.push(key); + metrics.increment_processing(); + + // If the batch is full or enough time has passed since the last refresh, the batch will be processed + if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT { + process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; + last_flush = Instant::now(); } } @@ -273,7 +262,6 @@ pub async fn stream_events_with_batching( /// # Notes /// This function processes a batch of events, sending each event to the target with retry async fn process_batch( - batch: &mut Vec>, batch_keys: &mut Vec, target: &dyn Target, max_retries: usize, @@ -281,8 +269,8 @@ async fn process_batch( metrics: &Arc, semaphore: &Arc, ) { - debug!("Processing batch of {} events for target: {}", batch.len(), target.name()); - if batch.is_empty() { + debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name()); + if batch_keys.is_empty() { return; } @@ -296,44 +284,42 @@ async fn process_batch( }; // Handle every event in the batch - for (_event, key) in batch.iter().zip(batch_keys.iter()) { + for key in batch_keys.iter() { let mut retry_count = 0; let mut success = false; // Retry logic while retry_count < max_retries && !success { - // After sending successfully, the event in the storage is deleted synchronously. match target.send_from_store(key.clone()).await { Ok(_) => { - info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string()); + debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string()); success = true; metrics.increment_processed(); } - Err(e) => { - // Different retry strategies are adopted according to the error type - match &e { - TargetError::NotConnected => { - warn!("Target {} not connected, retrying...", target.name()); - retry_count += 1; - tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff - } - TargetError::Timeout(_) => { - warn!("Timeout for target {}, retrying...", target.name()); - retry_count += 1; - tokio::time::sleep(base_delay * (1 << retry_count)).await; - } - _ => { - // Permanent error, skip this event - error!("Permanent error for target {}: {}", target.name(), e); - metrics.increment_failed(); - break; - } + Err(e) => match &e { + TargetError::NotConnected => { + warn!("Target {} not connected, retrying...", target.name()); + retry_count += 1; + let jitter = Duration::from_millis(key.to_string().len() as u64 % 500); + let backoff = 1u32 << retry_count as u32; + tokio::time::sleep(base_delay * backoff + jitter).await; } - } + TargetError::Timeout(_) => { + warn!("Timeout for target {}, retrying...", target.name()); + retry_count += 1; + let jitter = Duration::from_millis(key.to_string().len() as u64 % 500); + let backoff = 1u32 << retry_count as u32; + tokio::time::sleep(base_delay * backoff + jitter).await; + } + _ => { + error!("Permanent error for target {}: {}", target.name(), e); + metrics.increment_failed(); + break; + } + }, } } - // Handle the situation where the maximum number of retry exhaustion is exhausted if retry_count >= max_retries && !success { warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name()); metrics.increment_failed(); @@ -341,7 +327,6 @@ async fn process_batch( } // Clear processed batches - batch.clear(); batch_keys.clear(); // Release semaphore permission (via drop) diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index eb8d157dc..65fe2079c 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -28,5 +28,12 @@ url = { workspace = true } urlencoding = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } +[dev-dependencies] +criterion = { workspace = true } + +[[bench]] +name = "queue_store_benchmark" +harness = false + [lints] workspace = true diff --git a/crates/targets/benches/queue_store_benchmark.rs b/crates/targets/benches/queue_store_benchmark.rs new file mode 100644 index 000000000..205c3bc4a --- /dev/null +++ b/crates/targets/benches/queue_store_benchmark.rs @@ -0,0 +1,94 @@ +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use rustfs_targets::store::{QueueStore, Store}; +use serde::{Deserialize, Serialize}; +use std::hint::black_box; +use std::sync::Arc; +use uuid::Uuid; + +#[derive(Clone, Serialize, Deserialize)] +struct BenchEvent { + bucket: String, + key: String, + metadata: Vec<(String, String)>, + payload: String, +} + +fn bench_dir(prefix: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("rustfs-targets-bench-{prefix}-{}", Uuid::new_v4())) +} + +fn build_payload(payload_len: usize) -> Vec { + let event = BenchEvent { + bucket: "bench-bucket".to_string(), + key: format!("objects/{payload_len}/file.json"), + metadata: (0..8) + .map(|idx| (format!("x-amz-meta-{idx}"), "benchmark-value".repeat(4))) + .collect(), + payload: "abcdefghijklmnopqrstuvwxyz0123456789".repeat(payload_len / 36 + 1)[..payload_len].to_string(), + }; + serde_json::to_vec(&event).unwrap() +} + +fn queue_store_write_benchmark(c: &mut Criterion) { + let mut group = c.benchmark_group("queue_store_put_raw"); + + for payload_size in [512usize, 8 * 1024, 64 * 1024] { + let payload = build_payload(payload_size); + group.throughput(Throughput::Bytes(payload.len() as u64)); + + for compress in [false, true] { + let dir = bench_dir(if compress { "put-compress" } else { "put-plain" }); + let store = QueueStore::::new_with_compression(&dir, 100_000, ".bench", compress); + store.open().unwrap(); + + group.bench_with_input( + BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size), + &payload, + |b, payload| { + b.iter(|| { + let key = store.put_raw(payload).unwrap(); + store.del(&key).unwrap(); + }); + }, + ); + + let _ = store.delete(); + } + } + + group.finish(); +} + +fn queue_store_read_benchmark(c: &mut Criterion) { + let mut group = c.benchmark_group("queue_store_get_raw"); + + for payload_size in [512usize, 8 * 1024, 64 * 1024] { + let payload = build_payload(payload_size); + group.throughput(Throughput::Bytes(payload.len() as u64)); + + for compress in [false, true] { + let dir = bench_dir(if compress { "get-compress" } else { "get-plain" }); + let store = Arc::new(QueueStore::::new_with_compression(&dir, 100_000, ".bench", compress)); + store.open().unwrap(); + let key = store.put_raw(&payload).unwrap(); + + group.bench_with_input( + BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size), + &(Arc::clone(&store), key), + |b, (store, key)| { + b.iter(|| { + let raw = store.get_raw(key).unwrap(); + black_box(raw); + }); + }, + ); + + let _ = store.delete(); + } + } + + group.finish(); +} + +criterion_group!(benches, queue_store_write_benchmark, queue_store_read_benchmark); +criterion_main!(benches); diff --git a/crates/targets/src/store.rs b/crates/targets/src/store.rs index ee3a616f1..b568f9ba8 100644 --- a/crates/targets/src/store.rs +++ b/crates/targets/src/store.rs @@ -13,20 +13,34 @@ // limitations under the License. use crate::error::StoreError; -use rustfs_config::DEFAULT_LIMIT; use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT}; +use rustfs_config::{DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, ENV_TARGET_STORE_COMPRESS, EnableState}; use serde::{Serialize, de::DeserializeOwned}; use snap::raw::{Decoder, Encoder}; -use std::sync::{Arc, RwLock}; use std::{ collections::HashMap, marker::PhantomData, path::PathBuf, + sync::{ + Arc, RwLock, + atomic::{AtomicU64, Ordering}, + }, time::{SystemTime, UNIX_EPOCH}, }; use tracing::{debug, warn}; use uuid::Uuid; +fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool { + value + .and_then(|value| value.parse::().ok().map(|state| state.is_enabled())) + .unwrap_or(DEFAULT_TARGET_STORE_COMPRESS) +} + +fn queue_store_compression_enabled() -> bool { + let value = std::env::var(ENV_TARGET_STORE_COMPRESS).ok(); + resolve_queue_store_compression_from_env_value(value.as_deref()) +} + /// Represents a key for an entry in the store #[derive(Debug, Clone)] pub struct Key { @@ -63,21 +77,7 @@ impl Key { impl std::fmt::Display for Key { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name_part = if self.item_count > 1 { - format!("{}:{}", self.item_count, self.name) - } else { - self.name.clone() - }; - - let mut file_name = name_part; - if !self.extension.is_empty() { - file_name.push_str(&self.extension); - } - - if self.compress { - file_name.push_str(COMPRESS_EXT); - } - write!(f, "{file_name}") + f.write_str(&self.to_key_string()) } } @@ -123,6 +123,28 @@ pub fn parse_key(s: &str) -> Key { } } +pub fn ensure_store_entry_raw_readable( + store: &(dyn Store + Send), + key: &Key, +) -> Result +where + T: Send + Sync + 'static + Clone + Serialize, +{ + match store.get_raw(key) { + Ok(_) => Ok(true), + Err(StoreError::NotFound) => Ok(false), + Err(err) => { + match store.del(key) { + Ok(()) | Err(StoreError::NotFound) => {} + Err(del_err) => { + return Err(StoreError::Internal(format!("Failed to remove unreadable store entry {key}: {del_err}"))); + } + } + Err(err) + } + } +} + /// Trait for a store that can store and retrieve items of type T pub trait Store: Send + Sync where @@ -142,15 +164,24 @@ where /// Stores multiple items in a single batch fn put_multiple(&self, items: Vec) -> Result; + /// Stores raw bytes in a single entry. + fn put_raw(&self, data: &[u8]) -> Result; + /// Retrieves a single item by key fn get(&self, key: &Self::Key) -> Result; /// Retrieves multiple items by key fn get_multiple(&self, key: &Self::Key) -> Result, Self::Error>; + /// Retrieves the raw bytes stored for a key. + fn get_raw(&self, key: &Self::Key) -> Result, Self::Error>; + /// Deletes an item by key fn del(&self, key: &Self::Key) -> Result<(), Self::Error>; + /// Deletes the underlying store directory and clears all in-memory state. + fn delete(&self) -> Result<(), Self::Error>; + /// Lists all keys in the store fn list(&self) -> Vec; @@ -169,7 +200,10 @@ pub struct QueueStore { entry_limit: u64, directory: PathBuf, file_ext: String, + compress: bool, entries: Arc>>, // key -> modtime as unix nano + pending_entries: Arc, + fs_guard: Arc>, _phantom: PhantomData, } @@ -179,35 +213,70 @@ impl Clone for QueueStore { entry_limit: self.entry_limit, directory: self.directory.clone(), file_ext: self.file_ext.clone(), + compress: self.compress, entries: Arc::clone(&self.entries), + pending_entries: Arc::clone(&self.pending_entries), + fs_guard: Arc::clone(&self.fs_guard), _phantom: PhantomData, } } } +struct EntryReservation<'a> { + pending_entries: &'a AtomicU64, +} + +impl Drop for EntryReservation<'_> { + fn drop(&mut self) { + self.pending_entries.fetch_sub(1, Ordering::SeqCst); + } +} + impl QueueStore { /// Creates a new QueueStore pub fn new(directory: impl Into, limit: u64, ext: &str) -> Self { + Self::new_with_compression(directory, limit, ext, queue_store_compression_enabled()) + } + + /// Creates a new QueueStore with an explicit compression setting. + pub fn new_with_compression(directory: impl Into, limit: u64, ext: &str, compress: bool) -> Self { let file_ext = if ext.is_empty() { DEFAULT_EXT } else { ext }; + let entry_limit = if limit == 0 { DEFAULT_LIMIT } else { limit }; QueueStore { directory: directory.into(), - entry_limit: if limit == 0 { DEFAULT_LIMIT } else { limit }, + entry_limit, file_ext: file_ext.to_string(), - entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))), + compress, + entries: Arc::new(RwLock::new(HashMap::with_capacity(entry_limit as usize))), + pending_entries: Arc::new(AtomicU64::new(0)), + fs_guard: Arc::new(RwLock::new(())), _phantom: PhantomData, } } /// Returns the full path for a key fn file_path(&self, key: &Key) -> PathBuf { - self.directory.join(key.to_string()) + self.directory.join(key.to_key_string()) + } + + fn build_key(&self, item_count: usize) -> Key { + Key { + name: Uuid::new_v4().to_string(), + extension: self.file_ext.clone(), + item_count, + compress: self.compress, + } } /// Reads a file for the given key fn read_file(&self, key: &Key) -> Result, StoreError> { + let _fs_guard = self + .fs_guard + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; let path = self.file_path(key); - debug!("Reading file for key: {},path: {}", key.to_string(), path.display()); + debug!("Reading file for key: {},path: {}", key, path.display()); let data = std::fs::read(&path).map_err(|e| { if e.kind() == std::io::ErrorKind::NotFound { StoreError::NotFound @@ -220,41 +289,89 @@ impl QueueStore { return Err(StoreError::NotFound); } - if key.compress { - let mut decoder = Decoder::new(); - decoder - .decompress_vec(&data) - .map_err(|e| StoreError::Compression(e.to_string())) - } else { - Ok(data) + if !key.compress { + return Ok(data); + } + + let mut decoder = Decoder::new(); + decoder + .decompress_vec(&data) + .map_err(|e| StoreError::Compression(e.to_string())) + } + + fn reserve_entry_slot(&self) -> Result, StoreError> { + loop { + let entries = self + .entries + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?; + let entries_len = entries.len() as u64; + let pending = self.pending_entries.load(Ordering::SeqCst); + + if entries_len + pending >= self.entry_limit { + return Err(StoreError::LimitExceeded); + } + + if self + .pending_entries + .compare_exchange(pending, pending + 1, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + return Ok(EntryReservation { + pending_entries: self.pending_entries.as_ref(), + }); + } } } - /// Writes data to a file for the given key - fn write_file(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> { + /// Writes data to a file for the given key. + fn write_file(&self, key: &Key, data: &[u8]) -> Result { let path = self.file_path(key); // Create directory if it doesn't exist if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).map_err(StoreError::Io)?; } - let data = if key.compress { + if key.compress { let mut encoder = Encoder::new(); - encoder + let compressed = encoder .compress_vec(data) - .map_err(|e| StoreError::Compression(e.to_string()))? + .map_err(|e| StoreError::Compression(e.to_string()))?; + std::fs::write(&path, &compressed).map_err(StoreError::Io)?; } else { - data.to_vec() - }; - - std::fs::write(&path, &data).map_err(StoreError::Io)?; + std::fs::write(&path, data).map_err(StoreError::Io)?; + } let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64; + debug!("Wrote event to store: {}", key); + Ok(modified) + } + + fn insert_entry(&self, key: &Key, modified: i64) -> Result<(), StoreError> { let mut entries = self .entries .write() .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; - entries.insert(key.to_string(), modified); - debug!("Wrote event to store: {}", key.to_string()); + entries.insert(key.to_key_string(), modified); + Ok(()) + } + + fn remove_file_if_present(&self, key: &Key) -> Result<(), StoreError> { + let path = self.file_path(key); + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(StoreError::Io(err)), + } + } + + fn write_and_index(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> { + let modified = self.write_file(key, data)?; + if let Err(err) = self.insert_entry(key, modified) { + self.remove_file_if_present(key).map_err(|cleanup_err| { + StoreError::Internal(format!("Failed to index store entry {key}: {err}; cleanup failed: {cleanup_err}")) + })?; + return Err(err); + } Ok(()) } } @@ -267,15 +384,20 @@ where type Key = Key; fn open(&self) -> Result<(), Self::Error> { + let _fs_guard = self + .fs_guard + .write() + .map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?; std::fs::create_dir_all(&self.directory).map_err(StoreError::Io)?; - let entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?; - // Get the write lock to update the internal state + let dir_entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?; let mut entries_map = self .entries .write() .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; - for entry in entries { + self.pending_entries.store(0, Ordering::SeqCst); + entries_map.clear(); + for entry in dir_entries { let entry = entry.map_err(StoreError::Io)?; let metadata = entry.metadata().map_err(StoreError::Io)?; if metadata.is_file() { @@ -292,71 +414,47 @@ where } fn put(&self, item: Arc) -> Result { - // Check storage limits - { - let entries = self - .entries - .read() - .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?; - - if entries.len() as u64 >= self.entry_limit { - return Err(StoreError::LimitExceeded); - } - } - - let uuid = Uuid::new_v4(); - let key = Key { - name: uuid.to_string(), - extension: self.file_ext.clone(), - item_count: 1, - compress: true, - }; - + let _fs_guard = self + .fs_guard + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; + let _reservation = self.reserve_entry_slot()?; + let key = self.build_key(1); let data = serde_json::to_vec(&*item).map_err(|e| StoreError::Serialization(e.to_string()))?; - self.write_file(&key, &data)?; + self.write_and_index(&key, &data)?; Ok(key) } fn put_multiple(&self, items: Vec) -> Result { - // Check storage limits - { - let entries = self - .entries - .read() - .map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?; - - if entries.len() as u64 >= self.entry_limit { - return Err(StoreError::LimitExceeded); - } - } if items.is_empty() { - // Or return an error, or a special key? return Err(StoreError::Internal("Cannot put_multiple with empty items list".to_string())); } - let uuid = Uuid::new_v4(); - let key = Key { - name: uuid.to_string(), - extension: self.file_ext.clone(), - item_count: items.len(), - compress: true, - }; + let _fs_guard = self + .fs_guard + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; + let _reservation = self.reserve_entry_slot()?; + let key = self.build_key(items.len()); - // Serialize all items into a single Vec - // This current approach for get_multiple/put_multiple assumes items are concatenated JSON objects. - // This might be problematic for deserialization if not handled carefully. - // A better approach for multiple items might be to store them as a JSON array `Vec`. - // For now, sticking to current logic of concatenating. let mut buffer = Vec::new(); for item in items { - // If items are Vec, and Event is large, this could be inefficient. - // The current get_multiple deserializes one by one. - let item_data = serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?; - buffer.extend_from_slice(&item_data); - // If using JSON array: buffer = serde_json::to_vec(&items)? + serde_json::to_writer(&mut buffer, &item).map_err(|e| StoreError::Serialization(e.to_string()))?; } - self.write_file(&key, &buffer)?; + self.write_and_index(&key, &buffer)?; + + Ok(key) + } + + fn put_raw(&self, data: &[u8]) -> Result { + let _fs_guard = self + .fs_guard + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; + let _reservation = self.reserve_entry_slot()?; + let key = self.build_key(1); + self.write_and_index(&key, data)?; Ok(key) } @@ -373,8 +471,8 @@ where } fn get_multiple(&self, key: &Self::Key) -> Result, Self::Error> { - debug!("Reading items from store for key: {}", key.to_string()); - let data = self.read_file(key)?; + debug!("Reading items from store for key: {}", key); + let data = self.get_raw(key)?; if data.is_empty() { return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string())); } @@ -404,7 +502,7 @@ where warn!( "Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.", key.item_count, - key.to_string(), + key, items.len() ); // Depending on strictness, this could be an error. @@ -426,20 +524,24 @@ where Ok(items) } + fn get_raw(&self, key: &Self::Key) -> Result, Self::Error> { + self.read_file(key) + } + fn del(&self, key: &Self::Key) -> Result<(), Self::Error> { + let _fs_guard = self + .fs_guard + .read() + .map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?; let path = self.file_path(key); - std::fs::remove_file(&path).map_err(|e| { - if e.kind() == std::io::ErrorKind::NotFound { - // If file not found, still try to remove from entries map in case of inconsistency - warn!( - "File not found for key {} during del, but proceeding to remove from entries map.", - key.to_string() - ); - StoreError::NotFound - } else { - StoreError::Io(e) + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // File already gone — still clean up the entries map to avoid stale keys. + warn!("File not found for key {} during del, cleaning up entries map.", key); } - })?; + Err(e) => return Err(StoreError::Io(e)), + } // Get the write lock to update the internal state let mut entries = self @@ -447,15 +549,32 @@ where .write() .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; - if entries.remove(&key.to_string()).is_none() { - // Key was not in the map, could be an inconsistency or already deleted. - // This is not necessarily an error if the file deletion succeeded or was NotFound. + if entries.remove(&key.to_key_string()).is_none() { debug!("Key {} not found in entries map during del, might have been already removed.", key); } debug!("Deleted event from store: {}", key.to_string()); Ok(()) } + fn delete(&self) -> Result<(), Self::Error> { + let _fs_guard = self + .fs_guard + .write() + .map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?; + let mut entries = self + .entries + .write() + .map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?; + entries.clear(); + self.pending_entries.store(0, Ordering::SeqCst); + + match std::fs::remove_dir_all(&self.directory) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(StoreError::Io(err)), + } + } + fn list(&self) -> Vec { // Get the read lock to read the internal state let entries = match self.entries.read() { @@ -492,3 +611,136 @@ where Box::new(self.clone()) as Box + Send + Sync> } } + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + sync::{Arc, Barrier}, + thread, + }; + + fn temp_store_dir(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4())) + } + + #[test] + fn resolve_queue_store_compression_defaults_to_true() { + assert!(resolve_queue_store_compression_from_env_value(None)); + } + + #[test] + fn resolve_queue_store_compression_respects_disabled_env_value() { + assert!(!resolve_queue_store_compression_from_env_value(Some("off"))); + assert!(!resolve_queue_store_compression_from_env_value(Some("false"))); + } + + #[test] + fn put_uses_store_compression_setting_in_key() { + let dir = temp_store_dir("put-key"); + let store = QueueStore::::new_with_compression(&dir, 8, ".test", false); + store.open().unwrap(); + + let key = store.put(Arc::new("payload".to_string())).unwrap(); + + assert!(!key.compress); + assert!(store.file_path(&key).exists()); + + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn parse_key_round_trips_batch_and_compression_suffixes() { + let key = Key { + name: "event-id".to_string(), + extension: ".json".to_string(), + item_count: 3, + compress: true, + }; + + let parsed = parse_key(&key.to_key_string()); + + assert_eq!(parsed.name, key.name); + assert_eq!(parsed.extension, key.extension); + assert_eq!(parsed.item_count, key.item_count); + assert_eq!(parsed.compress, key.compress); + } + + #[test] + fn put_raw_and_get_raw_round_trip_bytes() { + let dir = temp_store_dir("raw-roundtrip"); + let store = QueueStore::::new_with_compression(&dir, 8, ".test", true); + store.open().unwrap(); + + let payload = br#"{"kind":"notify","bucket":"demo","key":"alpha.txt"}"#; + let key = store.put_raw(payload).unwrap(); + let raw = store.get_raw(&key).unwrap(); + + assert_eq!(raw, payload); + + let _ = store.delete(); + } + + #[test] + fn delete_removes_directory_and_clears_entries() { + let dir = temp_store_dir("delete-store"); + let store = QueueStore::::new_with_compression(&dir, 8, ".test", false); + store.open().unwrap(); + let _ = store.put(Arc::new("payload".to_string())).unwrap(); + + store.delete().unwrap(); + + assert!(store.list().is_empty()); + assert!(!dir.exists()); + } + + #[test] + fn put_enforces_entry_limit() { + let dir = temp_store_dir("limit"); + let store = QueueStore::::new_with_compression(&dir, 1, ".test", false); + store.open().unwrap(); + + let _ = store.put(Arc::new("first".to_string())).unwrap(); + let err = store.put(Arc::new("second".to_string())).unwrap_err(); + + assert!(matches!(err, StoreError::LimitExceeded)); + + let _ = store.delete(); + } + + #[test] + fn concurrent_put_raw_respects_entry_limit() { + let dir = temp_store_dir("concurrent-limit"); + let store = Arc::new(QueueStore::::new_with_compression(&dir, 1, ".test", true)); + store.open().unwrap(); + + let start = Arc::new(Barrier::new(4)); + let mut handles = Vec::new(); + + for idx in 0..4 { + let store = Arc::clone(&store); + let start = Arc::clone(&start); + handles.push(thread::spawn(move || { + let payload = vec![b'x'; 32 * 1024 + idx]; + start.wait(); + store.put_raw(&payload) + })); + } + + let mut successes = 0; + let mut limit_errors = 0; + for handle in handles { + match handle.join().unwrap() { + Ok(_) => successes += 1, + Err(StoreError::LimitExceeded) => limit_errors += 1, + Err(err) => panic!("unexpected error: {err}"), + } + } + + assert_eq!(successes, 1); + assert_eq!(limit_errors, 3); + assert_eq!(store.len(), 1); + + let _ = store.delete(); + } +} diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index 1a097a054..dc16dcae2 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -21,6 +21,8 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::fmt::Formatter; use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::warn; pub mod mqtt; pub mod webhook; @@ -45,14 +47,43 @@ where /// Saves an event (either sends it immediately or stores it for later) async fn save(&self, event: Arc>) -> Result<(), TargetError>; - /// Sends an event from the store - async fn send_from_store(&self, key: Key) -> Result<(), TargetError>; + /// Sends an event from the store using the queued raw body and metadata. + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError>; + + /// Sends an event from the store. + async fn send_from_store(&self, key: Key) -> Result<(), TargetError> { + let store = self + .store() + .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?; + + let raw = match store.get_raw(&key) { + Ok(raw) => raw, + Err(StoreError::NotFound) => return Ok(()), + Err(err) => return Err(TargetError::Storage(format!("Failed to read queued payload from store: {err}"))), + }; + + let queued = match QueuedPayload::decode(&raw) { + Ok(queued) => queued, + Err(err) => { + delete_stored_payload(store, &key).map_err(|delete_err| { + TargetError::Storage(format!( + "Failed to delete invalid queued payload {key} after decode error '{err}': {delete_err}" + )) + })?; + warn!("Dropped invalid queued payload {key}: {err}"); + return Ok(()); + } + }; + + self.send_raw_from_store(key.clone(), queued.body, queued.meta).await?; + delete_stored_payload(store, &key) + } /// Closes the target and releases resources async fn close(&self) -> Result<(), TargetError>; /// Returns the store associated with the target (if any) - fn store(&self) -> Option<&(dyn Store, Error = StoreError, Key = Key> + Send + Sync)>; + fn store(&self) -> Option<&(dyn Store + Send + Sync)>; /// Returns the type of the target fn clone_dyn(&self) -> Box + Send + Sync>; @@ -78,6 +109,106 @@ where pub data: E, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueuedPayloadMeta { + pub event_name: EventName, + pub bucket_name: String, + pub object_name: String, + pub content_type: String, + pub queued_at_unix_ms: u64, + pub payload_len: usize, +} + +impl QueuedPayloadMeta { + pub fn new( + event_name: EventName, + bucket_name: String, + object_name: String, + content_type: impl Into, + payload_len: usize, + ) -> Self { + Self { + event_name, + bucket_name, + object_name, + content_type: content_type.into(), + queued_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64, + payload_len, + } + } + + pub fn best_effort_preview(&self, body: &[u8], limit: usize) -> String { + if limit == 0 || body.is_empty() { + return String::new(); + } + + let slice = &body[..body.len().min(limit)]; + match std::str::from_utf8(slice) { + Ok(text) => { + if body.len() > limit { + format!("{text}...") + } else { + text.to_string() + } + } + Err(_) => format!("<{} bytes binary>", body.len()), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QueuedPayload { + pub meta: QueuedPayloadMeta, + pub body: Vec, +} + +impl QueuedPayload { + const MAGIC: [u8; 4] = *b"RQP1"; + + pub fn new(meta: QueuedPayloadMeta, body: Vec) -> Self { + Self { meta, body } + } + + pub fn encode(&self) -> Result, TargetError> { + let meta = serde_json::to_vec(&self.meta) + .map_err(|err| TargetError::Serialization(format!("Failed to serialize queued payload metadata: {err}")))?; + let meta_len = u32::try_from(meta.len()) + .map_err(|_| TargetError::Serialization("Queued payload metadata is too large".to_string()))?; + + let mut out = Vec::with_capacity(Self::MAGIC.len() + 4 + meta.len() + self.body.len()); + out.extend_from_slice(&Self::MAGIC); + out.extend_from_slice(&meta_len.to_le_bytes()); + out.extend_from_slice(&meta); + out.extend_from_slice(&self.body); + Ok(out) + } + + pub fn decode(raw: &[u8]) -> Result { + if raw.len() < Self::MAGIC.len() + 4 { + return Err(TargetError::Serialization("Queued payload is too short".to_string())); + } + if raw[..Self::MAGIC.len()] != Self::MAGIC { + return Err(TargetError::Serialization("Queued payload magic mismatch".to_string())); + } + + let mut meta_len_bytes = [0u8; 4]; + meta_len_bytes.copy_from_slice(&raw[Self::MAGIC.len()..Self::MAGIC.len() + 4]); + let meta_len = u32::from_le_bytes(meta_len_bytes) as usize; + let meta_start = Self::MAGIC.len() + 4; + let meta_end = meta_start + meta_len; + + if meta_end > raw.len() { + return Err(TargetError::Serialization("Queued payload metadata length exceeds input".to_string())); + } + + let meta = serde_json::from_slice(&raw[meta_start..meta_end]) + .map_err(|err| TargetError::Serialization(format!("Failed to deserialize queued payload metadata: {err}")))?; + let body = raw[meta_end..].to_vec(); + + Ok(Self { meta, body }) + } +} + /// The `ChannelTargetType` enum represents the different types of channel Target /// used in the notification system. /// @@ -187,3 +318,45 @@ pub fn decode_object_name(encoded: &str) -> Result { .map(|s| s.into_owned()) .map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}"))) } + +pub(crate) fn delete_stored_payload( + store: &(dyn Store + Send + Sync), + key: &Key, +) -> Result<(), TargetError> { + match store.del(key) { + Ok(()) | Err(StoreError::NotFound) => Ok(()), + Err(err) => Err(TargetError::Storage(format!("Failed to delete event from store: {err}"))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn queued_payload_round_trips_meta_and_body() { + let meta = QueuedPayloadMeta::new( + EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "folder/object.txt".to_string(), + "application/json", + 12, + ); + let payload = QueuedPayload::new(meta.clone(), br#"{"ok":true}"#.to_vec()); + + let encoded = payload.encode().unwrap(); + let decoded = QueuedPayload::decode(&encoded).unwrap(); + + assert_eq!(decoded.meta.event_name, meta.event_name); + assert_eq!(decoded.meta.bucket_name, meta.bucket_name); + assert_eq!(decoded.meta.object_name, meta.object_name); + assert_eq!(decoded.meta.content_type, meta.content_type); + assert_eq!(decoded.body, br#"{"ok":true}"#); + } + + #[test] + fn queued_payload_decode_rejects_invalid_magic() { + let err = QueuedPayload::decode(b"bad-payload").unwrap_err(); + assert!(err.to_string().contains("magic") || err.to_string().contains("short")); + } +} diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index eb876d9c1..df32b2fff 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -17,7 +17,7 @@ use crate::{ arn::TargetID, error::TargetError, store::{Key, QueueStore, Store}, - target::{ChannelTargetType, EntityTarget, TargetType}, + target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType}, }; use async_trait::async_trait; use rumqttc::{AsyncClient, ConnectionError, EventLoop, MqttOptions, Outgoing, Packet, QoS, mqttbytes::Error as MqttBytesError}; @@ -25,6 +25,7 @@ use serde::Serialize; use serde::de::DeserializeOwned; use std::sync::Arc; use std::{ + marker::PhantomData, path::PathBuf, sync::atomic::{AtomicBool, Ordering}, time::Duration, @@ -110,9 +111,10 @@ where id: TargetID, args: MQTTArgs, client: Arc>>, - store: Option, Error = StoreError, Key = Key> + Send + Sync>>, + store: Option + Send + Sync>>, connected: Arc, bg_task_manager: Arc, + _phantom: PhantomData, } impl MQTTTarget @@ -135,7 +137,7 @@ where TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, }; - let store = QueueStore::>::new(specific_queue_path, args.queue_limit, extension); + let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); if let Err(e) = store.open() { error!( target_id = %target_id, @@ -144,7 +146,7 @@ where ); return Err(TargetError::Storage(format!("{e}"))); } - Some(Box::new(store) as Box, Error = StoreError, Key = Key> + Send + Sync>) + Some(Box::new(store) as Box + Send + Sync>) } else { None }; @@ -157,13 +159,14 @@ where }); info!(target_id = %target_id, "MQTT target created"); - Ok(MQTTTarget { + Ok(MQTTTarget:: { id: target_id, args, client: Arc::new(Mutex::new(None)), store: queue_store, connected: Arc::new(AtomicBool::new(false)), bg_task_manager, + _phantom: PhantomData, }) } @@ -251,14 +254,7 @@ where } } - #[instrument(skip(self, event), fields(target_id = %self.id))] - async fn send(&self, event: &EntityTarget) -> Result<(), TargetError> { - let client_guard = self.client.lock().await; - let client = client_guard - .as_ref() - .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; - - // Decode form-urlencoded object name + fn build_queued_payload(&self, event: &EntityTarget) -> Result { let object_name = crate::target::decode_object_name(&event.object_name)?; let key = format!("{}/{}", event.bucket_name, object_name); @@ -269,14 +265,35 @@ where records: vec![event.clone()], }; - let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; + let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; + let meta = QueuedPayloadMeta::new( + event.event_name, + event.bucket_name.clone(), + event.object_name.clone(), + "application/json", + body.len(), + ); + Ok(QueuedPayload::new(meta, body)) + } - let data_string = String::from_utf8(data.clone()) - .map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?; - debug!("Sending event to mqtt target: {}, event log: {}", self.id, data_string); + #[instrument(skip(self, body, meta), fields(target_id = %self.id))] + async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { + let client_guard = self.client.lock().await; + let client = client_guard + .as_ref() + .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; + + debug!( + target = %self.id, + bucket = %meta.bucket_name, + object = %meta.object_name, + event = %meta.event_name, + preview = %meta.best_effort_preview(&body, 256), + "Sending MQTT payload" + ); client - .publish(&self.args.topic, self.args.qos, false, data) + .publish(&self.args.topic, self.args.qos, false, body) .await .map_err(|e| { if e.to_string().contains("Connection") || e.to_string().contains("Timeout") { @@ -293,13 +310,14 @@ where } pub fn clone_target(&self) -> Box + Send + Sync> { - Box::new(MQTTTarget { + Box::new(MQTTTarget:: { id: self.id.clone(), args: self.args.clone(), client: self.client.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), connected: self.connected.clone(), bg_task_manager: self.bg_task_manager.clone(), + _phantom: PhantomData, }) } } @@ -494,11 +512,15 @@ where #[instrument(skip(self, event), fields(target_id = %self.id))] async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = self.build_queued_payload(&event)?; + if let Some(store) = &self.store { debug!(target_id = %self.id, "Event saved to store start"); - // If store is configured, ONLY put the event into the store. - // Do NOT send it directly here. - match store.put(event.clone()) { + match store.put_raw( + &queued + .encode() + .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?, + ) { Ok(_) => { debug!(target_id = %self.id, "Event saved to store for MQTT target successfully."); Ok(()) @@ -516,7 +538,7 @@ where if !self.connected.load(Ordering::SeqCst) { warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init."); // Call the struct's init method, not the trait's default - match MQTTTarget::init(self).await { + match MQTTTarget::::init(self).await { Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), Err(e) => { error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); @@ -528,13 +550,13 @@ where return Err(TargetError::NotConnected); } } - self.send(&event).await + self.send_body(queued.body, &queued.meta).await } } - #[instrument(skip(self), fields(target_id = %self.id))] - async fn send_from_store(&self, key: Key) -> Result<(), TargetError> { - debug!(target_id = %self.id, ?key, "Attempting to send event from store with key."); + #[instrument(skip(self, body, meta), fields(target_id = %self.id))] + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { + debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store."); if !self.is_enabled() { return Err(TargetError::Disabled); @@ -542,7 +564,7 @@ where if !self.connected.load(Ordering::SeqCst) { warn!(target_id = %self.id, "Not connected; trying to init before sending from store."); - match MQTTTarget::init(self).await { + match MQTTTarget::::init(self).await { Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."), Err(e) => { error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target."); @@ -555,33 +577,8 @@ where } } - let store = self - .store - .as_ref() - .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?; - - let event = match store.get(&key) { - Ok(event) => { - debug!(target_id = %self.id, ?key, "Retrieved event from store for sending."); - event - } - Err(StoreError::NotFound) => { - // Assuming NotFound takes the key - debug!(target_id = %self.id, ?key, "Event not found in store for sending."); - return Ok(()); - } - Err(e) => { - error!( - target_id = %self.id, - error = %e, - "Failed to get event from store" - ); - return Err(TargetError::Storage(format!("Failed to get event from store: {e}"))); - } - }; - debug!(target_id = %self.id, ?key, "Sending event from store."); - if let Err(e) = self.send(&event).await { + if let Err(e) = self.send_body(body, &meta).await { if matches!(e, TargetError::NotConnected) { warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store."); return Err(TargetError::NotConnected); @@ -589,22 +586,7 @@ where error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error."); return Err(e); } - debug!(target_id = %self.id, ?key, "Event sent from store successfully. deleting from store. "); - - match store.del(&key) { - Ok(_) => { - debug!(target_id = %self.id, ?key, "Event deleted from store after successful send.") - } - Err(StoreError::NotFound) => { - debug!(target_id = %self.id, ?key, "Event already deleted from store."); - } - Err(e) => { - error!(target_id = %self.id, error = %e, "Failed to delete event from store after send."); - return Err(TargetError::Storage(format!("Failed to delete event from store: {e}"))); - } - } - - debug!(target_id = %self.id, ?key, "Event deleted from store."); + debug!(target_id = %self.id, ?key, "Event sent from store successfully."); Ok(()) } @@ -637,7 +619,7 @@ where Ok(()) } - fn store(&self) -> Option<&(dyn Store, Error = StoreError, Key = Key> + Send + Sync)> { + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { self.store.as_deref() } @@ -651,7 +633,7 @@ where return Ok(()); } // Call the internal init logic - MQTTTarget::init(self).await + MQTTTarget::::init(self).await } fn is_enabled(&self) -> bool { diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index 525cf5b9f..a34b16e10 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -17,7 +17,7 @@ use crate::{ arn::TargetID, error::TargetError, store::{Key, QueueStore, Store}, - target::{ChannelTargetType, EntityTarget, TargetType}, + target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType}, }; use async_trait::async_trait; use reqwest::{Client, StatusCode, Url}; @@ -26,6 +26,7 @@ use rustfs_config::notify::NOTIFY_STORE_EXTENSION; use serde::Serialize; use serde::de::DeserializeOwned; use std::{ + marker::PhantomData, path::PathBuf, sync::{ Arc, @@ -33,7 +34,6 @@ use std::{ }, time::Duration, }; -use tokio::net::lookup_host; use tokio::sync::mpsc; use tracing::{debug, error, info, instrument, warn}; @@ -105,10 +105,10 @@ where args: WebhookArgs, http_client: Arc, // Add Send + Sync constraints to ensure thread safety - store: Option, Error = StoreError, Key = Key> + Send + Sync>>, + store: Option + Send + Sync>>, initialized: AtomicBool, - addr: String, cancel_sender: mpsc::Sender<()>, + _phantom: PhantomData, } impl WebhookTarget @@ -117,14 +117,14 @@ where { /// Clones the WebhookTarget, creating a new instance with the same configuration pub fn clone_box(&self) -> Box + Send + Sync> { - Box::new(WebhookTarget { + Box::new(WebhookTarget:: { id: self.id.clone(), args: self.args.clone(), http_client: Arc::clone(&self.http_client), store: self.store.as_ref().map(|s| s.boxed_clone()), initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)), - addr: self.addr.clone(), cancel_sender: self.cancel_sender.clone(), + _phantom: PhantomData, }) } @@ -149,7 +149,7 @@ where TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION, }; - let store = QueueStore::>::new(queue_dir, args.queue_limit, extension); + let store = QueueStore::::new(queue_dir, args.queue_limit, extension); if let Err(e) = store.open() { error!("Failed to open store for Webhook target {}: {}", target_id.id, e); @@ -157,32 +157,22 @@ where } // Make sure that the Store trait implemented by QueueStore matches the expected error type - Some(Box::new(store) as Box, Error = StoreError, Key = Key> + Send + Sync>) + Some(Box::new(store) as Box + Send + Sync>) } else { None }; - // resolved address - let addr = { - let host = args.endpoint.host_str().unwrap_or("localhost"); - let port = args - .endpoint - .port() - .unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 }); - format!("{host}:{port}") - }; - // Create a cancel channel let (cancel_sender, _) = mpsc::channel(1); info!(target_id = %target_id.id, "Webhook target created"); - Ok(WebhookTarget { + Ok(WebhookTarget:: { id: target_id, args, http_client, store: queue_store, initialized: AtomicBool::new(false), - addr, cancel_sender, + _phantom: PhantomData, }) } @@ -226,53 +216,80 @@ where .map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}"))) } - async fn init(&self) -> Result<(), TargetError> { - // Use CAS operations to ensure thread-safe initialization - if !self.initialized.load(Ordering::SeqCst) { - // Check the connection - match self.is_active().await { - Ok(true) => { - info!("Webhook target {} is active", self.id); - } - Ok(false) => { - return Err(TargetError::NotConnected); - } - Err(e) => { - error!("Failed to check if Webhook target {} is active: {}", self.id, e); - return Err(e); + async fn init_inner(&self) -> Result<(), TargetError> { + if self.initialized.load(Ordering::SeqCst) { + return Ok(()); + } + + // HTTP HEAD probe: verifies the full request path (proxy, TLS, firewall) + // unlike TCP connect which can't detect proxy issues. + let probe_timeout = Duration::from_secs(5); + match tokio::time::timeout(probe_timeout, self.http_client.head(self.args.endpoint.as_str()).send()).await { + Ok(Ok(resp)) => { + let status = resp.status(); + if status.is_success() || status == StatusCode::NOT_FOUND { + // NOT_FOUND is acceptable for HEAD probes — the endpoint may not + // exist as a HEAD route, but the server is reachable. + debug!("Webhook target {} HEAD probe returned {}", self.id, status); + } else if status == StatusCode::METHOD_NOT_ALLOWED { + // Server is reachable but doesn't support HEAD — still valid. + debug!("Webhook target {} HEAD probe: METHOD_NOT_ALLOWED (reachable)", self.id); + } else { + warn!("Webhook target {} HEAD probe returned {}", self.id, status); } } - self.initialized.store(true, Ordering::SeqCst); - info!("Webhook target {} initialized", self.id); + Ok(Err(e)) => { + // Connection-level error (DNS, TLS, refused, timeout) + return Err(if e.is_timeout() || e.is_connect() { + TargetError::NotConnected + } else { + TargetError::Network(format!("Webhook HEAD probe failed: {e}")) + }); + } + Err(_) => { + return Err(TargetError::Timeout("Webhook HEAD probe timed out".to_string())); + } } + + self.initialized.store(true, Ordering::SeqCst); + info!("Webhook target {} initialized", self.id); Ok(()) } - async fn send(&self, event: &EntityTarget) -> Result<(), TargetError> { - info!("Webhook Sending event to webhook target: {}", self.id); - // Decode form-urlencoded object name + fn build_queued_payload(&self, event: &EntityTarget) -> Result { let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - let log = TargetLog { event_name: event.event_name, key, records: vec![event.data.clone()], }; + let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; + let meta = QueuedPayloadMeta::new( + event.event_name, + event.bucket_name.clone(), + event.object_name.clone(), + "application/json", + body.len(), + ); + Ok(QueuedPayload::new(meta, body)) + } - let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; + async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { + info!("Webhook sending queued payload to target: {}", self.id); + debug!( + target = %self.id, + bucket = %meta.bucket_name, + object = %meta.object_name, + event = %meta.event_name, + preview = %meta.best_effort_preview(&body, 256), + "Sending webhook payload" + ); - // Vec Convert to String - let data_string = String::from_utf8(data.clone()) - .map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?; - debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string); - - // build request let mut req_builder = self .http_client .post(self.args.endpoint.as_str()) - .header("Content-Type", "application/json"); + .header("Content-Type", meta.content_type.as_str()); if !self.args.auth_token.is_empty() { // Split auth_token string to check if the authentication type is included @@ -293,7 +310,7 @@ where } // Send a request - let resp = req_builder.body(data).send().await.map_err(|e| { + let resp = req_builder.body(body).send().await.map_err(|e| { if e.is_timeout() || e.is_connect() { TargetError::NotConnected } else { @@ -329,34 +346,39 @@ where } async fn is_active(&self) -> Result { - let socket_addr = lookup_host(&self.addr) - .await - .map_err(|e| TargetError::Network(format!("Failed to resolve host: {e}")))? - .next() - .ok_or_else(|| TargetError::Network("No address found".to_string()))?; - debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id); - match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await { - Ok(Ok(_)) => { - debug!("Connection to {} is active", self.addr); - Ok(true) - } - Ok(Err(e)) => { - debug!("Connection to {} failed: {}", self.addr, e); - if e.kind() == std::io::ErrorKind::ConnectionRefused { - Err(TargetError::NotConnected) + match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(self.args.endpoint.as_str()).send()).await { + Ok(Ok(resp)) => { + let status = resp.status(); + if status.is_server_error() { + debug!("Webhook {} server error: {}", self.id, status); + Ok(false) } else { - Err(TargetError::Network(format!("Connection failed: {e}"))) + debug!("Webhook {} is reachable (status: {})", self.id, status); + Ok(true) } } - Err(_) => Err(TargetError::Timeout("Connection timed out".to_string())), + Ok(Err(e)) => { + debug!("Webhook {} request failed: {}", self.id, e); + if e.is_timeout() || e.is_connect() { + Err(TargetError::NotConnected) + } else { + Err(TargetError::Network(format!("Webhook health check failed: {e}"))) + } + } + Err(_) => Err(TargetError::Timeout("Webhook health check timed out".to_string())), } } async fn save(&self, event: Arc>) -> Result<(), TargetError> { + let queued = self.build_queued_payload(&event)?; + if let Some(store) = &self.store { - // Call the store method directly, no longer need to acquire the lock store - .put(event) + .put_raw( + &queued + .encode() + .map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?, + ) .map_err(|e| TargetError::Storage(format!("Failed to save event to store: {e}")))?; debug!("Event saved to store for target: {}", self.id); Ok(()) @@ -368,12 +390,12 @@ where return Err(TargetError::NotConnected); } } - self.send(&event).await + self.send_body(queued.body, &queued.meta).await } } - async fn send_from_store(&self, key: Key) -> Result<(), TargetError> { - debug!("Sending event from store for target: {}", self.id); + async fn send_raw_from_store(&self, key: Key, body: Vec, meta: QueuedPayloadMeta) -> Result<(), TargetError> { + debug!("Sending queued payload from store for target: {}, key: {}", self.id, key); match self.init().await { Ok(_) => { debug!("Event sent to store for target: {}", self.name()); @@ -384,37 +406,13 @@ where } } - let store = self - .store - .as_ref() - .ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?; - - // Get events directly from the store, no longer need to acquire locks - let event = match store.get(&key) { - Ok(event) => event, - Err(StoreError::NotFound) => return Ok(()), - Err(e) => { - return Err(TargetError::Storage(format!("Failed to get event from store: {e}"))); - } - }; - - if let Err(e) = self.send(&event).await { + if let Err(e) = self.send_body(body, &meta).await { if let TargetError::NotConnected = e { return Err(TargetError::NotConnected); } return Err(e); } - // Use the immutable reference of the store to delete the event content corresponding to the key - debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string()); - match store.del(&key) { - Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()), - Err(e) => { - error!("Failed to delete event from store: {}", e); - return Err(TargetError::Storage(format!("Failed to delete event from store: {e}"))); - } - } - debug!("Event sent from store and deleted for target: {}", self.id); Ok(()) } @@ -426,7 +424,7 @@ where Ok(()) } - fn store(&self) -> Option<&(dyn Store, Error = StoreError, Key = Key> + Send + Sync)> { + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { // Returns the reference to the internal store self.store.as_deref() } @@ -436,14 +434,11 @@ where } async fn init(&self) -> Result<(), TargetError> { - // If the target is disabled, return to success directly if !self.is_enabled() { debug!("Webhook target {} is disabled, skipping initialization", self.id); return Ok(()); } - - // Use existing initialization logic - WebhookTarget::init(self).await + self.init_inner().await } fn is_enabled(&self) -> bool { diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs new file mode 100644 index 000000000..c8b91fed8 --- /dev/null +++ b/rustfs/src/admin/handlers/audit.rs @@ -0,0 +1,818 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::router::{AdminOperation, Operation, S3Router}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::ADMIN_PREFIX; +use futures::stream::{FuturesUnordered, StreamExt}; +use hashbrown::HashSet as HbHashSet; +use http::{HeaderMap, StatusCode}; +use hyper::Method; +use matchit::Params; +use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; +use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; +use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_ecstore::config::Config; +use rustfs_targets::check_mqtt_broker_available; +use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::io::{Error, ErrorKind}; +use std::path::Path; +use std::sync::Arc; +use tokio::sync::Semaphore; +use tokio::time::{Duration, sleep, timeout}; +use tracing::{Span, warn}; +use url::Url; + +pub fn register_audit_target_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/list").as_str(), + AdminOperation(&ListAuditTargets {}), + )?; + + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}").as_str(), + AdminOperation(&AuditTargetConfig {}), + )?; + + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}/reset").as_str(), + AdminOperation(&RemoveAuditTarget {}), + )?; + + Ok(()) +} + +#[derive(Debug, Deserialize)] +pub struct KeyValue { + pub key: String, + pub value: String, +} + +#[derive(Debug, Deserialize)] +pub struct AuditTargetBody { + pub key_values: Vec, +} + +#[derive(Serialize, Debug)] +struct AuditEndpoint { + account_id: String, + service: String, + status: String, + source: AuditEndpointSource, +} + +#[derive(Serialize, Debug)] +struct AuditEndpointsResponse { + audit_endpoints: Vec, +} + +type EndpointKey = (String, String); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +enum AuditEndpointSource { + Config, + Env, + Mixed, + Runtime, +} + +fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey { + (account_id.to_lowercase(), service.to_string()) +} + +async fn check_permissions(req: &S3Request) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "credentials not found")); + }; + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + Ok(()) +} + +fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> { + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, "application/json".parse().unwrap()); + if let Some(v) = request_id { + header.insert("x-request-id", v.clone()); + } + S3Response::with_headers((status, body), header) +} + +async fn retry_with_backoff(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut attempts = 0; + let mut delay = base_delay; + let mut last_err = None; + + while attempts < max_attempts { + match operation().await { + Ok(result) => return Ok(result), + Err(e) => { + last_err = Some(e); + attempts += 1; + if attempts < max_attempts { + sleep(delay).await; + delay = delay.saturating_mul(2); + } + } + } + } + Err(last_err.unwrap_or_else(|| Error::other("retry_with_backoff: unknown error"))) +} + +async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> { + if !queue_dir.is_empty() { + if !Path::new(queue_dir).is_absolute() { + return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path")); + } + retry_with_backoff( + || async { tokio::fs::metadata(queue_dir).await.map(|_| ()) }, + 3, + Duration::from_millis(100), + ) + .await + .map_err(|e| match e.kind() { + ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"), + ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"), + _ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e), + })?; + } + Ok(()) +} + +fn config_enable_is_on(value: &str) -> bool { + matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1") +} + +fn has_any_audit_targets(config: &Config) -> bool { + for subsystem in [AUDIT_WEBHOOK_SUB_SYS, AUDIT_MQTT_SUB_SYS] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + if targets.keys().any(|key| key != DEFAULT_DELIMITER) { + return true; + } + } + false +} + +fn collect_configured_audit_endpoint_keys(config: &Config) -> Vec { + let mut endpoints = Vec::new(); + for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + + for (target_name, kvs) in targets { + if target_name == DEFAULT_DELIMITER { + continue; + } + let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false); + if enabled { + endpoints.push((target_name.clone(), service.to_string())); + } + } + } + endpoints +} + +fn collect_config_entry_keys(config: &Config) -> HbHashSet { + let mut endpoints = HbHashSet::new(); + for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + + for target_name in targets.keys() { + if target_name == DEFAULT_DELIMITER { + continue; + } + endpoints.insert(normalized_endpoint_key(target_name, service)); + } + } + endpoints +} + +fn collect_env_endpoint_keys() -> HbHashSet { + let mut endpoints = HbHashSet::new(); + + for (service, valid_keys) in [("webhook", AUDIT_WEBHOOK_KEYS), ("mqtt", AUDIT_MQTT_KEYS)] { + let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase(); + + for (key, _value) in std::env::vars() { + let Some(rest) = key.strip_prefix(&env_prefix) else { + continue; + }; + + let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER); + let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER); + let field_name_part = parts.next(); + + let (field_name, instance_id) = match field_name_part { + Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()), + None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()), + }; + + if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() { + continue; + } + + if valid_keys.contains(&field_name.as_str()) { + endpoints.insert(normalized_endpoint_key(&instance_id, service)); + } + } + } + + endpoints +} + +fn classify_audit_endpoint_source( + config_targets: &HbHashSet, + env_targets: &HbHashSet, + key: &EndpointKey, +) -> AuditEndpointSource { + match (config_targets.contains(key), env_targets.contains(key)) { + (true, true) => AuditEndpointSource::Mixed, + (true, false) => AuditEndpointSource::Config, + (false, true) => AuditEndpointSource::Env, + (false, false) => AuditEndpointSource::Runtime, + } +} + +fn audit_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> AuditEndpointSource { + let config_targets = collect_config_entry_keys(config); + let env_targets = collect_env_endpoint_keys(); + let service = match target_type { + AUDIT_WEBHOOK_SUB_SYS => "webhook", + AUDIT_MQTT_SUB_SYS => "mqtt", + _ => "", + }; + + let key = normalized_endpoint_key(target_name, service); + classify_audit_endpoint_source(&config_targets, &env_targets, &key) +} + +fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { + match audit_endpoint_source(config, target_type, target_name) { + AuditEndpointSource::Env => Some(format!( + "audit target '{}' is managed by environment variables and cannot be modified from the console", + target_name + )), + AuditEndpointSource::Mixed => Some(format!( + "audit target '{}' is configured by both persisted config and environment variables; remove the environment variables first", + target_name + )), + AuditEndpointSource::Config | AuditEndpointSource::Runtime => None, + } +} + +fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap) -> Vec { + let mut audit_endpoints = Vec::new(); + let mut seen = HashSet::new(); + let configured_keys = collect_configured_audit_endpoint_keys(config); + let config_targets = collect_config_entry_keys(config); + let env_targets = collect_env_endpoint_keys(); + let mut normalized_runtime_statuses: HashMap = HashMap::new(); + for ((account_id, service), status) in runtime_statuses { + let normalized = normalized_endpoint_key(&account_id, &service); + normalized_runtime_statuses + .entry(normalized) + .or_insert((account_id, service, status)); + } + + for key in configured_keys { + let normalized = normalized_endpoint_key(&key.0, &key.1); + if !seen.insert(normalized.clone()) { + continue; + } + let status = normalized_runtime_statuses + .remove(&normalized) + .map(|(_, _, status)| status) + .unwrap_or_else(|| "offline".to_string()); + let source = classify_audit_endpoint_source(&config_targets, &env_targets, &normalized); + audit_endpoints.push(AuditEndpoint { + account_id: key.0, + service: key.1, + status, + source, + }); + } + + for (normalized, (account_id, service, status)) in normalized_runtime_statuses { + if seen.insert(normalized.clone()) { + audit_endpoints.push(AuditEndpoint { + account_id, + service, + status, + source: classify_audit_endpoint_source(&config_targets, &env_targets, &normalized), + }); + } + } + + for key in &env_targets { + if !seen.insert(key.clone()) { + continue; + } + + audit_endpoints.push(AuditEndpoint { + account_id: key.0.clone(), + service: key.1.clone(), + status: "offline".to_string(), + source: classify_audit_endpoint_source(&config_targets, &env_targets, key), + }); + } + + audit_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); + audit_endpoints +} + +fn collect_validated_key_values( + key_values: &[KeyValue], + allowed_keys: &HashSet<&str>, + target_type: &str, +) -> S3Result> { + let mut kv_map = HashMap::new(); + let mut seen = HashSet::new(); + + for kv in key_values { + if !allowed_keys.contains(kv.key.as_str()) { + return Err(s3_error!( + InvalidArgument, + "key '{}' not allowed for audit target type '{}'", + kv.key, + target_type + )); + } + + if !seen.insert(kv.key.as_str()) { + return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key)); + } + + kv_map.insert(kv.key.clone(), kv.value.clone()); + } + + Ok(kv_map) +} + +fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> { + let target_type = params + .get("target_type") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?; + if target_type != AUDIT_WEBHOOK_SUB_SYS && target_type != AUDIT_MQTT_SUB_SYS { + return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type)); + } + let target_name = params + .get("target_name") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?; + Ok((target_type, target_name)) +} + +async fn load_server_config_from_store() -> S3Result { + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { + return Ok(Config::new()); + }; + + rustfs_ecstore::config::com::read_config_without_migrate(store) + .await + .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e)) +} + +async fn apply_audit_runtime_config(config: Config) -> S3Result<()> { + let has_targets = has_any_audit_targets(&config); + + if let Some(system) = audit_system() { + match system.get_state().await { + AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { + if has_targets { + system + .reload_config(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?; + } else { + system + .close() + .await + .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?; + } + } + AuditSystemState::Stopped | AuditSystemState::Stopping => { + if has_targets { + system + .start(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + } + } + } + } else if has_targets { + start_global_audit_system(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + } + + Ok(()) +} + +async fn update_audit_config_and_reload(mut modifier: F) -> S3Result<()> +where + F: FnMut(&mut Config) -> bool, +{ + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { + return Err(s3_error!(InternalError, "server storage not initialized")); + }; + + let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; + + if !modifier(&mut config) { + return Ok(()); + } + + rustfs_ecstore::config::com::save_server_config(store, &config) + .await + .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; + + apply_audit_runtime_config(config).await +} + +pub struct AuditTargetConfig {} + +#[async_trait::async_trait] +impl Operation for AuditTargetConfig { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let span = Span::current(); + let _enter = span.enter(); + let (target_type, target_name) = extract_target_params(¶ms)?; + + check_permissions(&req).await?; + let config_snapshot = load_server_config_from_store().await?; + if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + let mut input = req.input; + let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| { + warn!("failed to read request body: {:?}", e); + s3_error!(InvalidRequest, "failed to read request body") + })?; + + let audit_body: AuditTargetBody = serde_json::from_slice(&body_bytes) + .map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?; + + let allowed_keys: HashSet<&str> = match target_type { + AUDIT_WEBHOOK_SUB_SYS => AUDIT_WEBHOOK_KEYS.iter().cloned().collect(), + AUDIT_MQTT_SUB_SYS => AUDIT_MQTT_KEYS.iter().cloned().collect(), + _ => unreachable!(), + }; + + let kv_map = collect_validated_key_values(&audit_body.key_values, &allowed_keys, target_type)?; + + if target_type == AUDIT_WEBHOOK_SUB_SYS { + let endpoint = kv_map + .get("endpoint") + .map(String::as_str) + .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?; + let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?; + match parsed_endpoint.scheme() { + "http" | "https" => {} + other => { + return Err(s3_error!( + InvalidArgument, + "unsupported endpoint scheme: {} (only http and https are allowed)", + other + )); + } + } + if let Some(queue_dir) = kv_map.get("queue_dir") { + validate_queue_dir(queue_dir.as_str()).await?; + } + if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") { + return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair")); + } + } else if target_type == AUDIT_MQTT_SUB_SYS { + let endpoint = kv_map + .get(rustfs_config::MQTT_BROKER) + .map(String::as_str) + .ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?; + let topic = kv_map + .get(rustfs_config::MQTT_TOPIC) + .map(String::as_str) + .ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?; + let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str); + let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str); + check_mqtt_broker_available(endpoint, topic, username, password) + .await + .map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?; + + if let Some(queue_dir) = kv_map.get("queue_dir") { + validate_queue_dir(queue_dir.as_str()).await?; + if let Some(qos) = kv_map.get("qos") { + match qos.parse::() { + Ok(1) | Ok(2) => {} + Ok(0) => return Err(s3_error!(InvalidArgument, "qos should be 1 or 2 if queue_dir is set")), + _ => return Err(s3_error!(InvalidArgument, "qos must be an integer 0, 1, or 2")), + } + } + } + } + + let mut kvs = rustfs_ecstore::config::KVS::new(); + for (key, value) in kv_map { + kvs.insert(key, value); + } + kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + + update_audit_config_and_reload(|config| { + config + .0 + .entry(target_type.to_lowercase()) + .or_default() + .insert(target_name.to_lowercase(), kvs.clone()); + true + }) + .await?; + + Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + } +} + +pub struct ListAuditTargets {} + +#[async_trait::async_trait] +impl Operation for ListAuditTargets { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + let span = Span::current(); + let _enter = span.enter(); + check_permissions(&req).await?; + + let mut runtime_statuses = HashMap::new(); + if let Some(system) = audit_system() { + let targets = system.get_target_values().await; + let semaphore = Arc::new(Semaphore::new(10)); + let mut futures = FuturesUnordered::new(); + + for target in targets { + let sem = Arc::clone(&semaphore); + futures.push(async move { + let _permit = sem.acquire().await; + let status = match timeout(Duration::from_secs(3), target.is_active()).await { + Ok(Ok(true)) => "online", + _ => "offline", + }; + ((target.id().id.clone(), target.id().name.to_string()), status.to_string()) + }); + } + + while let Some((key, status)) = futures.next().await { + runtime_statuses.insert(key, status); + } + } + + let config = load_server_config_from_store().await?; + let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses); + let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }) + .map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?; + + Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + } +} + +pub struct RemoveAuditTarget {} + +#[async_trait::async_trait] +impl Operation for RemoveAuditTarget { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + let span = Span::current(); + let _enter = span.enter(); + let (target_type, target_name) = extract_target_params(¶ms)?; + + check_permissions(&req).await?; + let config_snapshot = load_server_config_from_store().await?; + if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + update_audit_config_and_reload(|config| { + let mut changed = false; + if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) { + if targets.remove(&target_name.to_lowercase()).is_some() { + changed = true; + } + if targets.is_empty() { + config.0.remove(&target_type.to_lowercase()); + } + } + changed + }) + .await?; + + Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_ecstore::config::{KV, KVS}; + use std::collections::{HashMap, HashSet}; + use temp_env::{with_var, with_vars}; + + fn enabled_kvs(value: &str) -> KVS { + KVS(vec![KV { + key: ENABLE_KEY.to_string(), + value: value.to_string(), + hidden_if_empty: false, + }]) + } + + #[test] + fn merge_audit_endpoints_marks_config_env_and_mixed_sources() { + let config = Config(HashMap::from([( + AUDIT_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([ + ("mixed-target".to_string(), enabled_kvs("on")), + ("config-target".to_string(), enabled_kvs("on")), + ]), + )])); + + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")), + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), + ], + || { + let runtime = HashMap::from([ + (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()), + (("env-only".to_string(), "webhook".to_string()), "online".to_string()), + ]); + let merged = merge_audit_endpoints(&config, runtime); + + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-target") + .expect("mixed target should be present"); + assert_eq!(mixed.source, AuditEndpointSource::Mixed); + + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-only") + .expect("env-only target should be present"); + assert_eq!(env_only.source, AuditEndpointSource::Env); + + let config_only = merged + .iter() + .find(|entry| entry.account_id == "config-target") + .expect("config target should be present"); + assert_eq!(config_only.source, AuditEndpointSource::Config); + }, + ); + } + + #[test] + fn audit_target_mutation_block_reason_rejects_env_managed_target() { + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")), + ], + || { + let config = Config(HashMap::new()); + let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary"); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("managed by environment variables")); + }, + ); + } + + #[test] + fn audit_target_mutation_block_reason_rejects_mixed_target() { + with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || { + let config = Config(HashMap::from([( + AUDIT_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("primary".to_string(), enabled_kvs("on"))]), + )])); + let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary"); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("both persisted config and environment variables")); + }); + } + + #[test] + fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() { + let config = Config(HashMap::from([( + AUDIT_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]), + )])); + + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")), + ], + || { + let merged = merge_audit_endpoints(&config, HashMap::new()); + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-disabled") + .expect("mixed target should be present"); + assert_eq!(mixed.source, AuditEndpointSource::Mixed); + assert_eq!(mixed.status, "offline"); + }, + ); + } + + #[test] + fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() { + let config = Config(HashMap::new()); + + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), + ], + || { + let merged = merge_audit_endpoints(&config, HashMap::new()); + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-only") + .expect("env-only target should be present"); + assert_eq!(env_only.source, AuditEndpointSource::Env); + assert_eq!(env_only.status, "offline"); + }, + ); + } + + #[test] + fn collect_validated_key_values_rejects_duplicate_keys() { + let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect(); + let key_values = vec![ + KeyValue { + key: "endpoint".to_string(), + value: "https://example.com/one".to_string(), + }, + KeyValue { + key: "endpoint".to_string(), + value: "https://example.com/two".to_string(), + }, + ]; + + let err = collect_validated_key_values(&key_values, &allowed_keys, AUDIT_WEBHOOK_SUB_SYS).unwrap_err(); + assert!(err.to_string().contains("duplicate key")); + } + + #[test] + fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() { + let config = Config(HashMap::from([( + AUDIT_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]), + )])); + + with_vars( + [ + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")), + ], + || { + let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]); + let merged = merge_audit_endpoints(&config, runtime); + let mixed = merged + .iter() + .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook") + .expect("mixed target should be present"); + assert_eq!(mixed.source, AuditEndpointSource::Mixed); + }, + ); + } + + #[test] + fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() { + let config = Config(HashMap::from([( + AUDIT_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]), + )])); + + assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none()); + } +} diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index 4d0c879ee..07b6f73b0 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -16,21 +16,23 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::ADMIN_PREFIX; use futures::stream::{FuturesUnordered, StreamExt}; +use hashbrown::HashSet as HbHashSet; use http::{HeaderMap, StatusCode}; use hyper::Method; use matchit::Params; -use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; -use rustfs_config::{ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_config::notify::{ + NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, +}; +use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_ecstore::config::Config; use rustfs_targets::check_mqtt_broker_available; use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::io::{Error, ErrorKind}; -use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; -use tokio::net::lookup_host; use tokio::sync::Semaphore; use tokio::time::{Duration, sleep, timeout}; use tracing::{Span, info, warn}; @@ -80,6 +82,7 @@ struct NotificationEndpoint { account_id: String, service: String, status: String, + source: NotificationEndpointSource, } #[derive(Serialize, Debug)] @@ -87,6 +90,21 @@ struct NotificationEndpointsResponse { notification_endpoints: Vec, } +type EndpointKey = (String, String); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +enum NotificationEndpointSource { + Config, + Env, + Mixed, + Runtime, +} + +fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey { + (account_id.to_lowercase(), service.to_string()) +} + // --- Helper Functions --- async fn check_permissions(req: &S3Request) -> S3Result<()> { @@ -155,6 +173,215 @@ async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> { Ok(()) } +fn config_enable_is_on(value: &str) -> bool { + matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1") +} + +fn collect_configured_endpoint_keys(config: &Config) -> Vec { + let mut endpoints = Vec::new(); + for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + + for (target_name, kvs) in targets { + if target_name == DEFAULT_DELIMITER { + continue; + } + let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false); + if enabled { + endpoints.push((target_name.clone(), service.to_string())); + } + } + } + endpoints +} + +fn collect_config_entry_keys(config: &Config) -> HbHashSet { + let mut endpoints = HbHashSet::new(); + for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + + for target_name in targets.keys() { + if target_name == DEFAULT_DELIMITER { + continue; + } + endpoints.insert(normalized_endpoint_key(target_name, service)); + } + } + endpoints +} + +fn collect_env_endpoint_keys() -> HbHashSet { + let mut endpoints = HbHashSet::new(); + + for (service, valid_keys) in [("webhook", NOTIFY_WEBHOOK_KEYS), ("mqtt", NOTIFY_MQTT_KEYS)] { + let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase(); + + for (key, _value) in std::env::vars() { + let Some(rest) = key.strip_prefix(&env_prefix) else { + continue; + }; + + let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER); + let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER); + let field_name_part = parts.next(); + + let (field_name, instance_id) = match field_name_part { + Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()), + None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()), + }; + + if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() { + continue; + } + + if valid_keys.contains(&field_name.as_str()) { + endpoints.insert(normalized_endpoint_key(&instance_id, service)); + } + } + } + + endpoints +} + +fn classify_notification_endpoint_source( + config_targets: &HbHashSet, + env_targets: &HbHashSet, + key: &EndpointKey, +) -> NotificationEndpointSource { + match (config_targets.contains(key), env_targets.contains(key)) { + (true, true) => NotificationEndpointSource::Mixed, + (true, false) => NotificationEndpointSource::Config, + (false, true) => NotificationEndpointSource::Env, + (false, false) => NotificationEndpointSource::Runtime, + } +} + +fn notification_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> NotificationEndpointSource { + let config_targets = collect_config_entry_keys(config); + let env_targets = collect_env_endpoint_keys(); + let service = match target_type { + NOTIFY_WEBHOOK_SUB_SYS => "webhook", + NOTIFY_MQTT_SUB_SYS => "mqtt", + _ => "", + }; + + let key = normalized_endpoint_key(target_name, service); + classify_notification_endpoint_source(&config_targets, &env_targets, &key) +} + +fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { + match notification_endpoint_source(config, target_type, target_name) { + NotificationEndpointSource::Env => Some(format!( + "target '{}' is managed by environment variables and cannot be modified from the console", + target_name + )), + NotificationEndpointSource::Mixed => Some(format!( + "target '{}' is configured by both persisted config and environment variables; remove the environment variables first", + target_name + )), + NotificationEndpointSource::Config | NotificationEndpointSource::Runtime => None, + } +} + +fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap) -> Vec { + let mut notification_endpoints = Vec::new(); + let mut seen = HashSet::new(); + let configured_keys = collect_configured_endpoint_keys(config); + let config_targets = collect_config_entry_keys(config); + let env_targets = collect_env_endpoint_keys(); + let mut normalized_runtime_statuses: HashMap = HashMap::new(); + for ((account_id, service), status) in runtime_statuses { + let normalized = normalized_endpoint_key(&account_id, &service); + normalized_runtime_statuses + .entry(normalized) + .or_insert((account_id, service, status)); + } + + for key in configured_keys { + let normalized = normalized_endpoint_key(&key.0, &key.1); + if !seen.insert(normalized.clone()) { + continue; + } + let status = normalized_runtime_statuses + .remove(&normalized) + .map(|(_, _, status)| status) + .unwrap_or_else(|| "offline".to_string()); + let source = classify_notification_endpoint_source(&config_targets, &env_targets, &normalized); + notification_endpoints.push(NotificationEndpoint { + account_id: key.0, + service: key.1, + status, + source, + }); + } + + for (normalized, (account_id, service, status)) in normalized_runtime_statuses { + if seen.insert(normalized.clone()) { + notification_endpoints.push(NotificationEndpoint { + account_id, + service, + status, + source: classify_notification_endpoint_source(&config_targets, &env_targets, &normalized), + }); + } + } + + for key in &env_targets { + if !seen.insert(key.clone()) { + continue; + } + + notification_endpoints.push(NotificationEndpoint { + account_id: key.0.clone(), + service: key.1.clone(), + status: "offline".to_string(), + source: classify_notification_endpoint_source(&config_targets, &env_targets, key), + }); + } + + notification_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); + notification_endpoints +} + +fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec { + target_statuses + .into_iter() + .filter_map(|(target_id, status)| (status == "online").then(|| target_id.to_arn(region).to_string())) + .collect() +} + +fn collect_validated_key_values( + key_values: &[KeyValue], + allowed_keys: &HashSet<&str>, + target_type: &str, +) -> S3Result> { + let mut kv_map = HashMap::new(); + let mut seen = HashSet::new(); + + for kv in key_values { + if !allowed_keys.contains(kv.key.as_str()) { + return Err(s3_error!( + InvalidArgument, + "key '{}' not allowed for target type '{}'", + kv.key, + target_type + )); + } + + if !seen.insert(kv.key.as_str()) { + return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key)); + } + + kv_map.insert(kv.key.clone(), kv.value.clone()); + } + + Ok(kv_map) +} + // --- Operations --- pub struct NotificationTarget {} @@ -167,6 +394,10 @@ impl Operation for NotificationTarget { check_permissions(&req).await?; let ns = get_notification_system()?; + let config_snapshot = ns.config.read().await.clone(); + if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } let mut input = req.input; let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| { @@ -183,37 +414,27 @@ impl Operation for NotificationTarget { _ => unreachable!(), }; - let kv_map: HashMap<&str, &str> = notification_body - .key_values - .iter() - .map(|kv| (kv.key.as_str(), kv.value.as_str())) - .collect(); - - // Validate keys - for key in kv_map.keys() { - if !allowed_keys.contains(key) { - return Err(s3_error!(InvalidArgument, "key '{}' not allowed for target type '{}'", key, target_type)); - } - } + let kv_map = collect_validated_key_values(¬ification_body.key_values, &allowed_keys, target_type)?; // Type-specific validation if target_type == NOTIFY_WEBHOOK_SUB_SYS { let endpoint = kv_map .get("endpoint") + .map(String::as_str) .ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?; - let url = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?; - let host = url - .host_str() - .ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing host"))?; - let port = url - .port_or_known_default() - .ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing port"))?; - let addr = format!("{host}:{port}"); - if addr.parse::().is_err() && lookup_host(&addr).await.is_err() { - return Err(s3_error!(InvalidArgument, "invalid or unresolvable endpoint address")); + let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?; + match parsed_endpoint.scheme() { + "http" | "https" => {} + other => { + return Err(s3_error!( + InvalidArgument, + "unsupported endpoint scheme: {} (only http and https are allowed)", + other + )); + } } if let Some(queue_dir) = kv_map.get("queue_dir") { - validate_queue_dir(queue_dir).await?; + validate_queue_dir(queue_dir.as_str()).await?; } if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") { return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair")); @@ -221,18 +442,20 @@ impl Operation for NotificationTarget { } else if target_type == NOTIFY_MQTT_SUB_SYS { let endpoint = kv_map .get(rustfs_config::MQTT_BROKER) + .map(String::as_str) .ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?; let topic = kv_map .get(rustfs_config::MQTT_TOPIC) + .map(String::as_str) .ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?; - let username = kv_map.get(rustfs_config::MQTT_USERNAME).copied(); - let password = kv_map.get(rustfs_config::MQTT_PASSWORD).copied(); + let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str); + let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str); check_mqtt_broker_available(endpoint, topic, username, password) .await .map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?; if let Some(queue_dir) = kv_map.get("queue_dir") { - validate_queue_dir(queue_dir).await?; + validate_queue_dir(queue_dir.as_str()).await?; if let Some(qos) = kv_map.get("qos") { match qos.parse::() { Ok(1) | Ok(2) => {} @@ -243,24 +466,14 @@ impl Operation for NotificationTarget { } } - let mut kvs_vec: Vec<_> = notification_body - .key_values - .into_iter() - .map(|kv| rustfs_ecstore::config::KV { - key: kv.key, - value: kv.value, - hidden_if_empty: false, - }) - .collect(); - - kvs_vec.push(rustfs_ecstore::config::KV { - key: ENABLE_KEY.to_string(), - value: EnableState::On.to_string(), - hidden_if_empty: false, - }); + let mut kvs = rustfs_ecstore::config::KVS::new(); + for (key, value) in kv_map { + kvs.insert(key, value); + } + kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); info!("Setting target config for type '{}', name '{}'", target_type, target_name); - ns.set_target_config(target_type, target_name, rustfs_ecstore::config::KVS(kvs_vec)) + ns.set_target_config(target_type, target_name, kvs) .await .map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?; @@ -278,8 +491,6 @@ impl Operation for ListNotificationTargets { let ns = get_notification_system()?; let targets = ns.get_target_values().await; - let target_count = targets.len(); - let semaphore = Arc::new(Semaphore::new(10)); let mut futures = FuturesUnordered::new(); @@ -291,18 +502,16 @@ impl Operation for ListNotificationTargets { Ok(Ok(true)) => "online", _ => "offline", }; - NotificationEndpoint { - account_id: target.id().id.clone(), - service: target.id().name.to_string(), - status: status.to_string(), - } + ((target.id().id.clone(), target.id().name.to_string()), status.to_string()) }); } - let mut notification_endpoints = Vec::with_capacity(target_count); - while let Some(endpoint) = futures.next().await { - notification_endpoints.push(endpoint); + let mut runtime_statuses = HashMap::new(); + while let Some((key, status)) = futures.next().await { + runtime_statuses.insert(key, status); } + let config = ns.config.read().await.clone(); + let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses); let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }) .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?; @@ -320,16 +529,32 @@ impl Operation for ListTargetsArns { check_permissions(&req).await?; let ns = get_notification_system()?; - let active_targets = ns.get_active_targets().await; + let targets = ns.get_target_values().await; let region = req .region .clone() .ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?; + let semaphore = Arc::new(Semaphore::new(10)); + let mut futures = FuturesUnordered::new(); - let data_target_arn_list: Vec<_> = active_targets - .iter() - .map(|id| id.to_arn(region.as_str()).to_string()) - .collect(); + for target in targets { + let sem = Arc::clone(&semaphore); + futures.push(async move { + let _permit = sem.acquire().await; + let status = match timeout(Duration::from_secs(3), target.is_active()).await { + Ok(Ok(true)) => "online", + _ => "offline", + }; + (target.id(), status.to_string()) + }); + } + + let mut target_statuses = Vec::new(); + while let Some(target_status) = futures.next().await { + target_statuses.push(target_status); + } + + let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses); let data = serde_json::to_vec(&data_target_arn_list) .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?; @@ -348,6 +573,10 @@ impl Operation for RemoveNotificationTarget { check_permissions(&req).await?; let ns = get_notification_system()?; + let config_snapshot = ns.config.read().await.clone(); + if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } info!("Removing target config for type '{}', name '{}'", target_type, target_name); ns.remove_target_config(target_type, target_name) @@ -372,3 +601,284 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, & let target_name = extract_param(params, "target_name")?; Ok((target_type, target_name)) } + +#[cfg(test)] +mod tests { + use super::*; + use rustfs_ecstore::config::{KV, KVS}; + use rustfs_targets::arn::TargetID; + use std::collections::{HashMap, HashSet}; + use temp_env::{with_var, with_vars}; + + fn enabled_kvs(value: &str) -> KVS { + KVS(vec![KV { + key: ENABLE_KEY.to_string(), + value: value.to_string(), + hidden_if_empty: false, + }]) + } + + #[test] + fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() { + let mut cfg_map = HashMap::new(); + cfg_map.insert( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("webhook-a".to_string(), enabled_kvs("on"))]), + ); + cfg_map.insert( + NOTIFY_MQTT_SUB_SYS.to_string(), + HashMap::from([("mqtt-a".to_string(), enabled_kvs("on"))]), + ); + let config = Config(cfg_map); + + let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]); + let merged = merge_notification_endpoints(&config, runtime); + + let mqtt = merged + .iter() + .find(|entry| entry.account_id == "mqtt-a" && entry.service == "mqtt") + .expect("mqtt-a should be present"); + assert_eq!(mqtt.status, "offline"); + assert_eq!(mqtt.source, NotificationEndpointSource::Config); + + let webhook = merged + .iter() + .find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook") + .expect("webhook-a should be present"); + assert_eq!(webhook.status, "online"); + assert_eq!(webhook.source, NotificationEndpointSource::Config); + } + + #[test] + fn merge_notification_endpoints_skips_disabled_and_default_entries() { + let mut webhook_targets = HashMap::new(); + webhook_targets.insert(DEFAULT_DELIMITER.to_string(), enabled_kvs("on")); + webhook_targets.insert("webhook-disabled".to_string(), enabled_kvs("off")); + webhook_targets.insert("webhook-enabled".to_string(), enabled_kvs("on")); + let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)])); + + let runtime = HashMap::from([ + (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()), + (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()), + ]); + let merged = merge_notification_endpoints(&config, runtime); + + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-only" && entry.service == "mqtt") + .expect("env-only should be present"); + assert_eq!(env_only.status, "offline"); + assert_eq!(env_only.source, NotificationEndpointSource::Runtime); + + let enabled = merged + .iter() + .find(|entry| entry.account_id == "webhook-enabled" && entry.service == "webhook") + .expect("webhook-enabled should be present"); + assert_eq!(enabled.status, "online"); + assert_eq!(enabled.source, NotificationEndpointSource::Config); + } + + #[test] + fn merge_notification_endpoints_marks_env_and_mixed_sources() { + let config = Config(HashMap::from([ + ( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("mixed-target".to_string(), enabled_kvs("on"))]), + ), + ( + NOTIFY_MQTT_SUB_SYS.to_string(), + HashMap::from([("config-target".to_string(), enabled_kvs("on"))]), + ), + ])); + + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")), + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), + ], + || { + let runtime = HashMap::from([ + (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()), + (("env-only".to_string(), "webhook".to_string()), "online".to_string()), + ]); + let merged = merge_notification_endpoints(&config, runtime); + + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-target") + .expect("mixed target should be present"); + assert_eq!(mixed.source, NotificationEndpointSource::Mixed); + + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-only") + .expect("env-only target should be present"); + assert_eq!(env_only.source, NotificationEndpointSource::Env); + + let config_only = merged + .iter() + .find(|entry| entry.account_id == "config-target") + .expect("config target should be present"); + assert_eq!(config_only.source, NotificationEndpointSource::Config); + }, + ); + } + + #[test] + fn target_mutation_block_reason_rejects_env_managed_target() { + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")), + ], + || { + let config = Config(HashMap::new()); + let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary"); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("managed by environment variables")); + }, + ); + } + + #[test] + fn target_mutation_block_reason_rejects_mixed_target() { + with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || { + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("primary".to_string(), enabled_kvs("on"))]), + )])); + let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary"); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("both persisted config and environment variables")); + }); + } + + #[test] + fn target_mutation_block_reason_allows_config_only_target() { + let target_name = "config-only-target"; + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([(target_name.to_string(), enabled_kvs("on"))]), + )])); + assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none()); + } + + #[test] + fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() { + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]), + )])); + + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")), + ], + || { + let merged = merge_notification_endpoints(&config, HashMap::new()); + let mixed = merged + .iter() + .find(|entry| entry.account_id == "mixed-disabled") + .expect("mixed target should be present"); + assert_eq!(mixed.source, NotificationEndpointSource::Mixed); + assert_eq!(mixed.status, "offline"); + }, + ); + } + + #[test] + fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() { + let config = Config(HashMap::new()); + + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), + ], + || { + let merged = merge_notification_endpoints(&config, HashMap::new()); + let env_only = merged + .iter() + .find(|entry| entry.account_id == "env-only") + .expect("env-only target should be present"); + assert_eq!(env_only.source, NotificationEndpointSource::Env); + assert_eq!(env_only.status, "offline"); + }, + ); + } + + #[test] + fn collect_validated_key_values_rejects_duplicate_keys() { + let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect(); + let key_values = vec![ + KeyValue { + key: "endpoint".to_string(), + value: "https://example.com/one".to_string(), + }, + KeyValue { + key: "endpoint".to_string(), + value: "https://example.com/two".to_string(), + }, + ]; + + let err = collect_validated_key_values(&key_values, &allowed_keys, NOTIFY_WEBHOOK_SUB_SYS).unwrap_err(); + assert!(err.to_string().contains("duplicate key")); + } + + #[test] + fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() { + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]), + )])); + + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")), + ], + || { + let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]); + let merged = merge_notification_endpoints(&config, runtime); + let mixed = merged + .iter() + .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook") + .expect("mixed target should be present"); + assert_eq!(mixed.source, NotificationEndpointSource::Mixed); + }, + ); + } + + #[test] + fn collect_online_target_arns_filters_offline_targets() { + let arns = collect_online_target_arns( + "us-east-1", + vec![ + (TargetID::new("webhook-a".to_string(), "webhook".to_string()), "online".to_string()), + (TargetID::new("mqtt-a".to_string(), "mqtt".to_string()), "offline".to_string()), + ], + ); + + assert_eq!(arns, vec!["arn:rustfs:sqs:us-east-1:webhook-a:webhook".to_string()]); + } + + #[test] + fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() { + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]), + )])); + + with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", None::<&str>), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>), + ], + || { + assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none()); + }, + ); + } +} diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 522cb0056..f26f3ea89 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. pub mod account_info; +pub mod audit; pub mod bucket_meta; pub mod event; pub mod group; @@ -51,6 +52,7 @@ mod tests { fn test_handler_struct_creation() { // Test that handler structs can be created let _account_handler = account_info::AccountInfoHandler {}; + let _list_audit_targets = audit::ListAuditTargets {}; let _service_handler = system::ServiceHandle {}; let _server_info_handler = system::ServerInfoHandler {}; let _inspect_data_handler = system::InspectDataHandler {}; diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index bf54583ab..588781ecc 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -25,8 +25,8 @@ mod console_test; mod route_registration_test; use handlers::{ - bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system, - tier, user, + audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, + system, tier, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -55,6 +55,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result quota::register_quota_route(&mut r)?; bucket_meta::register_bucket_meta_route(&mut r)?; + audit::register_audit_target_route(&mut r)?; replication::register_replication_route(&mut r)?; site_replication::register_site_replication_route(&mut r)?; diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index d58556e8e..d490d4d9d 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -14,8 +14,8 @@ use crate::admin::{ handlers::{ - bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system, - tier, user, + audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, + system, tier, user, }, router::{AdminOperation, S3Router}, }; @@ -50,6 +50,7 @@ fn register_admin_routes(router: &mut S3Router) { tier::register_tier_route(router).expect("register tier route"); quota::register_quota_route(router).expect("register quota route"); bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route"); + audit::register_audit_target_route(router).expect("register audit target route"); replication::register_replication_route(router).expect("register replication route"); site_replication::register_site_replication_route(router).expect("register site replication route"); profile_admin::register_profiling_route(router).expect("register profile route"); @@ -60,7 +61,6 @@ fn register_admin_routes(router: &mut S3Router) { #[test] fn test_register_routes_cover_representative_admin_paths() { let mut router: S3Router = S3Router::new(false); - register_admin_routes(&mut router); assert_route(&router, Method::GET, HEALTH_PREFIX); assert_route(&router, Method::HEAD, HEALTH_PREFIX); @@ -91,6 +91,9 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach")); assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities")); assert_route(&router, Method::GET, &admin_path("/v3/target/list")); + assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list")); + assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit")); + assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset")); assert_route(&router, Method::GET, &admin_path("/v3/accountinfo")); assert_route(&router, Method::POST, &admin_path("/v3/service")); @@ -165,7 +168,6 @@ fn test_register_routes_cover_representative_admin_paths() { #[test] fn test_admin_alias_paths_match_existing_admin_routes() { let mut router: S3Router = S3Router::new(false); - register_admin_routes(&mut router); for (method, path) in [ diff --git a/rustfs/src/server/audit.rs b/rustfs/src/server/audit.rs index 98105be0a..7809684cd 100644 --- a/rustfs/src/server/audit.rs +++ b/rustfs/src/server/audit.rs @@ -14,13 +14,27 @@ use crate::app::context::resolve_server_config; use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState}; -use rustfs_config::DEFAULT_DELIMITER; use tracing::{info, warn}; fn server_config_from_context() -> Option { resolve_server_config() } +fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool { + for subsystem in [ + rustfs_config::audit::AUDIT_MQTT_SUB_SYS, + rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, + ] { + let Some(targets) = config.0.get(subsystem) else { + continue; + }; + if targets.keys().any(|key| key != rustfs_config::DEFAULT_DELIMITER) { + return true; + } + } + false +} + /// Start the audit system. /// This function checks if the audit subsystem is configured in the global server configuration. /// If configured, it initializes and starts the audit system. @@ -55,10 +69,8 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> { "The global server configuration is loaded" ); // 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't - let mqtt_config = server_config.get_value(rustfs_config::audit::AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER); - let webhook_config = server_config.get_value(rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER); - - if mqtt_config.is_none() && webhook_config.is_none() { + let has_targets = has_any_audit_targets(&server_config); + if !has_targets { info!( target: "rustfs::main::start_audit_system", "Audit subsystem (MQTT/Webhook) is not configured, and audit system initialization is skipped." @@ -68,9 +80,7 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> { info!( target: "rustfs::main::start_audit_system", - "Audit subsystem configuration detected (MQTT: {}, Webhook: {}) and started initializing the audit system.", - mqtt_config.is_some(), - webhook_config.is_some() + "Audit subsystem configuration detected and started initializing the audit system." ); // 3. Initialize and start the audit system let system = init_audit_system(); diff --git a/scripts/run.sh b/scripts/run.sh index 5aa862da5..36c4f788c 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -71,7 +71,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -#export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging +export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep