gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(admin): add audit target APIs and harden target source handling (#2350)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-04 09:07:22 +08:00
committed by GitHub
parent 67863630b2
commit d2901fd78c
29 changed files with 3534 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+3
View File
@@ -7703,6 +7703,7 @@ dependencies = [
"rustfs-targets",
"serde",
"serde_json",
"temp-env",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -7897,6 +7898,7 @@ dependencies = [
"rustfs-utils",
"s3s",
"serde",
"tempfile",
"thiserror 2.0.18",
"time",
"tokio",
@@ -8396,6 +8398,7 @@ name = "rustfs-targets"
version = "0.0.5"
dependencies = [
"async-trait",
"criterion",
"reqwest 0.13.2",
"rumqttc",
"rustfs-config",
crates/audit/Cargo.toml
+2
View File
@@ -44,6 +44,8 @@ tracing = { workspace = true, features = ["std", "attributes"] }
url = { workspace = true }
rumqttc = { workspace = true }
[dev-dependencies]
temp-env = { workspace = true }
[lints]
workspace = true
crates/audit/src/registry.rs
+10 -86
View File
@@ -109,9 +109,6 @@ impl AuditRegistry {
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
// A collection of asynchronous tasks for concurrently executing target creation
let mut tasks = FuturesUnordered::new();
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
// Record the defaults for each segment so that the segment can eventually be rebuilt
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
// 1. Traverse all registered plants and process them by target type
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
@@ -125,9 +122,6 @@ impl AuditRegistry {
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
debug!(?default_cfg, "Get the default configuration");
// Save defaults for eventual write back
section_defaults.insert(section_name.clone(), default_cfg.clone());
// *** Optimization point 1: Get all legitimate fields of the current target type ***
let valid_fields = factory.get_valid_fields();
debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -235,103 +229,28 @@ impl AuditRegistry {
if enabled {
info!(instance_id = %id, "Target is enabled, ready to create a task");
// 5.3. Create asynchronous tasks for enabled instances
let target_type_clone = target_type.clone();
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(target_type_clone, tid, result, Arc::clone(&merged_config_arc))
(tid, result)
});
} else {
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
// Remove disabled target from final configuration
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
info!(instance_id = %id, "Skip disabled target");
}
}
}
// 6. Concurrently execute all creation tasks and collect results
let mut successful_targets = Vec::new();
let mut successful_configs = Vec::new();
while let Some((target_type, id, result, final_config)) = tasks.next().await {
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
successful_targets.push(target);
successful_configs.push((target_type, id, final_config));
}
Err(e) => {
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
}
}
}
// 7. Aggregate new configuration and write back to system configuration
if !successful_configs.is_empty() || !section_defaults.is_empty() {
info!(
"Prepare to update {} successfully created target configurations to the system configuration...",
successful_configs.len()
);
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
for (target_type, id, kvs) in successful_configs {
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
successes_by_section
.entry(section_name)
.or_default()
.insert(id.to_lowercase(), (*kvs).clone());
}
let mut new_config = config.clone();
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
let mut sections: HashSet<String> = HashSet::new();
sections.extend(section_defaults.keys().cloned());
sections.extend(successes_by_section.keys().cloned());
for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item
if let Some(default_kvs) = section_defaults.get(&section)
&& !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
}
// Add successful instance item
if let Some(instances) = successes_by_section.get(&section) {
for (id, kvs) in instances {
section_map.insert(id.clone(), kvs.clone());
}
}
// Empty breaks are removed and non-empty breaks are replaced entirely.
if section_map.is_empty() {
new_config.0.remove(&section);
} else {
new_config.0.insert(section, section_map);
}
}
if &new_config == config {
info!("Audit target configuration unchanged, skip persisting server config");
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Err(AuditError::StorageNotAvailable(
"Failed to save target configuration: server storage not initialized".to_string(),
));
};
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
Ok(_) => {
info!("The new configuration was saved to the system successfully.")
}
Err(e) => {
error!("Failed to save the new configuration: {}", e);
return Err(AuditError::SaveConfig(Box::new(e)));
error!(instance_id = %id, error = %e, "Failed to create a target");
}
}
}
@@ -371,6 +290,11 @@ impl AuditRegistry {
self.targets.get(id).map(|t| t.as_ref())
}
/// Lists cloned target values for runtime inspection without exposing mutable registry access.
pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
self.targets.values().map(|target| target.clone_dyn()).collect()
}
/// Lists all target IDs
///
/// # Returns
crates/audit/src/system.rs
+160 -107
View File
@@ -13,14 +13,15 @@
// limitations under the License.
use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
use hashbrown::HashMap;
use rustfs_ecstore::config::Config;
use rustfs_targets::{
StoreError, Target, TargetError,
store::{Key, Store},
target::EntityTarget,
target::{EntityTarget, QueuedPayload},
};
use std::sync::Arc;
use tokio::sync::{Mutex, RwLock};
use tokio::sync::{Mutex, RwLock, mpsc};
use tracing::{error, info, warn};
/// State of the audit system
@@ -39,6 +40,8 @@ pub struct AuditSystem {
registry: Arc<Mutex<AuditRegistry>>,
state: Arc<RwLock<AuditSystemState>>,
config: Arc<RwLock<Option<Config>>>,
/// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
}
impl Default for AuditSystem {
@@ -54,6 +57,7 @@ impl AuditSystem {
registry: Arc::new(Mutex::new(AuditRegistry::new())),
state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
config: Arc::new(RwLock::new(None)),
stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
}
}
@@ -110,27 +114,7 @@ impl AuditSystem {
// Initialize all targets
for target in targets {
let target_id = target.id().to_string();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
} else {
// After successful initialization, if enabled and there is a store, start the send from storage task
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target");
let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
self.start_audit_stream_with_batching(store_clone, target_arc);
info!(target_id = %target_id, "Audit stream processing started");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing");
}
registry.add_target(target_id, target);
}
self.init_and_register_target(target, &mut registry).await;
}
// Update state to running
@@ -214,6 +198,9 @@ impl AuditSystem {
info!("Stopping audit system");
// Stop all stream tasks first
self.stop_all_streams().await;
// Close all targets
let mut registry = self.registry.lock().await;
if let Err(e) = registry.close_all().await {
@@ -258,52 +245,49 @@ impl AuditSystem {
let state = self.state.read().await;
match *state {
AuditSystemState::Running => {
// Continue with dispatch
info!("Dispatching audit log entry");
}
AuditSystemState::Running => {}
AuditSystemState::Paused => {
// Skip dispatch when paused
return Ok(());
}
_ => {
// Don't dispatch when not running
return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
}
}
drop(state);
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
// Collect cloned targets under lock, then dispatch without holding it
let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
if target_keys.is_empty() {
warn!("No audit targets configured for dispatch");
return Ok(());
}
if target_keys.is_empty() {
warn!("No audit targets configured for dispatch");
return Ok(());
}
// Dispatch to all targets concurrently
target_keys
.into_iter()
.filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
.collect()
};
// Dispatch to all targets concurrently (no lock held)
let mut tasks = Vec::new();
for target_key in target_keys {
if let Some(target) = registry.get_target(&target_key) {
let entry_clone = Arc::clone(&entry);
let target_key_clone = target_key.clone();
for (target_key, target) in targets {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
// Create EntityTarget for the audit log entry
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event, // Default, should be derived from entry
data: (*entry_clone).clone(),
};
let task = async move {
let result = target.save(Arc::new(entity_target)).await;
(target_key, result)
};
let task = async move {
let result = target.save(Arc::new(entity_target)).await;
(target_key_clone, result)
};
tasks.push(task);
}
tasks.push(task);
}
// Execute all dispatch tasks
@@ -359,39 +343,45 @@ impl AuditSystem {
}
drop(state);
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
// Collect targets under lock, then dispatch without holding it
let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
if target_keys.is_empty() {
warn!("No audit targets configured for batch dispatch");
return Ok(());
}
if target_keys.is_empty() {
warn!("No audit targets configured for batch dispatch");
return Ok(());
}
target_keys
.into_iter()
.filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
.collect()
};
let mut tasks = Vec::new();
for target_key in target_keys {
if let Some(target) = registry.get_target(&target_key) {
let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
let target_key_clone = target_key.clone();
for (target_key, target) in targets {
let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
let target_key_clone = target_key.clone();
let task = async move {
let mut success_count = 0;
let mut errors = Vec::new();
for entry in entries_clone {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
match target.save(Arc::new(entity_target)).await {
Ok(_) => success_count += 1,
Err(e) => errors.push(e),
}
let task = async move {
let mut success_count = 0;
let mut errors = Vec::new();
for entry in entries_clone {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
match target.save(Arc::new(entity_target)).await {
Ok(_) => success_count += 1,
Err(e) => errors.push(e),
}
(target_key_clone, success_count, errors)
};
tasks.push(task);
}
}
(target_key_clone, success_count, errors)
};
tasks.push(task);
}
let results = futures::future::join_all(tasks).await;
@@ -417,6 +407,60 @@ impl AuditSystem {
Ok(())
}
/// Stops all active audit stream tasks by sending cancellation signals.
async fn stop_all_streams(&self) {
let mut cancellers = self.stream_cancellers.write().await;
for (target_id, cancel_tx) in cancellers.drain() {
info!(target_id = %target_id, "Stopping audit stream");
let _ = cancel_tx.send(()).await;
}
}
/// Initializes a single target: runs init(), starts stream if store is present,
/// and adds it to the registry. For store-backed targets, registration and stream
/// startup proceed even if init() fails so queued entries can be drained later.
async fn init_and_register_target(
&self,
target: Box<dyn Target<AuditEntry> + Send + Sync>,
registry: &mut AuditRegistry,
) -> Option<String> {
let target_id = target.id().to_string();
let has_store = target.store().is_some();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
// Non-store targets: init failure is fatal.
if !has_store {
return None;
}
// Store-backed targets: still register and start the stream so queued
// entries can be drained when connectivity recovers.
warn!(
target_id = %target_id,
"Proceeding with store-backed audit target despite init failure"
);
}
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target");
let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
info!(target_id = %target_id, "Audit stream processing started");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing");
}
registry.add_target(target_id.clone(), target);
Some(target_id)
}
/// Starts the audit stream processing for a target with batching and retry logic
///
/// # Arguments
@@ -427,9 +471,10 @@ impl AuditSystem {
/// and attempts to send them to the specified target. It implements retry logic with exponential backoff
fn start_audit_stream_with_batching(
&self,
store: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send>,
store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<AuditEntry> + Send + Sync>,
) {
) -> mpsc::Sender<()> {
let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
let state = self.state.clone();
tokio::spawn(async move {
@@ -442,6 +487,12 @@ impl AuditSystem {
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
loop {
// Check for cancellation signal
if cancel_rx.try_recv().is_ok() {
info!("Audit stream cancelled for target: {}", target.id());
break;
}
match *state.read().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
_ => {
@@ -452,11 +503,22 @@ impl AuditSystem {
let keys: Vec<Key> = store.list();
if keys.is_empty() {
sleep(Duration::from_millis(500)).await;
tokio::select! {
_ = sleep(Duration::from_millis(500)) => {},
_ = cancel_rx.recv() => {
info!("Audit stream cancelled during idle for target: {}", target.id());
return;
}
}
continue;
}
for key in keys {
if cancel_rx.try_recv().is_ok() {
info!("Audit stream cancelled during processing for target: {}", target.id());
return;
}
let mut retries = 0usize;
let mut success = false;
@@ -497,6 +559,8 @@ impl AuditSystem {
sleep(Duration::from_millis(100)).await;
}
});
cancel_tx
}
/// Enables a specific target
@@ -594,6 +658,12 @@ impl AuditSystem {
registry.list_targets()
}
/// Returns cloned target values for read-only runtime inspection.
pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
let registry = self.registry.lock().await;
registry.list_target_values()
}
/// Gets information about a specific target
///
/// # Arguments
@@ -616,9 +686,11 @@ impl AuditSystem {
pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> {
info!("Reloading audit system configuration");
// Record config reload
observability::record_config_reload();
// Stop all existing stream tasks first
self.stop_all_streams().await;
// Store new configuration
{
let mut config_guard = self.config.write().await;
@@ -636,28 +708,9 @@ impl AuditSystem {
Ok(targets) => {
info!(target_count = targets.len(), "Reloaded audit targets successfully");
// Initialize all new targets
for target in targets {
let target_id = target.id().to_string();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target");
} else {
// Same starts the storage stream after a heavy load
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target (reload)");
let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
self.start_audit_stream_with_batching(store_clone, target_arc);
info!(target_id = %target_id, "Audit stream processing started (reload)");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)");
}
registry.add_target(target.id().to_string(), target);
if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
info!(target_id = %target_id, "Target initialized (reload)");
}
}
crates/audit/tests/integration_test.rs
+29 -20
View File
@@ -15,6 +15,7 @@
use rustfs_audit::*;
use rustfs_ecstore::config::{Config, KVS};
use std::collections::HashMap;
use temp_env::with_vars;
#[tokio::test]
async fn test_audit_system_creation() {
@@ -35,34 +36,42 @@ async fn test_config_parsing_webhook() {
let mut config = Config(HashMap::new());
let mut audit_webhook_section = HashMap::new();
// Create default configuration
let mut default_kvs = KVS::new();
default_kvs.insert("enable".to_string(), "on".to_string());
default_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
default_kvs.insert("enable".to_string(), "off".to_string());
default_kvs.insert("endpoint".to_string(), "".to_string());
audit_webhook_section.insert("_".to_string(), default_kvs);
let mut instance_kvs = KVS::new();
instance_kvs.insert("enable".to_string(), "on".to_string());
instance_kvs.insert("endpoint".to_string(), "http://localhost:3020/webhook".to_string());
audit_webhook_section.insert("primary".to_string(), instance_kvs);
config.0.insert("audit_webhook".to_string(), audit_webhook_section);
let registry = AuditRegistry::new();
// This should not fail even if server storage is not initialized
// as it's an integration test
let result = registry.create_audit_targets_from_config(&config).await;
assert!(result.is_ok(), "audit target creation should not require server storage");
}
// We expect this to fail due to server storage not being initialized
// but the parsing should work correctly
match result {
Err(AuditError::StorageNotAvailable(_)) => {
// This is expected in test environment
}
Err(e) => {
// Other errors might indicate parsing issues
println!("Unexpected error: {e}");
}
Ok(_) => {
// Unexpected success in test environment without server storage
}
}
#[test]
fn test_env_only_audit_target_does_not_require_server_storage() {
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("http://localhost:3020/webhook")),
],
|| {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("failed to create tokio runtime");
runtime.block_on(async {
let config = Config(HashMap::new());
let registry = AuditRegistry::new();
let result = registry.create_audit_targets_from_config(&config).await;
assert!(result.is_ok(), "env-only audit target creation should not require server storage");
});
},
)
}
#[test]
crates/config/src/constants/targets.rs
+7
View File
@@ -34,3 +34,10 @@ pub const MQTT_RECONNECT_INTERVAL: &str = "reconnect_interval";
pub const MQTT_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
pub const MQTT_QUEUE_DIR: &str = "queue_dir";
pub const MQTT_QUEUE_LIMIT: &str = "queue_limit";
/// Environment variable controlling whether target queue files are Snappy-compressed.
/// Applies to both notify and audit target queue stores.
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
/// Queue-store compression is enabled by default to reduce disk footprint.
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
crates/e2e_test/src/object_lambda_test.rs
+127 -12
View File
@@ -313,6 +313,31 @@ async fn list_target_arns(env: &RustFSTestEnvironment) -> Result<Vec<String>, Bo
Ok(serde_json::from_slice(&body)?)
}
async fn delete_webhook_target(env: &RustFSTestEnvironment, target_name: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/target/notify_webhook/{target_name}/reset", env.url);
let response = signed_request(http::Method::DELETE, &url, &env.access_key, &env.secret_key, None, None).await?;
let status = response.status();
let body = response.text().await.unwrap_or_default();
if status != StatusCode::OK {
return Err(format!("failed to delete webhook target {target_name}: {status} {body}").into());
}
Ok(())
}
fn notification_target_is_listed(targets: &serde_json::Value, target_name: &str) -> bool {
targets["notification_endpoints"]
.as_array()
.into_iter()
.flatten()
.any(|entry| {
entry["account_id"].as_str() == Some(target_name)
&& entry["service"]
.as_str()
.is_some_and(|service| service == "webhook" || service.starts_with("webhook-"))
})
}
async fn wait_for_target_visibility(
env: &RustFSTestEnvironment,
target_name: &str,
@@ -324,18 +349,7 @@ async fn wait_for_target_visibility(
last_targets = list_notification_targets(env).await?;
last_arns = list_target_arns(env).await?;
let listed = last_targets["notification_endpoints"]
.as_array()
.into_iter()
.flatten()
.any(|entry| {
entry["account_id"].as_str() == Some(target_name)
&& entry["service"]
.as_str()
.is_some_and(|service| service == "webhook" || service.starts_with("webhook-"))
});
if listed {
if notification_target_is_listed(&last_targets, target_name) {
return Ok((last_targets, last_arns));
}
@@ -345,10 +359,51 @@ async fn wait_for_target_visibility(
Err(format!("target {target_name} did not become visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
}
async fn wait_for_target_absence(
env: &RustFSTestEnvironment,
target_name: &str,
) -> Result<(serde_json::Value, Vec<String>), Box<dyn Error + Send + Sync>> {
let mut last_targets = serde_json::Value::Null;
let mut last_arns = Vec::new();
for _ in 0..20 {
last_targets = list_notification_targets(env).await?;
last_arns = list_target_arns(env).await?;
let listed = notification_target_is_listed(&last_targets, target_name);
let arn_listed = last_arns.iter().any(|arn| arn.ends_with(&format!(":{target_name}:webhook")));
if !listed && !arn_listed {
return Ok((last_targets, last_arns));
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
Err(format!("target {target_name} remained visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
}
async fn restart_rustfs_server(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn Error + Send + Sync>> {
env.stop_server();
env.start_rustfs_server_without_cleanup(vec![]).await
}
async fn read_persisted_server_config(env: &RustFSTestEnvironment) -> String {
let path = format!("{}/.rustfs.sys/config/config.json", env.temp_dir);
match tokio::fs::read_to_string(&path).await {
Ok(content) => content,
Err(err) if err.kind() == std::io::ErrorKind::IsADirectory => {
let mut entries = Vec::new();
match tokio::fs::read_dir(&path).await {
Ok(mut dir) => {
while let Ok(Some(entry)) = dir.next_entry().await {
entries.push(entry.file_name().to_string_lossy().to_string());
}
entries.sort();
format!("persisted config stored as object directory at {path}; entries={entries:?}")
}
Err(dir_err) => format!("persisted config directory exists at {path} but could not be listed: {dir_err}"),
}
}
Err(err) => format!("failed to read persisted config at {path}: {err}"),
}
}
@@ -400,6 +455,66 @@ async fn read_listen_notification_event(
}
}
#[tokio::test]
#[serial]
async fn test_notification_target_persists_across_restart_and_delete() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let target_name = "restart-target";
configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
let (visible_targets, visible_arns) = wait_for_target_visibility(&env, target_name).await?;
assert!(notification_target_is_listed(&visible_targets, target_name));
assert!(
visible_arns
.iter()
.any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
"target ARN missing after initial configure: {visible_arns:?}"
);
restart_rustfs_server(&mut env).await?;
let (targets_after_restart, arns_after_restart) = wait_for_target_visibility(&env, target_name).await?;
assert!(notification_target_is_listed(&targets_after_restart, target_name));
assert!(
arns_after_restart
.iter()
.any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
"target ARN missing after restart: {arns_after_restart:?}"
);
delete_webhook_target(&env, target_name).await?;
let (targets_after_delete, arns_after_delete) = wait_for_target_absence(&env, target_name).await?;
assert!(!notification_target_is_listed(&targets_after_delete, target_name));
assert!(
!arns_after_delete
.iter()
.any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
"target ARN still visible after delete: {arns_after_delete:?}"
);
restart_rustfs_server(&mut env).await?;
let (targets_after_delete_restart, arns_after_delete_restart) = wait_for_target_absence(&env, target_name).await?;
assert!(!notification_target_is_listed(&targets_after_delete_restart, target_name));
assert!(
!arns_after_delete_restart
.iter()
.any(|arn| arn.ends_with(&format!(":{target_name}:webhook"))),
"target ARN still visible after delete + restart: {arns_after_delete_restart:?}"
);
webhook_handle.abort();
let _ = webhook_handle.await;
Ok(())
}
#[tokio::test]
#[serial]
async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box<dyn Error + Send + Sync>> {
crates/ecstore/src/config/audit.rs
+17 -2
View File
@@ -16,8 +16,8 @@ use crate::config::{KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY,
WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -51,6 +51,16 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_CLIENT_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_BATCH_SIZE.to_owned(),
value: "1".to_owned(),
@@ -81,6 +91,11 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: "5s".to_owned(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
crates/ecstore/src/config/com.rs
+785 -3
View File
@@ -12,12 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, oidc, storageclass};
use crate::config::{Config, GLOBAL_STORAGE_CLASS, KVS, audit, notify, oidc, storageclass};
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
use rustfs_utils::path::SLASH_SEPARATOR;
@@ -261,6 +263,159 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
applied
}
fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
match value {
Value::String(v) => Some(v.trim().to_string()),
Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
EnableState::On.to_string()
} else {
EnableState::Off.to_string()
}),
Value::Bool(v) => Some(v.to_string()),
Value::Number(v) => Some(v.to_string()),
Value::Null => None,
_ => None,
}
}
fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
let mut kvs = KVS::new();
for (key, value) in instance {
if !valid_keys.contains(&key.as_str()) || key == COMMENT_KEY {
continue;
}
if let Some(parsed) = parse_notify_scalar_value(key, value) {
kvs.insert(key.clone(), parsed);
}
}
kvs
}
fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
match value {
Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
_ => None,
}
}
fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
section
.iter()
.any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
}
fn apply_external_notify_section(
cfg: &mut Config,
notify_obj: &Map<String, Value>,
external_key: &str,
subsystem_key: &str,
default_kvs: &KVS,
valid_keys: &[&str],
) -> bool {
let Some(Value::Object(section_obj)) = notify_obj.get(external_key).or_else(|| notify_obj.get(subsystem_key)) else {
return false;
};
if section_obj.is_empty() {
return false;
}
let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
let mut applied = false;
if is_notify_instance_shorthand(section_obj, valid_keys) {
let kvs = decode_notify_instance_object(section_obj, valid_keys);
if !kvs.is_empty() {
let mut merged = default_kvs.clone();
merged.extend(kvs);
subsystem.insert(DEFAULT_DELIMITER.to_string(), merged);
applied = true;
}
return applied;
}
for (raw_instance, value) in section_obj {
let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
continue;
};
if kvs.is_empty() {
continue;
}
let instance_key = if raw_instance == "default" {
DEFAULT_DELIMITER.to_string()
} else {
raw_instance.to_string()
};
if instance_key == DEFAULT_DELIMITER {
let mut merged = default_kvs.clone();
merged.extend(kvs);
kvs = merged;
}
subsystem.insert(instance_key, kvs);
applied = true;
}
applied
}
fn apply_external_notify_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
let Some(Value::Object(notify_obj)) = root.get("notify") else {
return false;
};
let mut applied = false;
applied |= apply_external_notify_section(
cfg,
notify_obj,
"webhook",
NOTIFY_WEBHOOK_SUB_SYS,
&notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
NOTIFY_WEBHOOK_KEYS,
);
applied |= apply_external_notify_section(
cfg,
notify_obj,
"mqtt",
NOTIFY_MQTT_SUB_SYS,
&notify::DEFAULT_NOTIFY_MQTT_KVS,
NOTIFY_MQTT_KEYS,
);
applied
}
fn apply_external_audit_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
let audit_root = root.get("audit").or_else(|| root.get("logger")).and_then(Value::as_object);
let Some(audit_obj) = audit_root else {
return false;
};
let mut applied = false;
applied |= apply_external_notify_section(
cfg,
audit_obj,
"webhook",
AUDIT_WEBHOOK_SUB_SYS,
&audit::DEFAULT_AUDIT_WEBHOOK_KVS,
AUDIT_WEBHOOK_KEYS,
);
applied |= apply_external_notify_section(
cfg,
audit_obj,
"mqtt",
AUDIT_MQTT_SUB_SYS,
&audit::DEFAULT_AUDIT_MQTT_KVS,
AUDIT_MQTT_KEYS,
);
applied
}
fn apply_external_storage_class_map(cfg: &mut Config, root: &Map<String, Value>) -> bool {
let sc = root.get("storageclass").or_else(|| root.get("storage_class"));
let Some(Value::Object(sc_obj)) = sc else {
@@ -305,8 +460,10 @@ fn decode_server_config_blob(data: &[u8]) -> Result<Config> {
let mut cfg = Config::new();
let has_storage = apply_external_storage_class_map(&mut cfg, &root);
let has_oidc = apply_external_oidc_map(&mut cfg, &root);
let has_notify = apply_external_notify_map(&mut cfg, &root);
let has_audit = apply_external_audit_map(&mut cfg, &root);
let has_header = root.contains_key("version") || root.contains_key("region") || root.contains_key("credential");
if !has_storage && !has_oidc && !has_header {
if !has_storage && !has_oidc && !has_notify && !has_audit && !has_header {
return Err(Error::other("unrecognized external server config shape"));
}
Ok(cfg)
@@ -449,6 +606,139 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
oidc_obj
}
fn is_notify_bool_key(key: &str) -> bool {
key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
}
fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
if is_notify_bool_key(key) {
if let Ok(state) = value.parse::<EnableState>() {
return Value::Bool(state.is_enabled());
}
if let Ok(boolean) = value.parse::<bool>() {
return Value::Bool(boolean);
}
}
Value::String(value.to_string())
}
fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
default_kvs
.0
.iter()
.find(|kv| kv.key == key)
.map(|kv| kv.hidden_if_empty)
.unwrap_or(false)
}
fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
let mut instance = Map::new();
for key in valid_keys {
if *key == COMMENT_KEY {
continue;
}
let baseline_value = baseline.lookup(key).unwrap_or_default();
let effective_value = kvs.lookup(key).unwrap_or_else(|| baseline_value.clone());
if effective_value == baseline_value {
continue;
}
if effective_value.trim().is_empty() && baseline_value.trim().is_empty() {
continue;
}
if is_hidden_if_empty(default_kvs, key) && effective_value.trim().is_empty() && baseline_value.trim().is_empty() {
continue;
}
instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
}
instance
}
fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
let mut merged = default_kvs.clone();
if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
merged.extend(kvs.clone());
}
merged
}
fn build_notify_subsystem_object(
cfg: &Config,
subsystem_key: &str,
default_kvs: &KVS,
valid_keys: &[&str],
) -> Map<String, Value> {
let Some(subsystem) = cfg.0.get(subsystem_key) else {
return Map::new();
};
let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
let mut subsystem_obj = Map::new();
if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
if !default_obj.is_empty() {
subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
}
}
let mut instances = subsystem
.iter()
.filter(|(instance_key, _)| instance_key.as_str() != DEFAULT_DELIMITER)
.collect::<Vec<_>>();
instances.sort_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs));
for (instance_key, kvs) in instances {
let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
if !instance_obj.is_empty() {
subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
}
}
subsystem_obj
}
fn build_notify_object(cfg: &Config) -> Map<String, Value> {
let mut notify_obj = Map::new();
let webhook_obj =
build_notify_subsystem_object(cfg, NOTIFY_WEBHOOK_SUB_SYS, &notify::DEFAULT_NOTIFY_WEBHOOK_KVS, NOTIFY_WEBHOOK_KEYS);
if !webhook_obj.is_empty() {
notify_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
}
let mqtt_obj = build_notify_subsystem_object(cfg, NOTIFY_MQTT_SUB_SYS, &notify::DEFAULT_NOTIFY_MQTT_KVS, NOTIFY_MQTT_KEYS);
if !mqtt_obj.is_empty() {
notify_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
}
notify_obj
}
fn build_audit_object(cfg: &Config) -> Map<String, Value> {
let mut audit_obj = Map::new();
let webhook_obj =
build_notify_subsystem_object(cfg, AUDIT_WEBHOOK_SUB_SYS, &audit::DEFAULT_AUDIT_WEBHOOK_KVS, AUDIT_WEBHOOK_KEYS);
if !webhook_obj.is_empty() {
audit_obj.insert("webhook".to_string(), Value::Object(webhook_obj));
}
let mqtt_obj = build_notify_subsystem_object(cfg, AUDIT_MQTT_SUB_SYS, &audit::DEFAULT_AUDIT_MQTT_KVS, AUDIT_MQTT_KEYS);
if !mqtt_obj.is_empty() {
audit_obj.insert("mqtt".to_string(), Value::Object(mqtt_obj));
}
audit_obj
}
fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8>> {
let mut root = seed.and_then(parse_object_seed).unwrap_or_default();
@@ -478,6 +768,73 @@ fn encode_server_config_blob(cfg: &Config, seed: Option<&[u8]>) -> Result<Vec<u8
root.remove(IDENTITY_OPENID_SUB_SYS);
}
let mut notify_obj = match root.remove("notify") {
Some(Value::Object(v)) => v,
_ => Map::new(),
};
let rendered_notify = build_notify_object(cfg);
match rendered_notify.get("webhook") {
Some(Value::Object(v)) => {
notify_obj.insert("webhook".to_string(), Value::Object(v.clone()));
notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
}
_ => {
notify_obj.remove("webhook");
notify_obj.remove(NOTIFY_WEBHOOK_SUB_SYS);
}
}
match rendered_notify.get("mqtt") {
Some(Value::Object(v)) => {
notify_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
}
_ => {
notify_obj.remove("mqtt");
notify_obj.remove(NOTIFY_MQTT_SUB_SYS);
}
}
if notify_obj.is_empty() {
root.remove("notify");
} else {
root.insert("notify".to_string(), Value::Object(notify_obj));
}
root.remove(NOTIFY_WEBHOOK_SUB_SYS);
root.remove(NOTIFY_MQTT_SUB_SYS);
let mut logger_obj = match root.remove("logger") {
Some(Value::Object(v)) => v,
_ => Map::new(),
};
let rendered_audit = build_audit_object(cfg);
match rendered_audit.get("webhook") {
Some(Value::Object(v)) => {
logger_obj.insert("webhook".to_string(), Value::Object(v.clone()));
logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
}
_ => {
logger_obj.remove("webhook");
logger_obj.remove(AUDIT_WEBHOOK_SUB_SYS);
}
}
match rendered_audit.get("mqtt") {
Some(Value::Object(v)) => {
logger_obj.insert("mqtt".to_string(), Value::Object(v.clone()));
logger_obj.remove(AUDIT_MQTT_SUB_SYS);
}
_ => {
logger_obj.remove("mqtt");
logger_obj.remove(AUDIT_MQTT_SUB_SYS);
}
}
if logger_obj.is_empty() {
root.remove("logger");
} else {
root.insert("logger".to_string(), Value::Object(logger_obj));
}
root.remove("audit");
root.remove(AUDIT_WEBHOOK_SUB_SYS);
root.remove(AUDIT_MQTT_SUB_SYS);
Ok(serde_json::to_vec(&Value::Object(root))?)
}
@@ -496,6 +853,8 @@ fn is_standard_object_server_config(data: &[u8]) -> bool {
fn configs_semantically_equal(lhs: &Config, rhs: &Config) -> bool {
build_storageclass_object(lhs) == build_storageclass_object(rhs)
&& build_semantic_oidc_object(lhs) == build_semantic_oidc_object(rhs)
&& build_notify_object(lhs) == build_notify_object(rhs)
&& build_audit_object(lhs) == build_audit_object(rhs)
}
fn is_object_not_found(err: &Error) -> bool {
@@ -712,7 +1071,7 @@ mod tests {
configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, is_standard_object_server_config,
read_config_with_metadata, storage_class_kvs_mut,
};
use crate::config::{Config, oidc};
use crate::config::{Config, audit, notify, oidc};
use crate::disk::endpoint::Endpoint;
use crate::endpoints::SetupType;
use crate::error::{Error, Result};
@@ -725,6 +1084,8 @@ mod tests {
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
use rustfs_filemeta::FileInfo;
@@ -1332,6 +1693,152 @@ mod tests {
);
}
#[test]
fn test_decode_server_config_reads_notify_targets() {
let input = r#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"notify":{
"webhook":{
"primary":{
"enable":true,
"endpoint":"https://example.com/hook",
"queue_dir":"/tmp/webhook-queue"
}
},
"mqtt":{
"default":{
"enable":true,
"topic":"events"
},
"analytics":{
"enable":true,
"broker":"tcp://127.0.0.1:1883",
"topic":"events",
"queue_dir":""
}
}
}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let webhook = cfg
.get_value(NOTIFY_WEBHOOK_SUB_SYS, "primary")
.expect("webhook target should be decoded");
assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/hook");
assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/webhook-queue");
let mqtt_default = cfg
.get_value(NOTIFY_MQTT_SUB_SYS, DEFAULT_DELIMITER)
.expect("mqtt default should be decoded");
assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "events");
assert_eq!(
mqtt_default.get(rustfs_config::MQTT_QUEUE_DIR),
notify::DEFAULT_NOTIFY_MQTT_KVS.get(rustfs_config::MQTT_QUEUE_DIR)
);
let mqtt = cfg
.get_value(NOTIFY_MQTT_SUB_SYS, "analytics")
.expect("mqtt target should be decoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
}
#[test]
fn test_decode_server_config_reads_notify_shorthand_default() {
let input = r#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"notify":{
"webhook":{
"enable":true,
"endpoint":"https://example.com/shorthand"
}
}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let webhook_default = cfg
.get_value(NOTIFY_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER)
.expect("default webhook config should be decoded");
assert_eq!(webhook_default.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(webhook_default.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/shorthand");
}
#[test]
fn test_decode_server_config_keeps_instance_named_like_field() {
let input = r#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"notify":{
"webhook":{
"enable":{
"enable":true,
"endpoint":"https://example.com/instance-enable"
}
}
}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let named = cfg
.get_value(NOTIFY_WEBHOOK_SUB_SYS, "enable")
.expect("instance named 'enable' should be decoded");
assert_eq!(named.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(named.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/instance-enable");
}
#[test]
fn test_decode_server_config_reads_audit_targets() {
let input = r#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1"},
"logger":{
"webhook":{
"primary":{
"enable":true,
"endpoint":"https://example.com/audit-hook",
"queue_dir":"/tmp/audit-queue"
}
},
"mqtt":{
"default":{
"enable":true,
"topic":"audit-events"
},
"analytics":{
"enable":true,
"broker":"tcp://127.0.0.1:1883",
"topic":"audit-events"
}
}
}
}"#;
let cfg = decode_server_config_blob(input.as_bytes()).expect("decode should succeed");
let webhook = cfg
.get_value(AUDIT_WEBHOOK_SUB_SYS, "primary")
.expect("audit webhook target should be decoded");
assert_eq!(webhook.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/audit-hook");
assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/audit-queue");
let mqtt_default = cfg
.get_value(AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER)
.expect("audit mqtt default should be decoded");
assert_eq!(mqtt_default.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC), "audit-events");
let mqtt = cfg
.get_value(AUDIT_MQTT_SUB_SYS, "analytics")
.expect("audit mqtt target should be decoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
}
#[test]
fn test_encode_server_config_writes_external_object_shape() {
let mut cfg = Config::new();
@@ -1388,6 +1895,174 @@ mod tests {
assert_eq!(default_provider.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
}
#[test]
fn test_encode_server_config_writes_notify_object_shape() {
let mut cfg = Config::new();
let mut webhook_section = std::collections::HashMap::new();
webhook_section.insert(DEFAULT_DELIMITER.to_string(), notify::DEFAULT_NOTIFY_WEBHOOK_KVS.clone());
webhook_section.insert(
"primary".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::WEBHOOK_ENDPOINT.to_string(),
value: "https://example.com/hook".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(),
value: "/tmp/webhook-queue".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_section);
let mut mqtt_default = notify::DEFAULT_NOTIFY_MQTT_KVS.clone();
mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "events".to_string());
let mut mqtt_section = std::collections::HashMap::new();
mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default);
mqtt_section.insert(
"analytics".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::MQTT_BROKER.to_string(),
value: "tcp://127.0.0.1:1883".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::MQTT_QUEUE_DIR.to_string(),
value: "".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
let notify = v
.get("notify")
.and_then(Value::as_object)
.expect("notify object should be present");
let webhook = notify
.get("webhook")
.and_then(Value::as_object)
.and_then(|targets| targets.get("primary"))
.and_then(Value::as_object)
.expect("webhook target should be encoded");
assert_eq!(
webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str),
Some("https://example.com/hook")
);
assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
let mqtt_default = notify
.get("mqtt")
.and_then(Value::as_object)
.and_then(|targets| targets.get("default"))
.and_then(Value::as_object)
.expect("mqtt default should be encoded");
assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("events"));
let mqtt = notify
.get("mqtt")
.and_then(Value::as_object)
.and_then(|targets| targets.get("analytics"))
.and_then(Value::as_object)
.expect("mqtt target should be encoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
}
#[test]
fn test_encode_server_config_writes_audit_object_shape() {
let mut cfg = Config::new();
let mut webhook_section = std::collections::HashMap::new();
webhook_section.insert(DEFAULT_DELIMITER.to_string(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
webhook_section.insert(
"primary".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::WEBHOOK_ENDPOINT.to_string(),
value: "https://example.com/audit-hook".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::WEBHOOK_QUEUE_DIR.to_string(),
value: "/tmp/audit-queue".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), webhook_section);
let mut mqtt_default = audit::DEFAULT_AUDIT_MQTT_KVS.clone();
mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "audit-events".to_string());
let mut mqtt_section = std::collections::HashMap::new();
mqtt_section.insert(DEFAULT_DELIMITER.to_string(), mqtt_default);
mqtt_section.insert(
"analytics".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::MQTT_BROKER.to_string(),
value: "tcp://127.0.0.1:1883".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
let logger = v
.get("logger")
.and_then(Value::as_object)
.expect("logger object should be present");
let webhook = logger
.get("webhook")
.and_then(Value::as_object)
.and_then(|targets| targets.get("primary"))
.and_then(Value::as_object)
.expect("audit webhook target should be encoded");
assert_eq!(
webhook.get(rustfs_config::WEBHOOK_ENDPOINT).and_then(Value::as_str),
Some("https://example.com/audit-hook")
);
assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
let mqtt_default = logger
.get("mqtt")
.and_then(Value::as_object)
.and_then(|targets| targets.get("default"))
.and_then(Value::as_object)
.expect("audit mqtt default should be encoded");
assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
}
#[test]
fn test_is_standard_object_server_config_detection() {
let external = br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"}}"#;
@@ -1441,6 +2116,113 @@ mod tests {
assert!(configs_semantically_equal(&lhs, &rhs));
}
#[test]
fn test_configs_semantically_equal_accounts_for_notify() {
let external = br#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
"notify":{
"webhook":{
"primary":{
"enable":true,
"endpoint":"https://example.com/hook"
}
}
}
}"#;
let legacy = br#"{
"storage_class":{"_":[
{"key":"standard","value":"EC:2"},
{"key":"rrs","value":"EC:1"},
{"key":"optimize","value":"availability"}
]},
"notify_webhook":{
"_":[
{"key":"enable","value":"off"},
{"key":"endpoint","value":""},
{"key":"queue_limit","value":"100000"},
{"key":"queue_dir","value":"/opt/rustfs/events"},
{"key":"client_cert","value":""},
{"key":"client_key","value":""},
{"key":"comment","value":""},
{"key":"client_ca","value":""},
{"key":"skip_tls_verify","value":"off"}
],
"primary":[
{"key":"enable","value":"on"},
{"key":"endpoint","value":"https://example.com/hook"}
]
}
}"#;
let lhs = decode_server_config_blob(external).expect("decode external");
let rhs = decode_server_config_blob(legacy).expect("decode legacy");
assert!(configs_semantically_equal(&lhs, &rhs));
}
#[test]
fn test_configs_semantically_equal_detects_notify_changes() {
let lhs = decode_server_config_blob(
br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/a"}}}}"#,
)
.expect("decode lhs");
let rhs = decode_server_config_blob(
br#"{"version":"33","storageclass":{"standard":"EC:2","rrs":"EC:1"},"notify":{"webhook":{"primary":{"enable":true,"endpoint":"https://example.com/b"}}}}"#,
)
.expect("decode rhs");
assert!(!configs_semantically_equal(&lhs, &rhs));
}
#[test]
fn test_configs_semantically_equal_accounts_for_audit() {
let external = br#"{
"version":"33",
"storageclass":{"standard":"EC:2","rrs":"EC:1","optimize":"availability"},
"logger":{
"webhook":{
"primary":{
"enable":true,
"endpoint":"https://example.com/audit-hook"
}
}
}
}"#;
let legacy = br#"{
"storage_class":{"_":[
{"key":"standard","value":"EC:2"},
{"key":"rrs","value":"EC:1"},
{"key":"optimize","value":"availability"}
]},
"audit_webhook":{
"_":[
{"key":"enable","value":"off"},
{"key":"endpoint","value":""},
{"key":"auth_token","value":""},
{"key":"client_cert","value":""},
{"key":"client_key","value":""},
{"key":"client_ca","value":""},
{"key":"skip_tls_verify","value":"off"},
{"key":"batch_size","value":"1"},
{"key":"queue_limit","value":"100000"},
{"key":"queue_dir","value":"/opt/rustfs/events"},
{"key":"max_retry","value":"0"},
{"key":"retry_interval","value":"3s"},
{"key":"http_timeout","value":"5s"},
{"key":"comment","value":""}
],
"primary":[
{"key":"enable","value":"on"},
{"key":"endpoint","value":"https://example.com/audit-hook"}
]
}
}"#;
let lhs = decode_server_config_blob(external).expect("decode external");
let rhs = decode_server_config_blob(legacy).expect("decode legacy");
assert!(configs_semantically_equal(&lhs, &rhs));
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn test_read_config_with_metadata_succeeds_with_one_healthy_locker_in_two_node_dist_setup() {
crates/ecstore/src/config/notify.rs
+12 -1
View File
@@ -16,7 +16,8 @@ use crate::config::{KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -60,6 +61,16 @@ pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_CLIENT_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: WEBHOOK_SKIP_TLS_VERIFY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
crates/filemeta/Cargo.toml
+1
View File
@@ -44,6 +44,7 @@ regex.workspace = true
[dev-dependencies]
criterion = { workspace = true }
tempfile = { workspace = true }
[[bench]]
name = "xl_meta_bench"
crates/filemeta/src/filemeta.rs
+5 -6
View File
@@ -1907,7 +1907,6 @@ mod test {
#[tokio::test]
async fn test_read_xl_meta_no_data() {
use tokio::fs;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
@@ -1926,13 +1925,15 @@ async fn test_read_xl_meta_no_data() {
buff.resize(buff.len() + 100, 0);
let filepath = "./test_xl.meta";
// Use tempfile to avoid conflicts with parallel tests or previous runs
let dir = tempfile::tempdir().unwrap();
let filepath = dir.path().join("test_xl.meta");
let mut file = File::create(filepath).await.unwrap();
let mut file = File::create(&filepath).await.unwrap();
// Write string data
file.write_all(&buff).await.unwrap();
let mut f = File::open(filepath).await.unwrap();
let mut f = File::open(&filepath).await.unwrap();
let stat = f.metadata().await.unwrap();
@@ -1941,7 +1942,5 @@ async fn test_read_xl_meta_no_data() {
let mut newfm = FileMeta::default();
newfm.unmarshal_msg(&data).unwrap();
fs::remove_file(filepath).await.unwrap();
assert_eq!(fm, newfm)
}
crates/notify/src/integration.rs
+108 -94
View File
@@ -18,12 +18,14 @@ use crate::{
Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
};
use hashbrown::HashMap;
use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
use rustfs_config::notify::{
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_ecstore::config::{Config, KVS};
use rustfs_s3_common::EventName;
use rustfs_targets::arn::TargetID;
use rustfs_targets::store::{Key, Store};
use rustfs_targets::target::EntityTarget;
use rustfs_targets::target::QueuedPayload;
use rustfs_targets::{StoreError, Target};
use std::collections::VecDeque;
use std::sync::Arc;
@@ -34,6 +36,21 @@ use tracing::{debug, error, info, warn};
const MAX_RECENT_LIVE_EVENTS: usize = 1024;
fn subsystem_target_type(target_type: &str) -> &str {
match target_type {
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_MQTT_SUB_SYS => "mqtt",
_ => target_type,
}
}
fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
TargetID {
id: target_name.to_lowercase(),
name: subsystem_target_type(target_type).to_string(),
}
}
#[derive(Clone)]
pub struct LiveEventBatch {
pub events: Vec<Arc<Event>>,
@@ -183,58 +200,84 @@ impl NotificationSystem {
}
}
/// Initializes targets and starts event streams for those with stores.
/// Returns a map of (target_id -> cancel_sender) for streams that were started.
async fn init_targets_and_start_streams(
&self,
targets: &[Box<dyn Target<Event> + Send + Sync>],
) -> HashMap<TargetID, mpsc::Sender<()>> {
let mut cancellers = HashMap::new();
for target in targets {
let target_id = target.id();
info!("Initializing target: {}", target_id);
let has_store = target.store().is_some();
if let Err(e) = target.init().await {
warn!("Target {} Initialization failed: {}", target_id, e);
// For targets without a store, init failure is fatal — skip.
// For store-backed targets, still start the stream so queued events
// can be drained when connectivity recovers (send_from_store retries).
if !has_store {
continue;
}
warn!(
"Target {} has a store, starting stream despite init failure — \
connectivity will be retried by send_from_store",
target_id
);
} else {
debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
}
if !target.is_enabled() {
info!("Target {} is not enabled, event stream processing is skipped", target_id);
continue;
}
if let Some(store) = target.store() {
info!("Start event stream processing for target {}", target_id);
let store_clone = store.boxed_clone();
let target_arc = Arc::from(target.clone_dyn());
let cancel_tx = self.enhanced_start_event_stream(
store_clone,
target_arc,
self.metrics.clone(),
self.concurrency_limiter.clone(),
);
let target_id_clone = target_id.clone();
cancellers.insert(target_id, cancel_tx);
info!("Event stream processing for target {} is started successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
}
cancellers
}
/// Initializes the notification system
pub async fn init(&self) -> Result<(), NotificationError> {
info!("Initialize notification system...");
let config = self.config.read().await;
debug!("Initializing notification system with config: {:?}", *config);
let config = {
let guard = self.config.read().await;
debug!("Initializing notification system with config: {:?}", *guard);
guard.clone()
};
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
info!("{} notification targets were created", targets.len());
// Initiate event stream processing for each storage enabled target
let mut cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
info!("Initializing target: {}", target.id());
// Initialize the target
if let Err(e) = target.init().await {
warn!("Target {} Initialization failed:{}", target.id(), e);
continue;
}
debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled());
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start event stream processing for target {}", target.id());
// Initialize targets and start event streams
let cancellers = self.init_targets_and_start_streams(&targets).await;
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
let target_box = target.clone_dyn();
let target_arc = Arc::from(target_box);
// Add a reference to the monitoring metrics
let metrics = self.metrics.clone();
let semaphore = self.concurrency_limiter.clone();
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
// Start event stream processing and save cancel sender
let target_id_clone = target_id.clone();
cancellers.insert(target_id, cancel_tx);
info!("Event stream processing for target {} is started successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} is not enabled, event stream processing is skipped", target_id);
}
}
// Update canceler collection
// Update canceller collection
*self.stream_cancellers.write().await = cancellers;
// Initialize the bucket target
self.notifier.init_bucket_targets(targets).await?;
info!("Notification system initialized");
@@ -333,7 +376,7 @@ impl NotificationSystem {
info!("Attempting to remove target: {}", target_id);
let ttype = target_type.to_lowercase();
let tname = target_id.name.to_lowercase();
let tname = target_id.id.to_lowercase();
self.update_config_and_reload(|config| {
let mut changed = false;
@@ -405,11 +448,7 @@ impl NotificationSystem {
let ttype = target_type.to_lowercase();
let tname = target_name.to_lowercase();
let target_id = TargetID {
id: tname.clone(),
name: ttype.clone(),
};
let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
// Deletion is prohibited if bucket rules refer to it
if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
@@ -451,7 +490,7 @@ impl NotificationSystem {
/// Enhanced event stream startup function, including monitoring and concurrency control
fn enhanced_start_event_stream(
&self,
store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
@@ -476,14 +515,13 @@ impl NotificationSystem {
let _ = cancel_tx.send(()).await;
}
// Clear the target_list and ensure that reload is a replacement reconstruction (solve the target_list len unchanged/residual problem)
// Clear the target_list and ensure that reload is a replacement reconstruction
self.notifier.remove_all_bucket_targets().await;
// Update the config
self.update_config(new_config.clone()).await;
// Create a new target from configuration
// This function will now be responsible for merging env, creating and persisting the final configuration.
// Create new targets from configuration
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
.registry
.create_targets_from_config(&new_config)
@@ -492,46 +530,8 @@ impl NotificationSystem {
info!("{} notification targets were created from the new configuration", targets.len());
// Start new event stream processing for each storage enabled target
let mut new_cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
// Initialize the target
if let Err(e) = target.init().await {
error!("Target {} Initialization failed:{}", target_id, e);
continue;
}
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start new event stream processing for target {}", target_id);
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
// let target_box = target.clone_dyn();
let target_arc = Arc::from(target.clone_dyn());
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(
store_clone,
target_arc,
self.metrics.clone(),
self.concurrency_limiter.clone(),
);
// Start event stream processing and save cancel sender
// let cancel_tx = start_event_stream(store_clone, target_clone);
let target_id_clone = target_id.clone();
new_cancellers.insert(target_id, cancel_tx);
info!("Event stream processing of target {} is restarted successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} disabled, event stream processing is skipped", target_id);
}
}
// Initialize targets and start event streams using shared helper
let new_cancellers = self.init_targets_and_start_streams(&targets).await;
// Update canceler collection
*cancellers = new_cancellers;
@@ -665,4 +665,18 @@ mod tests {
assert_eq!(batch.events.len(), 1);
assert_eq!(batch.events[0].s3.object.key, "one");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "webhook");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
assert_eq!(target_id.id, "analytics");
assert_eq!(target_id.name, "mqtt");
}
}
crates/notify/src/notifier.rs
+3 -3
View File
@@ -399,7 +399,7 @@ mod tests {
use rustfs_targets::{
TargetError,
store::{Key, Store},
target::EntityTarget,
target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
};
use serde::{Serialize, de::DeserializeOwned};
use std::sync::{
@@ -442,7 +442,7 @@ mod tests {
Ok(())
}
async fn send_from_store(&self, _key: Key) -> Result<(), TargetError> {
async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
Ok(())
}
@@ -450,7 +450,7 @@ mod tests {
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
None
}
crates/notify/src/registry.rs
+5 -93
View File
@@ -89,9 +89,6 @@ impl TargetRegistry {
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
// A collection of asynchronous tasks for concurrently executing target creation
let mut tasks = FuturesUnordered::new();
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
// Record the defaults for each segment so that the segment can eventually be rebuilt
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
// 1. Traverse all registered plants and process them by target type
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
@@ -105,9 +102,6 @@ impl TargetRegistry {
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
debug!(?default_cfg, "Get the default configuration");
// Save defaults for eventual write back
section_defaults.insert(section_name.clone(), default_cfg.clone());
// *** Optimization point 1: Get all legitimate fields of the current target type ***
let valid_fields = factory.get_valid_fields();
debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -215,110 +209,28 @@ impl TargetRegistry {
if enabled {
info!(instance_id = %id, "Target is enabled, ready to create a task");
// 5.3. Create asynchronous tasks for enabled instances
let target_type_clone = target_type.clone();
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(target_type_clone, tid, result, Arc::clone(&merged_config_arc))
(tid, result)
});
} else {
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
// Remove disabled target from final configuration
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
info!(instance_id = %id, "Skip disabled target");
}
}
}
// 6. Concurrently execute all creation tasks and collect results
let mut successful_targets = Vec::new();
let mut successful_configs = Vec::new();
while let Some((target_type, id, result, final_config)) = tasks.next().await {
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
info!(instance_id = %id, "Create target successfully");
successful_targets.push(target);
successful_configs.push((target_type, id, final_config));
}
Err(e) => {
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
}
}
}
// 7. Aggregate new configuration and write back to system configuration
if !successful_configs.is_empty() || !section_defaults.is_empty() {
info!(
"Prepare to update {} successfully created target configurations to the system configuration...",
successful_configs.len()
);
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
for (target_type, id, kvs) in successful_configs {
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
successes_by_section
.entry(section_name)
.or_default()
.insert(id.to_lowercase(), (*kvs).clone());
}
let mut new_config = config.clone();
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
let mut sections: HashSet<String> = HashSet::new();
sections.extend(section_defaults.keys().cloned());
sections.extend(successes_by_section.keys().cloned());
for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item
if let Some(default_kvs) = section_defaults.get(&section)
&& !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
}
// Add successful instance item
if let Some(instances) = successes_by_section.get(&section) {
for (id, kvs) in instances {
section_map.insert(id.clone(), kvs.clone());
}
}
// Empty breaks are removed and non-empty breaks are replaced entirely.
if section_map.is_empty() {
new_config.0.remove(&section);
} else {
new_config.0.insert(section, section_map);
}
}
if &new_config == config {
info!("Notification target configuration unchanged, skip persisting server config");
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
let store = match rustfs_ecstore::global::new_object_layer_fn() {
Some(s) => s,
None => {
warn!(
"Object store not available at notification init; skipping config persistence. \
{} target(s) active in memory.",
successful_targets.len()
);
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
};
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
Ok(_) => {
info!("The new configuration was saved to the system successfully.")
}
Err(e) => {
error!("Failed to save the new configuration: {}", e);
return Err(TargetError::SaveConfig(e.to_string()));
error!(instance_id = %id, error = %e, "Failed to create target");
}
}
}
crates/notify/src/stream.rs
+51 -66
View File
@@ -15,8 +15,8 @@
use crate::{Event, integration::NotificationMetrics};
use rustfs_targets::{
StoreError, Target, TargetError,
store::{Key, Store},
target::EntityTarget,
store::{Key, Store, ensure_store_entry_raw_readable},
target::QueuedPayload,
};
use rustfs_utils::get_env_usize;
use std::sync::Arc;
@@ -32,7 +32,7 @@ use tracing::{debug, error, info, warn};
/// - `target`: The target to send events to
/// - `cancel_rx`: Receiver to listen for cancellation signals
pub async fn stream_events(
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
target: &dyn Target<Event>,
mut cancel_rx: mpsc::Receiver<()>,
) {
@@ -119,7 +119,7 @@ pub async fn stream_events(
/// # Returns
/// A sender to signal cancellation of the event stream
pub fn start_event_stream(
mut store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
) -> mpsc::Sender<()> {
let (cancel_tx, cancel_rx) = mpsc::channel(1);
@@ -143,7 +143,7 @@ pub fn start_event_stream(
/// # Returns
/// A sender to signal cancellation of the event stream
pub fn start_event_stream_with_batching(
mut store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
@@ -170,7 +170,7 @@ pub fn start_event_stream_with_batching(
/// # Notes
/// This function processes events in batches to improve efficiency.
pub async fn stream_events_with_batching(
store: &mut (dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send),
store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
target: &dyn Target<Event>,
mut cancel_rx: mpsc::Receiver<()>,
metrics: Arc<NotificationMetrics>,
@@ -185,7 +185,6 @@ pub async fn stream_events_with_batching(
const MAX_RETRIES: usize = 5;
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
let mut batch: Vec<EntityTarget<Event>> = Vec::with_capacity(batch_size);
let mut batch_keys = Vec::with_capacity(batch_size);
let mut last_flush = Instant::now();
@@ -201,8 +200,8 @@ pub async fn stream_events_with_batching(
debug!("Found {} keys in store for target: {}", keys.len(), target.name());
if keys.is_empty() {
// If there is data in the batch and timeout, refresh the batch
if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
@@ -218,41 +217,31 @@ pub async fn stream_events_with_batching(
info!("Cancellation received during processing for target: {}", target.name());
// Processing collected batches before exiting
if !batch.is_empty() {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
if !batch_keys.is_empty() {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
}
return;
}
// Try to get events from storage
match store.get(&key) {
Ok(event) => {
// Add to batch
batch.push(event);
batch_keys.push(key);
metrics.increment_processing();
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore)
.await;
last_flush = Instant::now();
}
// Skip unreadable entries so a single corrupt file cannot stall the stream.
// ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to
// remove the corrupt entry before returning Err, so no cleanup is needed here.
match ensure_store_entry_raw_readable(&*store, &key) {
Ok(true) => {} // entry is readable, proceed
Ok(false) => continue, // entry not found (already removed), skip
Err(err) => {
warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err);
continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable
}
Err(e) => {
error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e);
// Consider deleting unreadable events to prevent infinite loops from trying to read
match store.del(&key) {
Ok(_) => {
info!("Deleted corrupted event {} from store", key.to_string());
}
Err(del_err) => {
error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err);
}
}
}
metrics.increment_failed();
}
batch_keys.push(key);
metrics.increment_processing();
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
}
@@ -273,7 +262,6 @@ pub async fn stream_events_with_batching(
/// # Notes
/// This function processes a batch of events, sending each event to the target with retry
async fn process_batch(
batch: &mut Vec<EntityTarget<Event>>,
batch_keys: &mut Vec<Key>,
target: &dyn Target<Event>,
max_retries: usize,
@@ -281,8 +269,8 @@ async fn process_batch(
metrics: &Arc<NotificationMetrics>,
semaphore: &Arc<Semaphore>,
) {
debug!("Processing batch of {} events for target: {}", batch.len(), target.name());
if batch.is_empty() {
debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name());
if batch_keys.is_empty() {
return;
}
@@ -296,44 +284,42 @@ async fn process_batch(
};
// Handle every event in the batch
for (_event, key) in batch.iter().zip(batch_keys.iter()) {
for key in batch_keys.iter() {
let mut retry_count = 0;
let mut success = false;
// Retry logic
while retry_count < max_retries && !success {
// After sending successfully, the event in the storage is deleted synchronously.
match target.send_from_store(key.clone()).await {
Ok(_) => {
info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
success = true;
metrics.increment_processed();
}
Err(e) => {
// Different retry strategies are adopted according to the error type
match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await;
}
_ => {
// Permanent error, skip this event
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
}
Err(e) => match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
let backoff = 1u32 << retry_count as u32;
tokio::time::sleep(base_delay * backoff + jitter).await;
}
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
let backoff = 1u32 << retry_count as u32;
tokio::time::sleep(base_delay * backoff + jitter).await;
}
_ => {
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
}
},
}
}
// Handle the situation where the maximum number of retry exhaustion is exhausted
if retry_count >= max_retries && !success {
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
metrics.increment_failed();
@@ -341,7 +327,6 @@ async fn process_batch(
}
// Clear processed batches
batch.clear();
batch_keys.clear();
// Release semaphore permission (via drop)
crates/targets/Cargo.toml
+7
View File
@@ -28,5 +28,12 @@ url = { workspace = true }
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
[dev-dependencies]
criterion = { workspace = true }
[[bench]]
name = "queue_store_benchmark"
harness = false
[lints]
workspace = true
crates/targets/benches/queue_store_benchmark.rs
+94
View File
@@ -0,0 +1,94 @@
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_targets::store::{QueueStore, Store};
use serde::{Deserialize, Serialize};
use std::hint::black_box;
use std::sync::Arc;
use uuid::Uuid;
#[derive(Clone, Serialize, Deserialize)]
struct BenchEvent {
bucket: String,
key: String,
metadata: Vec<(String, String)>,
payload: String,
}
fn bench_dir(prefix: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("rustfs-targets-bench-{prefix}-{}", Uuid::new_v4()))
}
fn build_payload(payload_len: usize) -> Vec<u8> {
let event = BenchEvent {
bucket: "bench-bucket".to_string(),
key: format!("objects/{payload_len}/file.json"),
metadata: (0..8)
.map(|idx| (format!("x-amz-meta-{idx}"), "benchmark-value".repeat(4)))
.collect(),
payload: "abcdefghijklmnopqrstuvwxyz0123456789".repeat(payload_len / 36 + 1)[..payload_len].to_string(),
};
serde_json::to_vec(&event).unwrap()
}
fn queue_store_write_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("queue_store_put_raw");
for payload_size in [512usize, 8 * 1024, 64 * 1024] {
let payload = build_payload(payload_size);
group.throughput(Throughput::Bytes(payload.len() as u64));
for compress in [false, true] {
let dir = bench_dir(if compress { "put-compress" } else { "put-plain" });
let store = QueueStore::<BenchEvent>::new_with_compression(&dir, 100_000, ".bench", compress);
store.open().unwrap();
group.bench_with_input(
BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size),
&payload,
|b, payload| {
b.iter(|| {
let key = store.put_raw(payload).unwrap();
store.del(&key).unwrap();
});
},
);
let _ = store.delete();
}
}
group.finish();
}
fn queue_store_read_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("queue_store_get_raw");
for payload_size in [512usize, 8 * 1024, 64 * 1024] {
let payload = build_payload(payload_size);
group.throughput(Throughput::Bytes(payload.len() as u64));
for compress in [false, true] {
let dir = bench_dir(if compress { "get-compress" } else { "get-plain" });
let store = Arc::new(QueueStore::<BenchEvent>::new_with_compression(&dir, 100_000, ".bench", compress));
store.open().unwrap();
let key = store.put_raw(&payload).unwrap();
group.bench_with_input(
BenchmarkId::new(if compress { "snap_on" } else { "snap_off" }, payload_size),
&(Arc::clone(&store), key),
|b, (store, key)| {
b.iter(|| {
let raw = store.get_raw(key).unwrap();
black_box(raw);
});
},
);
let _ = store.delete();
}
}
group.finish();
}
criterion_group!(benches, queue_store_write_benchmark, queue_store_read_benchmark);
criterion_main!(benches);
crates/targets/src/store.rs
+362 -110
View File
@@ -13,20 +13,34 @@
// limitations under the License.
use crate::error::StoreError;
use rustfs_config::DEFAULT_LIMIT;
use rustfs_config::notify::{COMPRESS_EXT, DEFAULT_EXT};
use rustfs_config::{DEFAULT_LIMIT, DEFAULT_TARGET_STORE_COMPRESS, ENV_TARGET_STORE_COMPRESS, EnableState};
use serde::{Serialize, de::DeserializeOwned};
use snap::raw::{Decoder, Encoder};
use std::sync::{Arc, RwLock};
use std::{
collections::HashMap,
marker::PhantomData,
path::PathBuf,
sync::{
Arc, RwLock,
atomic::{AtomicU64, Ordering},
},
time::{SystemTime, UNIX_EPOCH},
};
use tracing::{debug, warn};
use uuid::Uuid;
fn resolve_queue_store_compression_from_env_value(value: Option<&str>) -> bool {
value
.and_then(|value| value.parse::<EnableState>().ok().map(|state| state.is_enabled()))
.unwrap_or(DEFAULT_TARGET_STORE_COMPRESS)
}
fn queue_store_compression_enabled() -> bool {
let value = std::env::var(ENV_TARGET_STORE_COMPRESS).ok();
resolve_queue_store_compression_from_env_value(value.as_deref())
}
/// Represents a key for an entry in the store
#[derive(Debug, Clone)]
pub struct Key {
@@ -63,21 +77,7 @@ impl Key {
impl std::fmt::Display for Key {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name_part = if self.item_count > 1 {
format!("{}:{}", self.item_count, self.name)
} else {
self.name.clone()
};
let mut file_name = name_part;
if !self.extension.is_empty() {
file_name.push_str(&self.extension);
}
if self.compress {
file_name.push_str(COMPRESS_EXT);
}
write!(f, "{file_name}")
f.write_str(&self.to_key_string())
}
}
@@ -123,6 +123,28 @@ pub fn parse_key(s: &str) -> Key {
}
}
pub fn ensure_store_entry_raw_readable<T>(
store: &(dyn Store<T, Error = StoreError, Key = Key> + Send),
key: &Key,
) -> Result<bool, StoreError>
where
T: Send + Sync + 'static + Clone + Serialize,
{
match store.get_raw(key) {
Ok(_) => Ok(true),
Err(StoreError::NotFound) => Ok(false),
Err(err) => {
match store.del(key) {
Ok(()) | Err(StoreError::NotFound) => {}
Err(del_err) => {
return Err(StoreError::Internal(format!("Failed to remove unreadable store entry {key}: {del_err}")));
}
}
Err(err)
}
}
}
/// Trait for a store that can store and retrieve items of type T
pub trait Store<T>: Send + Sync
where
@@ -142,15 +164,24 @@ where
/// Stores multiple items in a single batch
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error>;
/// Stores raw bytes in a single entry.
fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error>;
/// Retrieves a single item by key
fn get(&self, key: &Self::Key) -> Result<T, Self::Error>;
/// Retrieves multiple items by key
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error>;
/// Retrieves the raw bytes stored for a key.
fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error>;
/// Deletes an item by key
fn del(&self, key: &Self::Key) -> Result<(), Self::Error>;
/// Deletes the underlying store directory and clears all in-memory state.
fn delete(&self) -> Result<(), Self::Error>;
/// Lists all keys in the store
fn list(&self) -> Vec<Self::Key>;
@@ -169,7 +200,10 @@ pub struct QueueStore<T> {
entry_limit: u64,
directory: PathBuf,
file_ext: String,
compress: bool,
entries: Arc<RwLock<HashMap<String, i64>>>, // key -> modtime as unix nano
pending_entries: Arc<AtomicU64>,
fs_guard: Arc<RwLock<()>>,
_phantom: PhantomData<T>,
}
@@ -179,35 +213,70 @@ impl<T> Clone for QueueStore<T> {
entry_limit: self.entry_limit,
directory: self.directory.clone(),
file_ext: self.file_ext.clone(),
compress: self.compress,
entries: Arc::clone(&self.entries),
pending_entries: Arc::clone(&self.pending_entries),
fs_guard: Arc::clone(&self.fs_guard),
_phantom: PhantomData,
}
}
}
struct EntryReservation<'a> {
pending_entries: &'a AtomicU64,
}
impl Drop for EntryReservation<'_> {
fn drop(&mut self) {
self.pending_entries.fetch_sub(1, Ordering::SeqCst);
}
}
impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
/// Creates a new QueueStore
pub fn new(directory: impl Into<PathBuf>, limit: u64, ext: &str) -> Self {
Self::new_with_compression(directory, limit, ext, queue_store_compression_enabled())
}
/// Creates a new QueueStore with an explicit compression setting.
pub fn new_with_compression(directory: impl Into<PathBuf>, limit: u64, ext: &str, compress: bool) -> Self {
let file_ext = if ext.is_empty() { DEFAULT_EXT } else { ext };
let entry_limit = if limit == 0 { DEFAULT_LIMIT } else { limit };
QueueStore {
directory: directory.into(),
entry_limit: if limit == 0 { DEFAULT_LIMIT } else { limit },
entry_limit,
file_ext: file_ext.to_string(),
entries: Arc::new(RwLock::new(HashMap::with_capacity(limit as usize))),
compress,
entries: Arc::new(RwLock::new(HashMap::with_capacity(entry_limit as usize))),
pending_entries: Arc::new(AtomicU64::new(0)),
fs_guard: Arc::new(RwLock::new(())),
_phantom: PhantomData,
}
}
/// Returns the full path for a key
fn file_path(&self, key: &Key) -> PathBuf {
self.directory.join(key.to_string())
self.directory.join(key.to_key_string())
}
fn build_key(&self, item_count: usize) -> Key {
Key {
name: Uuid::new_v4().to_string(),
extension: self.file_ext.clone(),
item_count,
compress: self.compress,
}
}
/// Reads a file for the given key
fn read_file(&self, key: &Key) -> Result<Vec<u8>, StoreError> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
debug!("Reading file for key: {},path: {}", key.to_string(), path.display());
debug!("Reading file for key: {},path: {}", key, path.display());
let data = std::fs::read(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
StoreError::NotFound
@@ -220,41 +289,89 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> QueueStore<T> {
return Err(StoreError::NotFound);
}
if key.compress {
let mut decoder = Decoder::new();
decoder
.decompress_vec(&data)
.map_err(|e| StoreError::Compression(e.to_string()))
} else {
Ok(data)
if !key.compress {
return Ok(data);
}
let mut decoder = Decoder::new();
decoder
.decompress_vec(&data)
.map_err(|e| StoreError::Compression(e.to_string()))
}
fn reserve_entry_slot(&self) -> Result<EntryReservation<'_>, StoreError> {
loop {
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
let entries_len = entries.len() as u64;
let pending = self.pending_entries.load(Ordering::SeqCst);
if entries_len + pending >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
if self
.pending_entries
.compare_exchange(pending, pending + 1, Ordering::SeqCst, Ordering::SeqCst)
.is_ok()
{
return Ok(EntryReservation {
pending_entries: self.pending_entries.as_ref(),
});
}
}
}
/// Writes data to a file for the given key
fn write_file(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
/// Writes data to a file for the given key.
fn write_file(&self, key: &Key, data: &[u8]) -> Result<i64, StoreError> {
let path = self.file_path(key);
// Create directory if it doesn't exist
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(StoreError::Io)?;
}
let data = if key.compress {
if key.compress {
let mut encoder = Encoder::new();
encoder
let compressed = encoder
.compress_vec(data)
.map_err(|e| StoreError::Compression(e.to_string()))?
.map_err(|e| StoreError::Compression(e.to_string()))?;
std::fs::write(&path, &compressed).map_err(StoreError::Io)?;
} else {
data.to_vec()
};
std::fs::write(&path, &data).map_err(StoreError::Io)?;
std::fs::write(&path, data).map_err(StoreError::Io)?;
}
let modified = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as i64;
debug!("Wrote event to store: {}", key);
Ok(modified)
}
fn insert_entry(&self, key: &Key, modified: i64) -> Result<(), StoreError> {
let mut entries = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
entries.insert(key.to_string(), modified);
debug!("Wrote event to store: {}", key.to_string());
entries.insert(key.to_key_string(), modified);
Ok(())
}
fn remove_file_if_present(&self, key: &Key) -> Result<(), StoreError> {
let path = self.file_path(key);
match std::fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(StoreError::Io(err)),
}
}
fn write_and_index(&self, key: &Key, data: &[u8]) -> Result<(), StoreError> {
let modified = self.write_file(key, data)?;
if let Err(err) = self.insert_entry(key, modified) {
self.remove_file_if_present(key).map_err(|cleanup_err| {
StoreError::Internal(format!("Failed to index store entry {key}: {err}; cleanup failed: {cleanup_err}"))
})?;
return Err(err);
}
Ok(())
}
}
@@ -267,15 +384,20 @@ where
type Key = Key;
fn open(&self) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
std::fs::create_dir_all(&self.directory).map_err(StoreError::Io)?;
let entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
// Get the write lock to update the internal state
let dir_entries = std::fs::read_dir(&self.directory).map_err(StoreError::Io)?;
let mut entries_map = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
for entry in entries {
self.pending_entries.store(0, Ordering::SeqCst);
entries_map.clear();
for entry in dir_entries {
let entry = entry.map_err(StoreError::Io)?;
let metadata = entry.metadata().map_err(StoreError::Io)?;
if metadata.is_file() {
@@ -292,71 +414,47 @@ where
}
fn put(&self, item: Arc<T>) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: 1,
compress: true,
};
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(1);
let data = serde_json::to_vec(&*item).map_err(|e| StoreError::Serialization(e.to_string()))?;
self.write_file(&key, &data)?;
self.write_and_index(&key, &data)?;
Ok(key)
}
fn put_multiple(&self, items: Vec<T>) -> Result<Self::Key, Self::Error> {
// Check storage limits
{
let entries = self
.entries
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on entries".to_string()))?;
if entries.len() as u64 >= self.entry_limit {
return Err(StoreError::LimitExceeded);
}
}
if items.is_empty() {
// Or return an error, or a special key?
return Err(StoreError::Internal("Cannot put_multiple with empty items list".to_string()));
}
let uuid = Uuid::new_v4();
let key = Key {
name: uuid.to_string(),
extension: self.file_ext.clone(),
item_count: items.len(),
compress: true,
};
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(items.len());
// Serialize all items into a single Vec<u8>
// This current approach for get_multiple/put_multiple assumes items are concatenated JSON objects.
// This might be problematic for deserialization if not handled carefully.
// A better approach for multiple items might be to store them as a JSON array `Vec<T>`.
// For now, sticking to current logic of concatenating.
let mut buffer = Vec::new();
for item in items {
// If items are Vec<Event>, and Event is large, this could be inefficient.
// The current get_multiple deserializes one by one.
let item_data = serde_json::to_vec(&item).map_err(|e| StoreError::Serialization(e.to_string()))?;
buffer.extend_from_slice(&item_data);
// If using JSON array: buffer = serde_json::to_vec(&items)?
serde_json::to_writer(&mut buffer, &item).map_err(|e| StoreError::Serialization(e.to_string()))?;
}
self.write_file(&key, &buffer)?;
self.write_and_index(&key, &buffer)?;
Ok(key)
}
fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let _reservation = self.reserve_entry_slot()?;
let key = self.build_key(1);
self.write_and_index(&key, data)?;
Ok(key)
}
@@ -373,8 +471,8 @@ where
}
fn get_multiple(&self, key: &Self::Key) -> Result<Vec<T>, Self::Error> {
debug!("Reading items from store for key: {}", key.to_string());
let data = self.read_file(key)?;
debug!("Reading items from store for key: {}", key);
let data = self.get_raw(key)?;
if data.is_empty() {
return Err(StoreError::Deserialization("Cannot deserialize empty data".to_string()));
}
@@ -404,7 +502,7 @@ where
warn!(
"Expected {} items for key {}, but only found {}. Possible data corruption or incorrect item_count.",
key.item_count,
key.to_string(),
key,
items.len()
);
// Depending on strictness, this could be an error.
@@ -426,20 +524,24 @@ where
Ok(items)
}
fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
self.read_file(key)
}
fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.read()
.map_err(|_| StoreError::Internal("Failed to acquire read lock on store filesystem".to_string()))?;
let path = self.file_path(key);
std::fs::remove_file(&path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
// If file not found, still try to remove from entries map in case of inconsistency
warn!(
"File not found for key {} during del, but proceeding to remove from entries map.",
key.to_string()
);
StoreError::NotFound
} else {
StoreError::Io(e)
match std::fs::remove_file(&path) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// File already gone — still clean up the entries map to avoid stale keys.
warn!("File not found for key {} during del, cleaning up entries map.", key);
}
})?;
Err(e) => return Err(StoreError::Io(e)),
}
// Get the write lock to update the internal state
let mut entries = self
@@ -447,15 +549,32 @@ where
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
if entries.remove(&key.to_string()).is_none() {
// Key was not in the map, could be an inconsistency or already deleted.
// This is not necessarily an error if the file deletion succeeded or was NotFound.
if entries.remove(&key.to_key_string()).is_none() {
debug!("Key {} not found in entries map during del, might have been already removed.", key);
}
debug!("Deleted event from store: {}", key.to_string());
Ok(())
}
fn delete(&self) -> Result<(), Self::Error> {
let _fs_guard = self
.fs_guard
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on store filesystem".to_string()))?;
let mut entries = self
.entries
.write()
.map_err(|_| StoreError::Internal("Failed to acquire write lock on entries".to_string()))?;
entries.clear();
self.pending_entries.store(0, Ordering::SeqCst);
match std::fs::remove_dir_all(&self.directory) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(StoreError::Io(err)),
}
}
fn list(&self) -> Vec<Self::Key> {
// Get the read lock to read the internal state
let entries = match self.entries.read() {
@@ -492,3 +611,136 @@ where
Box::new(self.clone()) as Box<dyn Store<T, Error = Self::Error, Key = Self::Key> + Send + Sync>
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{
sync::{Arc, Barrier},
thread,
};
fn temp_store_dir(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("rustfs-targets-{name}-{}", Uuid::new_v4()))
}
#[test]
fn resolve_queue_store_compression_defaults_to_true() {
assert!(resolve_queue_store_compression_from_env_value(None));
}
#[test]
fn resolve_queue_store_compression_respects_disabled_env_value() {
assert!(!resolve_queue_store_compression_from_env_value(Some("off")));
assert!(!resolve_queue_store_compression_from_env_value(Some("false")));
}
#[test]
fn put_uses_store_compression_setting_in_key() {
let dir = temp_store_dir("put-key");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
store.open().unwrap();
let key = store.put(Arc::new("payload".to_string())).unwrap();
assert!(!key.compress);
assert!(store.file_path(&key).exists());
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn parse_key_round_trips_batch_and_compression_suffixes() {
let key = Key {
name: "event-id".to_string(),
extension: ".json".to_string(),
item_count: 3,
compress: true,
};
let parsed = parse_key(&key.to_key_string());
assert_eq!(parsed.name, key.name);
assert_eq!(parsed.extension, key.extension);
assert_eq!(parsed.item_count, key.item_count);
assert_eq!(parsed.compress, key.compress);
}
#[test]
fn put_raw_and_get_raw_round_trip_bytes() {
let dir = temp_store_dir("raw-roundtrip");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", true);
store.open().unwrap();
let payload = br#"{"kind":"notify","bucket":"demo","key":"alpha.txt"}"#;
let key = store.put_raw(payload).unwrap();
let raw = store.get_raw(&key).unwrap();
assert_eq!(raw, payload);
let _ = store.delete();
}
#[test]
fn delete_removes_directory_and_clears_entries() {
let dir = temp_store_dir("delete-store");
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
store.open().unwrap();
let _ = store.put(Arc::new("payload".to_string())).unwrap();
store.delete().unwrap();
assert!(store.list().is_empty());
assert!(!dir.exists());
}
#[test]
fn put_enforces_entry_limit() {
let dir = temp_store_dir("limit");
let store = QueueStore::<String>::new_with_compression(&dir, 1, ".test", false);
store.open().unwrap();
let _ = store.put(Arc::new("first".to_string())).unwrap();
let err = store.put(Arc::new("second".to_string())).unwrap_err();
assert!(matches!(err, StoreError::LimitExceeded));
let _ = store.delete();
}
#[test]
fn concurrent_put_raw_respects_entry_limit() {
let dir = temp_store_dir("concurrent-limit");
let store = Arc::new(QueueStore::<String>::new_with_compression(&dir, 1, ".test", true));
store.open().unwrap();
let start = Arc::new(Barrier::new(4));
let mut handles = Vec::new();
for idx in 0..4 {
let store = Arc::clone(&store);
let start = Arc::clone(&start);
handles.push(thread::spawn(move || {
let payload = vec![b'x'; 32 * 1024 + idx];
start.wait();
store.put_raw(&payload)
}));
}
let mut successes = 0;
let mut limit_errors = 0;
for handle in handles {
match handle.join().unwrap() {
Ok(_) => successes += 1,
Err(StoreError::LimitExceeded) => limit_errors += 1,
Err(err) => panic!("unexpected error: {err}"),
}
}
assert_eq!(successes, 1);
assert_eq!(limit_errors, 3);
assert_eq!(store.len(), 1);
let _ = store.delete();
}
}
crates/targets/src/target/mod.rs
+176 -3
View File
@@ -21,6 +21,8 @@ use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::fmt::Formatter;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::warn;
pub mod mqtt;
pub mod webhook;
@@ -45,14 +47,43 @@ where
/// Saves an event (either sends it immediately or stores it for later)
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
/// Sends an event from the store
async fn send_from_store(&self, key: Key) -> Result<(), TargetError>;
/// Sends an event from the store using the queued raw body and metadata.
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError>;
/// Sends an event from the store.
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
let store = self
.store()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
let raw = match store.get_raw(&key) {
Ok(raw) => raw,
Err(StoreError::NotFound) => return Ok(()),
Err(err) => return Err(TargetError::Storage(format!("Failed to read queued payload from store: {err}"))),
};
let queued = match QueuedPayload::decode(&raw) {
Ok(queued) => queued,
Err(err) => {
delete_stored_payload(store, &key).map_err(|delete_err| {
TargetError::Storage(format!(
"Failed to delete invalid queued payload {key} after decode error '{err}': {delete_err}"
))
})?;
warn!("Dropped invalid queued payload {key}: {err}");
return Ok(());
}
};
self.send_raw_from_store(key.clone(), queued.body, queued.meta).await?;
delete_stored_payload(store, &key)
}
/// Closes the target and releases resources
async fn close(&self) -> Result<(), TargetError>;
/// Returns the store associated with the target (if any)
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)>;
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)>;
/// Returns the type of the target
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync>;
@@ -78,6 +109,106 @@ where
pub data: E,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueuedPayloadMeta {
pub event_name: EventName,
pub bucket_name: String,
pub object_name: String,
pub content_type: String,
pub queued_at_unix_ms: u64,
pub payload_len: usize,
}
impl QueuedPayloadMeta {
pub fn new(
event_name: EventName,
bucket_name: String,
object_name: String,
content_type: impl Into<String>,
payload_len: usize,
) -> Self {
Self {
event_name,
bucket_name,
object_name,
content_type: content_type.into(),
queued_at_unix_ms: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis() as u64,
payload_len,
}
}
pub fn best_effort_preview(&self, body: &[u8], limit: usize) -> String {
if limit == 0 || body.is_empty() {
return String::new();
}
let slice = &body[..body.len().min(limit)];
match std::str::from_utf8(slice) {
Ok(text) => {
if body.len() > limit {
format!("{text}...")
} else {
text.to_string()
}
}
Err(_) => format!("<{} bytes binary>", body.len()),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QueuedPayload {
pub meta: QueuedPayloadMeta,
pub body: Vec<u8>,
}
impl QueuedPayload {
const MAGIC: [u8; 4] = *b"RQP1";
pub fn new(meta: QueuedPayloadMeta, body: Vec<u8>) -> Self {
Self { meta, body }
}
pub fn encode(&self) -> Result<Vec<u8>, TargetError> {
let meta = serde_json::to_vec(&self.meta)
.map_err(|err| TargetError::Serialization(format!("Failed to serialize queued payload metadata: {err}")))?;
let meta_len = u32::try_from(meta.len())
.map_err(|_| TargetError::Serialization("Queued payload metadata is too large".to_string()))?;
let mut out = Vec::with_capacity(Self::MAGIC.len() + 4 + meta.len() + self.body.len());
out.extend_from_slice(&Self::MAGIC);
out.extend_from_slice(&meta_len.to_le_bytes());
out.extend_from_slice(&meta);
out.extend_from_slice(&self.body);
Ok(out)
}
pub fn decode(raw: &[u8]) -> Result<Self, TargetError> {
if raw.len() < Self::MAGIC.len() + 4 {
return Err(TargetError::Serialization("Queued payload is too short".to_string()));
}
if raw[..Self::MAGIC.len()] != Self::MAGIC {
return Err(TargetError::Serialization("Queued payload magic mismatch".to_string()));
}
let mut meta_len_bytes = [0u8; 4];
meta_len_bytes.copy_from_slice(&raw[Self::MAGIC.len()..Self::MAGIC.len() + 4]);
let meta_len = u32::from_le_bytes(meta_len_bytes) as usize;
let meta_start = Self::MAGIC.len() + 4;
let meta_end = meta_start + meta_len;
if meta_end > raw.len() {
return Err(TargetError::Serialization("Queued payload metadata length exceeds input".to_string()));
}
let meta = serde_json::from_slice(&raw[meta_start..meta_end])
.map_err(|err| TargetError::Serialization(format!("Failed to deserialize queued payload metadata: {err}")))?;
let body = raw[meta_end..].to_vec();
Ok(Self { meta, body })
}
}
/// The `ChannelTargetType` enum represents the different types of channel Target
/// used in the notification system.
///
@@ -187,3 +318,45 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
.map(|s| s.into_owned())
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}")))
}
pub(crate) fn delete_stored_payload(
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
key: &Key,
) -> Result<(), TargetError> {
match store.del(key) {
Ok(()) | Err(StoreError::NotFound) => Ok(()),
Err(err) => Err(TargetError::Storage(format!("Failed to delete event from store: {err}"))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn queued_payload_round_trips_meta_and_body() {
let meta = QueuedPayloadMeta::new(
EventName::ObjectCreatedPut,
"bucket-a".to_string(),
"folder/object.txt".to_string(),
"application/json",
12,
);
let payload = QueuedPayload::new(meta.clone(), br#"{"ok":true}"#.to_vec());
let encoded = payload.encode().unwrap();
let decoded = QueuedPayload::decode(&encoded).unwrap();
assert_eq!(decoded.meta.event_name, meta.event_name);
assert_eq!(decoded.meta.bucket_name, meta.bucket_name);
assert_eq!(decoded.meta.object_name, meta.object_name);
assert_eq!(decoded.meta.content_type, meta.content_type);
assert_eq!(decoded.body, br#"{"ok":true}"#);
}
#[test]
fn queued_payload_decode_rejects_invalid_magic() {
let err = QueuedPayload::decode(b"bad-payload").unwrap_err();
assert!(err.to_string().contains("magic") || err.to_string().contains("short"));
}
}
crates/targets/src/target/mqtt.rs
+54 -72
View File
@@ -17,7 +17,7 @@ use crate::{
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{ChannelTargetType, EntityTarget, TargetType},
target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
};
use async_trait::async_trait;
use rumqttc::{AsyncClient, ConnectionError, EventLoop, MqttOptions, Outgoing, Packet, QoS, mqttbytes::Error as MqttBytesError};
@@ -25,6 +25,7 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use std::sync::Arc;
use std::{
marker::PhantomData,
path::PathBuf,
sync::atomic::{AtomicBool, Ordering},
time::Duration,
@@ -110,9 +111,10 @@ where
id: TargetID,
args: MQTTArgs,
client: Arc<Mutex<Option<AsyncClient>>>,
store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
connected: Arc<AtomicBool>,
bg_task_manager: Arc<BgTaskManager>,
_phantom: PhantomData<E>,
}
impl<E> MQTTTarget<E>
@@ -135,7 +137,7 @@ where
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<EntityTarget<E>>::new(specific_queue_path, args.queue_limit, extension);
let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
if let Err(e) = store.open() {
error!(
target_id = %target_id,
@@ -144,7 +146,7 @@ where
);
return Err(TargetError::Storage(format!("{e}")));
}
Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
@@ -157,13 +159,14 @@ where
});
info!(target_id = %target_id, "MQTT target created");
Ok(MQTTTarget {
Ok(MQTTTarget::<E> {
id: target_id,
args,
client: Arc::new(Mutex::new(None)),
store: queue_store,
connected: Arc::new(AtomicBool::new(false)),
bg_task_manager,
_phantom: PhantomData,
})
}
@@ -251,14 +254,7 @@ where
}
}
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
let client_guard = self.client.lock().await;
let client = client_guard
.as_ref()
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
// Decode form-urlencoded object name
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
@@ -269,14 +265,35 @@ where
records: vec![event.clone()],
};
let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
debug!("Sending event to mqtt target: {}, event log: {}", self.id, data_string);
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
let client_guard = self.client.lock().await;
let client = client_guard
.as_ref()
.ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
debug!(
target = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
preview = %meta.best_effort_preview(&body, 256),
"Sending MQTT payload"
);
client
.publish(&self.args.topic, self.args.qos, false, data)
.publish(&self.args.topic, self.args.qos, false, body)
.await
.map_err(|e| {
if e.to_string().contains("Connection") || e.to_string().contains("Timeout") {
@@ -293,13 +310,14 @@ where
}
pub fn clone_target(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(MQTTTarget {
Box::new(MQTTTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
client: self.client.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
connected: self.connected.clone(),
bg_task_manager: self.bg_task_manager.clone(),
_phantom: PhantomData,
})
}
}
@@ -494,11 +512,15 @@ where
#[instrument(skip(self, event), fields(target_id = %self.id))]
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = self.build_queued_payload(&event)?;
if let Some(store) = &self.store {
debug!(target_id = %self.id, "Event saved to store start");
// If store is configured, ONLY put the event into the store.
// Do NOT send it directly here.
match store.put(event.clone()) {
match store.put_raw(
&queued
.encode()
.map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
) {
Ok(_) => {
debug!(target_id = %self.id, "Event saved to store for MQTT target successfully.");
Ok(())
@@ -516,7 +538,7 @@ where
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Attempting to send directly but not connected; trying to init.");
// Call the struct's init method, not the trait's default
match MQTTTarget::init(self).await {
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -528,13 +550,13 @@ where
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
self.send_body(queued.body, &queued.meta).await
}
}
#[instrument(skip(self), fields(target_id = %self.id))]
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send event from store with key.");
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(target_id = %self.id, ?key, "Attempting to send queued payload from store.");
if !self.is_enabled() {
return Err(TargetError::Disabled);
@@ -542,7 +564,7 @@ where
if !self.connected.load(Ordering::SeqCst) {
warn!(target_id = %self.id, "Not connected; trying to init before sending from store.");
match MQTTTarget::init(self).await {
match MQTTTarget::<E>::init(self).await {
Ok(_) => debug!(target_id = %self.id, "MQTT target initialized successfully."),
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to initialize MQTT target.");
@@ -555,33 +577,8 @@ where
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
let event = match store.get(&key) {
Ok(event) => {
debug!(target_id = %self.id, ?key, "Retrieved event from store for sending.");
event
}
Err(StoreError::NotFound) => {
// Assuming NotFound takes the key
debug!(target_id = %self.id, ?key, "Event not found in store for sending.");
return Ok(());
}
Err(e) => {
error!(
target_id = %self.id,
error = %e,
"Failed to get event from store"
);
return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
}
};
debug!(target_id = %self.id, ?key, "Sending event from store.");
if let Err(e) = self.send(&event).await {
if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Failed to send event from store: Not connected. Event remains in store.");
return Err(TargetError::NotConnected);
@@ -589,22 +586,7 @@ where
error!(target_id = %self.id, error = %e, "Failed to send event from store with an unexpected error.");
return Err(e);
}
debug!(target_id = %self.id, ?key, "Event sent from store successfully. deleting from store. ");
match store.del(&key) {
Ok(_) => {
debug!(target_id = %self.id, ?key, "Event deleted from store after successful send.")
}
Err(StoreError::NotFound) => {
debug!(target_id = %self.id, ?key, "Event already deleted from store.");
}
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to delete event from store after send.");
return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
}
}
debug!(target_id = %self.id, ?key, "Event deleted from store.");
debug!(target_id = %self.id, ?key, "Event sent from store successfully.");
Ok(())
}
@@ -637,7 +619,7 @@ where
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
self.store.as_deref()
}
@@ -651,7 +633,7 @@ where
return Ok(());
}
// Call the internal init logic
MQTTTarget::init(self).await
MQTTTarget::<E>::init(self).await
}
fn is_enabled(&self) -> bool {
crates/targets/src/target/webhook.rs
+97 -102
View File
@@ -17,7 +17,7 @@ use crate::{
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{ChannelTargetType, EntityTarget, TargetType},
target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetType},
};
use async_trait::async_trait;
use reqwest::{Client, StatusCode, Url};
@@ -26,6 +26,7 @@ use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{
marker::PhantomData,
path::PathBuf,
sync::{
Arc,
@@ -33,7 +34,6 @@ use std::{
},
time::Duration,
};
use tokio::net::lookup_host;
use tokio::sync::mpsc;
use tracing::{debug, error, info, instrument, warn};
@@ -105,10 +105,10 @@ where
args: WebhookArgs,
http_client: Arc<Client>,
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
addr: String,
cancel_sender: mpsc::Sender<()>,
_phantom: PhantomData<E>,
}
impl<E> WebhookTarget<E>
@@ -117,14 +117,14 @@ where
{
/// Clones the WebhookTarget, creating a new instance with the same configuration
pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(WebhookTarget {
Box::new(WebhookTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
http_client: Arc::clone(&self.http_client),
store: self.store.as_ref().map(|s| s.boxed_clone()),
initialized: AtomicBool::new(self.initialized.load(Ordering::SeqCst)),
addr: self.addr.clone(),
cancel_sender: self.cancel_sender.clone(),
_phantom: PhantomData,
})
}
@@ -149,7 +149,7 @@ where
TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<EntityTarget<E>>::new(queue_dir, args.queue_limit, extension);
let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
if let Err(e) = store.open() {
error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
@@ -157,32 +157,22 @@ where
}
// Make sure that the Store trait implemented by QueueStore matches the expected error type
Some(Box::new(store) as Box<dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync>)
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
// resolved address
let addr = {
let host = args.endpoint.host_str().unwrap_or("localhost");
let port = args
.endpoint
.port()
.unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 });
format!("{host}:{port}")
};
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
Ok(WebhookTarget {
Ok(WebhookTarget::<E> {
id: target_id,
args,
http_client,
store: queue_store,
initialized: AtomicBool::new(false),
addr,
cancel_sender,
_phantom: PhantomData,
})
}
@@ -226,53 +216,80 @@ where
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {e}")))
}
async fn init(&self) -> Result<(), TargetError> {
// Use CAS operations to ensure thread-safe initialization
if !self.initialized.load(Ordering::SeqCst) {
// Check the connection
match self.is_active().await {
Ok(true) => {
info!("Webhook target {} is active", self.id);
}
Ok(false) => {
return Err(TargetError::NotConnected);
}
Err(e) => {
error!("Failed to check if Webhook target {} is active: {}", self.id, e);
return Err(e);
async fn init_inner(&self) -> Result<(), TargetError> {
if self.initialized.load(Ordering::SeqCst) {
return Ok(());
}
// HTTP HEAD probe: verifies the full request path (proxy, TLS, firewall)
// unlike TCP connect which can't detect proxy issues.
let probe_timeout = Duration::from_secs(5);
match tokio::time::timeout(probe_timeout, self.http_client.head(self.args.endpoint.as_str()).send()).await {
Ok(Ok(resp)) => {
let status = resp.status();
if status.is_success() || status == StatusCode::NOT_FOUND {
// NOT_FOUND is acceptable for HEAD probes — the endpoint may not
// exist as a HEAD route, but the server is reachable.
debug!("Webhook target {} HEAD probe returned {}", self.id, status);
} else if status == StatusCode::METHOD_NOT_ALLOWED {
// Server is reachable but doesn't support HEAD — still valid.
debug!("Webhook target {} HEAD probe: METHOD_NOT_ALLOWED (reachable)", self.id);
} else {
warn!("Webhook target {} HEAD probe returned {}", self.id, status);
}
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
Ok(Err(e)) => {
// Connection-level error (DNS, TLS, refused, timeout)
return Err(if e.is_timeout() || e.is_connect() {
TargetError::NotConnected
} else {
TargetError::Network(format!("Webhook HEAD probe failed: {e}"))
});
}
Err(_) => {
return Err(TargetError::Timeout("Webhook HEAD probe timed out".to_string()));
}
}
self.initialized.store(true, Ordering::SeqCst);
info!("Webhook target {} initialized", self.id);
Ok(())
}
async fn send(&self, event: &EntityTarget<E>) -> Result<(), TargetError> {
info!("Webhook Sending event to webhook target: {}", self.id);
// Decode form-urlencoded object name
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
let log = TargetLog {
event_name: event.event_name,
key,
records: vec![event.data.clone()],
};
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
let data = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
info!("Webhook sending queued payload to target: {}", self.id);
debug!(
target = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
preview = %meta.best_effort_preview(&body, 256),
"Sending webhook payload"
);
// Vec<u8> Convert to String
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {e}")))?;
debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string);
// build request
let mut req_builder = self
.http_client
.post(self.args.endpoint.as_str())
.header("Content-Type", "application/json");
.header("Content-Type", meta.content_type.as_str());
if !self.args.auth_token.is_empty() {
// Split auth_token string to check if the authentication type is included
@@ -293,7 +310,7 @@ where
}
// Send a request
let resp = req_builder.body(data).send().await.map_err(|e| {
let resp = req_builder.body(body).send().await.map_err(|e| {
if e.is_timeout() || e.is_connect() {
TargetError::NotConnected
} else {
@@ -329,34 +346,39 @@ where
}
async fn is_active(&self) -> Result<bool, TargetError> {
let socket_addr = lookup_host(&self.addr)
.await
.map_err(|e| TargetError::Network(format!("Failed to resolve host: {e}")))?
.next()
.ok_or_else(|| TargetError::Network("No address found".to_string()))?;
debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id);
match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
Ok(Ok(_)) => {
debug!("Connection to {} is active", self.addr);
Ok(true)
}
Ok(Err(e)) => {
debug!("Connection to {} failed: {}", self.addr, e);
if e.kind() == std::io::ErrorKind::ConnectionRefused {
Err(TargetError::NotConnected)
match tokio::time::timeout(Duration::from_secs(5), self.http_client.head(self.args.endpoint.as_str()).send()).await {
Ok(Ok(resp)) => {
let status = resp.status();
if status.is_server_error() {
debug!("Webhook {} server error: {}", self.id, status);
Ok(false)
} else {
Err(TargetError::Network(format!("Connection failed: {e}")))
debug!("Webhook {} is reachable (status: {})", self.id, status);
Ok(true)
}
}
Err(_) => Err(TargetError::Timeout("Connection timed out".to_string())),
Ok(Err(e)) => {
debug!("Webhook {} request failed: {}", self.id, e);
if e.is_timeout() || e.is_connect() {
Err(TargetError::NotConnected)
} else {
Err(TargetError::Network(format!("Webhook health check failed: {e}")))
}
}
Err(_) => Err(TargetError::Timeout("Webhook health check timed out".to_string())),
}
}
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = self.build_queued_payload(&event)?;
if let Some(store) = &self.store {
// Call the store method directly, no longer need to acquire the lock
store
.put(event)
.put_raw(
&queued
.encode()
.map_err(|e| TargetError::Storage(format!("Failed to encode queued payload: {e}")))?,
)
.map_err(|e| TargetError::Storage(format!("Failed to save event to store: {e}")))?;
debug!("Event saved to store for target: {}", self.id);
Ok(())
@@ -368,12 +390,12 @@ where
return Err(TargetError::NotConnected);
}
}
self.send(&event).await
self.send_body(queued.body, &queued.meta).await
}
}
async fn send_from_store(&self, key: Key) -> Result<(), TargetError> {
debug!("Sending event from store for target: {}", self.id);
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for target: {}, key: {}", self.id, key);
match self.init().await {
Ok(_) => {
debug!("Event sent to store for target: {}", self.name());
@@ -384,37 +406,13 @@ where
}
}
let store = self
.store
.as_ref()
.ok_or_else(|| TargetError::Configuration("No store configured".to_string()))?;
// Get events directly from the store, no longer need to acquire locks
let event = match store.get(&key) {
Ok(event) => event,
Err(StoreError::NotFound) => return Ok(()),
Err(e) => {
return Err(TargetError::Storage(format!("Failed to get event from store: {e}")));
}
};
if let Err(e) = self.send(&event).await {
if let Err(e) = self.send_body(body, &meta).await {
if let TargetError::NotConnected = e {
return Err(TargetError::NotConnected);
}
return Err(e);
}
// Use the immutable reference of the store to delete the event content corresponding to the key
debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string());
match store.del(&key) {
Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()),
Err(e) => {
error!("Failed to delete event from store: {}", e);
return Err(TargetError::Storage(format!("Failed to delete event from store: {e}")));
}
}
debug!("Event sent from store and deleted for target: {}", self.id);
Ok(())
}
@@ -426,7 +424,7 @@ where
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
// Returns the reference to the internal store
self.store.as_deref()
}
@@ -436,14 +434,11 @@ where
}
async fn init(&self) -> Result<(), TargetError> {
// If the target is disabled, return to success directly
if !self.is_enabled() {
debug!("Webhook target {} is disabled, skipping initialization", self.id);
return Ok(());
}
// Use existing initialization logic
WebhookTarget::init(self).await
self.init_inner().await
}
fn is_enabled(&self) -> bool {
rustfs/src/admin/handlers/audit.rs
+818
View File
@@ -0,0 +1,818 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::ADMIN_PREFIX;
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_targets::check_mqtt_broker_available;
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::io::{Error, ErrorKind};
use std::path::Path;
use std::sync::Arc;
use tokio::sync::Semaphore;
use tokio::time::{Duration, sleep, timeout};
use tracing::{Span, warn};
use url::Url;
pub fn register_audit_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/list").as_str(),
AdminOperation(&ListAuditTargets {}),
)?;
r.insert(
Method::PUT,
format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}").as_str(),
AdminOperation(&AuditTargetConfig {}),
)?;
r.insert(
Method::DELETE,
format!("{}{}", ADMIN_PREFIX, "/v3/audit/target/{target_type}/{target_name}/reset").as_str(),
AdminOperation(&RemoveAuditTarget {}),
)?;
Ok(())
}
#[derive(Debug, Deserialize)]
pub struct KeyValue {
pub key: String,
pub value: String,
}
#[derive(Debug, Deserialize)]
pub struct AuditTargetBody {
pub key_values: Vec<KeyValue>,
}
#[derive(Serialize, Debug)]
struct AuditEndpoint {
account_id: String,
service: String,
status: String,
source: AuditEndpointSource,
}
#[derive(Serialize, Debug)]
struct AuditEndpointsResponse {
audit_endpoints: Vec<AuditEndpoint>,
}
type EndpointKey = (String, String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
enum AuditEndpointSource {
Config,
Env,
Mixed,
Runtime,
}
fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
(account_id.to_lowercase(), service.to_string())
}
async fn check_permissions(req: &S3Request<Body>) -> S3Result<()> {
let Some(input_cred) = &req.credentials else {
return Err(s3_error!(InvalidRequest, "credentials not found"));
};
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
Ok(())
}
fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
if let Some(v) = request_id {
header.insert("x-request-id", v.clone());
}
S3Response::with_headers((status, body), header)
}
async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, Error>>,
{
let mut attempts = 0;
let mut delay = base_delay;
let mut last_err = None;
while attempts < max_attempts {
match operation().await {
Ok(result) => return Ok(result),
Err(e) => {
last_err = Some(e);
attempts += 1;
if attempts < max_attempts {
sleep(delay).await;
delay = delay.saturating_mul(2);
}
}
}
}
Err(last_err.unwrap_or_else(|| Error::other("retry_with_backoff: unknown error")))
}
async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
if !queue_dir.is_empty() {
if !Path::new(queue_dir).is_absolute() {
return Err(s3_error!(InvalidArgument, "queue_dir must be absolute path"));
}
retry_with_backoff(
|| async { tokio::fs::metadata(queue_dir).await.map(|_| ()) },
3,
Duration::from_millis(100),
)
.await
.map_err(|e| match e.kind() {
ErrorKind::NotFound => s3_error!(InvalidArgument, "queue_dir does not exist"),
ErrorKind::PermissionDenied => s3_error!(InvalidArgument, "queue_dir exists but permission denied"),
_ => s3_error!(InvalidArgument, "failed to access queue_dir: {}", e),
})?;
}
Ok(())
}
fn config_enable_is_on(value: &str) -> bool {
matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
}
fn has_any_audit_targets(config: &Config) -> bool {
for subsystem in [AUDIT_WEBHOOK_SUB_SYS, AUDIT_MQTT_SUB_SYS] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
if targets.keys().any(|key| key != DEFAULT_DELIMITER) {
return true;
}
}
false
}
fn collect_configured_audit_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
let mut endpoints = Vec::new();
for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
for (target_name, kvs) in targets {
if target_name == DEFAULT_DELIMITER {
continue;
}
let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
if enabled {
endpoints.push((target_name.clone(), service.to_string()));
}
}
}
endpoints
}
fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for (subsystem, service) in [(AUDIT_WEBHOOK_SUB_SYS, "webhook"), (AUDIT_MQTT_SUB_SYS, "mqtt")] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
for target_name in targets.keys() {
if target_name == DEFAULT_DELIMITER {
continue;
}
endpoints.insert(normalized_endpoint_key(target_name, service));
}
}
endpoints
}
fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for (service, valid_keys) in [("webhook", AUDIT_WEBHOOK_KEYS), ("mqtt", AUDIT_MQTT_KEYS)] {
let env_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase();
for (key, _value) in std::env::vars() {
let Some(rest) = key.strip_prefix(&env_prefix) else {
continue;
};
let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
let field_name_part = parts.next();
let (field_name, instance_id) = match field_name_part {
Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
};
if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() {
continue;
}
if valid_keys.contains(&field_name.as_str()) {
endpoints.insert(normalized_endpoint_key(&instance_id, service));
}
}
}
endpoints
}
fn classify_audit_endpoint_source(
config_targets: &HbHashSet<EndpointKey>,
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> AuditEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
(true, true) => AuditEndpointSource::Mixed,
(true, false) => AuditEndpointSource::Config,
(false, true) => AuditEndpointSource::Env,
(false, false) => AuditEndpointSource::Runtime,
}
}
fn audit_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> AuditEndpointSource {
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let service = match target_type {
AUDIT_WEBHOOK_SUB_SYS => "webhook",
AUDIT_MQTT_SUB_SYS => "mqtt",
_ => "",
};
let key = normalized_endpoint_key(target_name, service);
classify_audit_endpoint_source(&config_targets, &env_targets, &key)
}
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
match audit_endpoint_source(config, target_type, target_name) {
AuditEndpointSource::Env => Some(format!(
"audit target '{}' is managed by environment variables and cannot be modified from the console",
target_name
)),
AuditEndpointSource::Mixed => Some(format!(
"audit target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
target_name
)),
AuditEndpointSource::Config | AuditEndpointSource::Runtime => None,
}
}
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
let mut audit_endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_audit_endpoint_keys(config);
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for key in configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
}
let status = normalized_runtime_statuses
.remove(&normalized)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
let source = classify_audit_endpoint_source(&config_targets, &env_targets, &normalized);
audit_endpoints.push(AuditEndpoint {
account_id: key.0,
service: key.1,
status,
source,
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if seen.insert(normalized.clone()) {
audit_endpoints.push(AuditEndpoint {
account_id,
service,
status,
source: classify_audit_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
}
for key in &env_targets {
if !seen.insert(key.clone()) {
continue;
}
audit_endpoints.push(AuditEndpoint {
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_audit_endpoint_source(&config_targets, &env_targets, key),
});
}
audit_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
audit_endpoints
}
fn collect_validated_key_values(
key_values: &[KeyValue],
allowed_keys: &HashSet<&str>,
target_type: &str,
) -> S3Result<HashMap<String, String>> {
let mut kv_map = HashMap::new();
let mut seen = HashSet::new();
for kv in key_values {
if !allowed_keys.contains(kv.key.as_str()) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for audit target type '{}'",
kv.key,
target_type
));
}
if !seen.insert(kv.key.as_str()) {
return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
}
kv_map.insert(kv.key.clone(), kv.value.clone());
}
Ok(kv_map)
}
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
let target_type = params
.get("target_type")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
if target_type != AUDIT_WEBHOOK_SUB_SYS && target_type != AUDIT_MQTT_SUB_SYS {
return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
}
let target_name = params
.get("target_name")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
Ok((target_type, target_name))
}
async fn load_server_config_from_store() -> S3Result<Config> {
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Ok(Config::new());
};
rustfs_ecstore::config::com::read_config_without_migrate(store)
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
}
async fn apply_audit_runtime_config(config: Config) -> S3Result<()> {
let has_targets = has_any_audit_targets(&config);
if let Some(system) = audit_system() {
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
if has_targets {
system
.reload_config(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
} else {
system
.close()
.await
.map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
}
}
AuditSystemState::Stopped | AuditSystemState::Stopping => {
if has_targets {
system
.start(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
}
}
} else if has_targets {
start_global_audit_system(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
Ok(())
}
async fn update_audit_config_and_reload<F>(mut modifier: F) -> S3Result<()>
where
F: FnMut(&mut Config) -> bool,
{
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
if !modifier(&mut config) {
return Ok(());
}
rustfs_ecstore::config::com::save_server_config(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
apply_audit_runtime_config(config).await
}
pub struct AuditTargetConfig {}
#[async_trait::async_trait]
impl Operation for AuditTargetConfig {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let span = Span::current();
let _enter = span.enter();
let (target_type, target_name) = extract_target_params(&params)?;
check_permissions(&req).await?;
let config_snapshot = load_server_config_from_store().await?;
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let mut input = req.input;
let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| {
warn!("failed to read request body: {:?}", e);
s3_error!(InvalidRequest, "failed to read request body")
})?;
let audit_body: AuditTargetBody = serde_json::from_slice(&body_bytes)
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
let allowed_keys: HashSet<&str> = match target_type {
AUDIT_WEBHOOK_SUB_SYS => AUDIT_WEBHOOK_KEYS.iter().cloned().collect(),
AUDIT_MQTT_SUB_SYS => AUDIT_MQTT_KEYS.iter().cloned().collect(),
_ => unreachable!(),
};
let kv_map = collect_validated_key_values(&audit_body.key_values, &allowed_keys, target_type)?;
if target_type == AUDIT_WEBHOOK_SUB_SYS {
let endpoint = kv_map
.get("endpoint")
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
match parsed_endpoint.scheme() {
"http" | "https" => {}
other => {
return Err(s3_error!(
InvalidArgument,
"unsupported endpoint scheme: {} (only http and https are allowed)",
other
));
}
}
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir.as_str()).await?;
}
if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") {
return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair"));
}
} else if target_type == AUDIT_MQTT_SUB_SYS {
let endpoint = kv_map
.get(rustfs_config::MQTT_BROKER)
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?;
let topic = kv_map
.get(rustfs_config::MQTT_TOPIC)
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?;
let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str);
let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str);
check_mqtt_broker_available(endpoint, topic, username, password)
.await
.map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?;
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir.as_str()).await?;
if let Some(qos) = kv_map.get("qos") {
match qos.parse::<u8>() {
Ok(1) | Ok(2) => {}
Ok(0) => return Err(s3_error!(InvalidArgument, "qos should be 1 or 2 if queue_dir is set")),
_ => return Err(s3_error!(InvalidArgument, "qos must be an integer 0, 1, or 2")),
}
}
}
}
let mut kvs = rustfs_ecstore::config::KVS::new();
for (key, value) in kv_map {
kvs.insert(key, value);
}
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
update_audit_config_and_reload(|config| {
config
.0
.entry(target_type.to_lowercase())
.or_default()
.insert(target_name.to_lowercase(), kvs.clone());
true
})
.await?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
pub struct ListAuditTargets {}
#[async_trait::async_trait]
impl Operation for ListAuditTargets {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let span = Span::current();
let _enter = span.enter();
check_permissions(&req).await?;
let mut runtime_statuses = HashMap::new();
if let Some(system) = audit_system() {
let targets = system.get_target_values().await;
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
((target.id().id.clone(), target.id().name.to_string()), status.to_string())
});
}
while let Some((key, status)) = futures.next().await {
runtime_statuses.insert(key, status);
}
}
let config = load_server_config_from_store().await?;
let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints })
.map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?;
Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
}
}
pub struct RemoveAuditTarget {}
#[async_trait::async_trait]
impl Operation for RemoveAuditTarget {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let span = Span::current();
let _enter = span.enter();
let (target_type, target_name) = extract_target_params(&params)?;
check_permissions(&req).await?;
let config_snapshot = load_server_config_from_store().await?;
if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
update_audit_config_and_reload(|config| {
let mut changed = false;
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
if targets.remove(&target_name.to_lowercase()).is_some() {
changed = true;
}
if targets.is_empty() {
config.0.remove(&target_type.to_lowercase());
}
}
changed
})
.await?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_ecstore::config::{KV, KVS};
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars};
fn enabled_kvs(value: &str) -> KVS {
KVS(vec![KV {
key: ENABLE_KEY.to_string(),
value: value.to_string(),
hidden_if_empty: false,
}])
}
#[test]
fn merge_audit_endpoints_marks_config_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([
("mixed-target".to_string(), enabled_kvs("on")),
("config-target".to_string(), enabled_kvs("on")),
]),
)]));
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")),
("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let runtime = HashMap::from([
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-target")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, AuditEndpointSource::Env);
let config_only = merged
.iter()
.find(|entry| entry.account_id == "config-target")
.expect("config target should be present");
assert_eq!(config_only.source, AuditEndpointSource::Config);
},
);
}
#[test]
fn audit_target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")),
],
|| {
let config = Config(HashMap::new());
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
assert!(reason.is_some());
assert!(reason.unwrap().contains("managed by environment variables"));
},
);
}
#[test]
fn audit_target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
)]));
let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
assert!(reason.is_some());
assert!(reason.unwrap().contains("both persisted config and environment variables"));
});
}
#[test]
fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]),
)]));
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
],
|| {
let merged = merge_audit_endpoints(&config, HashMap::new());
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
assert_eq!(mixed.status, "offline");
},
);
}
#[test]
fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let merged = merge_audit_endpoints(&config, HashMap::new());
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, AuditEndpointSource::Env);
assert_eq!(env_only.status, "offline");
},
);
}
#[test]
fn collect_validated_key_values_rejects_duplicate_keys() {
let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
let key_values = vec![
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/one".to_string(),
},
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/two".to_string(),
},
];
let err = collect_validated_key_values(&key_values, &allowed_keys, AUDIT_WEBHOOK_SUB_SYS).unwrap_err();
assert!(err.to_string().contains("duplicate key"));
}
#[test]
fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")),
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
],
|| {
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_audit_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
},
);
}
#[test]
fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
)]));
assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
}
}
rustfs/src/admin/handlers/event.rs
+571 -61
View File
@@ -16,21 +16,23 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::ADMIN_PREFIX;
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::{ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_config::notify::{
NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_targets::check_mqtt_broker_available;
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::io::{Error, ErrorKind};
use std::net::SocketAddr;
use std::path::Path;
use std::sync::Arc;
use tokio::net::lookup_host;
use tokio::sync::Semaphore;
use tokio::time::{Duration, sleep, timeout};
use tracing::{Span, info, warn};
@@ -80,6 +82,7 @@ struct NotificationEndpoint {
account_id: String,
service: String,
status: String,
source: NotificationEndpointSource,
}
#[derive(Serialize, Debug)]
@@ -87,6 +90,21 @@ struct NotificationEndpointsResponse {
notification_endpoints: Vec<NotificationEndpoint>,
}
type EndpointKey = (String, String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
enum NotificationEndpointSource {
Config,
Env,
Mixed,
Runtime,
}
fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
(account_id.to_lowercase(), service.to_string())
}
// --- Helper Functions ---
async fn check_permissions(req: &S3Request<Body>) -> S3Result<()> {
@@ -155,6 +173,215 @@ async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
Ok(())
}
fn config_enable_is_on(value: &str) -> bool {
matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
}
fn collect_configured_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
let mut endpoints = Vec::new();
for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
for (target_name, kvs) in targets {
if target_name == DEFAULT_DELIMITER {
continue;
}
let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
if enabled {
endpoints.push((target_name.clone(), service.to_string()));
}
}
}
endpoints
}
fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for (subsystem, service) in [(NOTIFY_WEBHOOK_SUB_SYS, "webhook"), (NOTIFY_MQTT_SUB_SYS, "mqtt")] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
for target_name in targets.keys() {
if target_name == DEFAULT_DELIMITER {
continue;
}
endpoints.insert(normalized_endpoint_key(target_name, service));
}
}
endpoints
}
fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for (service, valid_keys) in [("webhook", NOTIFY_WEBHOOK_KEYS), ("mqtt", NOTIFY_MQTT_KEYS)] {
let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{service}{DEFAULT_DELIMITER}").to_uppercase();
for (key, _value) in std::env::vars() {
let Some(rest) = key.strip_prefix(&env_prefix) else {
continue;
};
let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
let field_name_part = parts.next();
let (field_name, instance_id) = match field_name_part {
Some(field) => (field.to_lowercase(), instance_id_part.to_lowercase()),
None => (instance_id_part.to_lowercase(), DEFAULT_DELIMITER.to_string()),
};
if instance_id == DEFAULT_DELIMITER || instance_id.is_empty() {
continue;
}
if valid_keys.contains(&field_name.as_str()) {
endpoints.insert(normalized_endpoint_key(&instance_id, service));
}
}
}
endpoints
}
fn classify_notification_endpoint_source(
config_targets: &HbHashSet<EndpointKey>,
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> NotificationEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
(true, true) => NotificationEndpointSource::Mixed,
(true, false) => NotificationEndpointSource::Config,
(false, true) => NotificationEndpointSource::Env,
(false, false) => NotificationEndpointSource::Runtime,
}
}
fn notification_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> NotificationEndpointSource {
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let service = match target_type {
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_MQTT_SUB_SYS => "mqtt",
_ => "",
};
let key = normalized_endpoint_key(target_name, service);
classify_notification_endpoint_source(&config_targets, &env_targets, &key)
}
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
match notification_endpoint_source(config, target_type, target_name) {
NotificationEndpointSource::Env => Some(format!(
"target '{}' is managed by environment variables and cannot be modified from the console",
target_name
)),
NotificationEndpointSource::Mixed => Some(format!(
"target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
target_name
)),
NotificationEndpointSource::Config | NotificationEndpointSource::Runtime => None,
}
}
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
let mut notification_endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_endpoint_keys(config);
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for key in configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
}
let status = normalized_runtime_statuses
.remove(&normalized)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
let source = classify_notification_endpoint_source(&config_targets, &env_targets, &normalized);
notification_endpoints.push(NotificationEndpoint {
account_id: key.0,
service: key.1,
status,
source,
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if seen.insert(normalized.clone()) {
notification_endpoints.push(NotificationEndpoint {
account_id,
service,
status,
source: classify_notification_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
}
for key in &env_targets {
if !seen.insert(key.clone()) {
continue;
}
notification_endpoints.push(NotificationEndpoint {
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_notification_endpoint_source(&config_targets, &env_targets, key),
});
}
notification_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
notification_endpoints
}
fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
target_statuses
.into_iter()
.filter_map(|(target_id, status)| (status == "online").then(|| target_id.to_arn(region).to_string()))
.collect()
}
fn collect_validated_key_values(
key_values: &[KeyValue],
allowed_keys: &HashSet<&str>,
target_type: &str,
) -> S3Result<HashMap<String, String>> {
let mut kv_map = HashMap::new();
let mut seen = HashSet::new();
for kv in key_values {
if !allowed_keys.contains(kv.key.as_str()) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for target type '{}'",
kv.key,
target_type
));
}
if !seen.insert(kv.key.as_str()) {
return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
}
kv_map.insert(kv.key.clone(), kv.value.clone());
}
Ok(kv_map)
}
// --- Operations ---
pub struct NotificationTarget {}
@@ -167,6 +394,10 @@ impl Operation for NotificationTarget {
check_permissions(&req).await?;
let ns = get_notification_system()?;
let config_snapshot = ns.config.read().await.clone();
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let mut input = req.input;
let body_bytes = input.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE).await.map_err(|e| {
@@ -183,37 +414,27 @@ impl Operation for NotificationTarget {
_ => unreachable!(),
};
let kv_map: HashMap<&str, &str> = notification_body
.key_values
.iter()
.map(|kv| (kv.key.as_str(), kv.value.as_str()))
.collect();
// Validate keys
for key in kv_map.keys() {
if !allowed_keys.contains(key) {
return Err(s3_error!(InvalidArgument, "key '{}' not allowed for target type '{}'", key, target_type));
}
}
let kv_map = collect_validated_key_values(&notification_body.key_values, &allowed_keys, target_type)?;
// Type-specific validation
if target_type == NOTIFY_WEBHOOK_SUB_SYS {
let endpoint = kv_map
.get("endpoint")
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint is required"))?;
let url = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
let host = url
.host_str()
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing host"))?;
let port = url
.port_or_known_default()
.ok_or_else(|| s3_error!(InvalidArgument, "endpoint missing port"))?;
let addr = format!("{host}:{port}");
if addr.parse::<SocketAddr>().is_err() && lookup_host(&addr).await.is_err() {
return Err(s3_error!(InvalidArgument, "invalid or unresolvable endpoint address"));
let parsed_endpoint = Url::parse(endpoint).map_err(|e| s3_error!(InvalidArgument, "invalid endpoint url: {}", e))?;
match parsed_endpoint.scheme() {
"http" | "https" => {}
other => {
return Err(s3_error!(
InvalidArgument,
"unsupported endpoint scheme: {} (only http and https are allowed)",
other
));
}
}
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir).await?;
validate_queue_dir(queue_dir.as_str()).await?;
}
if kv_map.contains_key("client_cert") != kv_map.contains_key("client_key") {
return Err(s3_error!(InvalidArgument, "client_cert and client_key must be specified as a pair"));
@@ -221,18 +442,20 @@ impl Operation for NotificationTarget {
} else if target_type == NOTIFY_MQTT_SUB_SYS {
let endpoint = kv_map
.get(rustfs_config::MQTT_BROKER)
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "broker endpoint is required"))?;
let topic = kv_map
.get(rustfs_config::MQTT_TOPIC)
.map(String::as_str)
.ok_or_else(|| s3_error!(InvalidArgument, "topic is required"))?;
let username = kv_map.get(rustfs_config::MQTT_USERNAME).copied();
let password = kv_map.get(rustfs_config::MQTT_PASSWORD).copied();
let username = kv_map.get(rustfs_config::MQTT_USERNAME).map(String::as_str);
let password = kv_map.get(rustfs_config::MQTT_PASSWORD).map(String::as_str);
check_mqtt_broker_available(endpoint, topic, username, password)
.await
.map_err(|e| s3_error!(InvalidArgument, "MQTT Broker unavailable: {}", e))?;
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir).await?;
validate_queue_dir(queue_dir.as_str()).await?;
if let Some(qos) = kv_map.get("qos") {
match qos.parse::<u8>() {
Ok(1) | Ok(2) => {}
@@ -243,24 +466,14 @@ impl Operation for NotificationTarget {
}
}
let mut kvs_vec: Vec<_> = notification_body
.key_values
.into_iter()
.map(|kv| rustfs_ecstore::config::KV {
key: kv.key,
value: kv.value,
hidden_if_empty: false,
})
.collect();
kvs_vec.push(rustfs_ecstore::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
});
let mut kvs = rustfs_ecstore::config::KVS::new();
for (key, value) in kv_map {
kvs.insert(key, value);
}
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
info!("Setting target config for type '{}', name '{}'", target_type, target_name);
ns.set_target_config(target_type, target_name, rustfs_ecstore::config::KVS(kvs_vec))
ns.set_target_config(target_type, target_name, kvs)
.await
.map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?;
@@ -278,8 +491,6 @@ impl Operation for ListNotificationTargets {
let ns = get_notification_system()?;
let targets = ns.get_target_values().await;
let target_count = targets.len();
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
@@ -291,18 +502,16 @@ impl Operation for ListNotificationTargets {
Ok(Ok(true)) => "online",
_ => "offline",
};
NotificationEndpoint {
account_id: target.id().id.clone(),
service: target.id().name.to_string(),
status: status.to_string(),
}
((target.id().id.clone(), target.id().name.to_string()), status.to_string())
});
}
let mut notification_endpoints = Vec::with_capacity(target_count);
while let Some(endpoint) = futures.next().await {
notification_endpoints.push(endpoint);
let mut runtime_statuses = HashMap::new();
while let Some((key, status)) = futures.next().await {
runtime_statuses.insert(key, status);
}
let config = ns.config.read().await.clone();
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
.map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
@@ -320,16 +529,32 @@ impl Operation for ListTargetsArns {
check_permissions(&req).await?;
let ns = get_notification_system()?;
let active_targets = ns.get_active_targets().await;
let targets = ns.get_target_values().await;
let region = req
.region
.clone()
.ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?;
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
let data_target_arn_list: Vec<_> = active_targets
.iter()
.map(|id| id.to_arn(region.as_str()).to_string())
.collect();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
(target.id(), status.to_string())
});
}
let mut target_statuses = Vec::new();
while let Some(target_status) = futures.next().await {
target_statuses.push(target_status);
}
let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
let data = serde_json::to_vec(&data_target_arn_list)
.map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
@@ -348,6 +573,10 @@ impl Operation for RemoveNotificationTarget {
check_permissions(&req).await?;
let ns = get_notification_system()?;
let config_snapshot = ns.config.read().await.clone();
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
info!("Removing target config for type '{}', name '{}'", target_type, target_name);
ns.remove_target_config(target_type, target_name)
@@ -372,3 +601,284 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &
let target_name = extract_param(params, "target_name")?;
Ok((target_type, target_name))
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_ecstore::config::{KV, KVS};
use rustfs_targets::arn::TargetID;
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars};
fn enabled_kvs(value: &str) -> KVS {
KVS(vec![KV {
key: ENABLE_KEY.to_string(),
value: value.to_string(),
hidden_if_empty: false,
}])
}
#[test]
fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
let mut cfg_map = HashMap::new();
cfg_map.insert(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("webhook-a".to_string(), enabled_kvs("on"))]),
);
cfg_map.insert(
NOTIFY_MQTT_SUB_SYS.to_string(),
HashMap::from([("mqtt-a".to_string(), enabled_kvs("on"))]),
);
let config = Config(cfg_map);
let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_notification_endpoints(&config, runtime);
let mqtt = merged
.iter()
.find(|entry| entry.account_id == "mqtt-a" && entry.service == "mqtt")
.expect("mqtt-a should be present");
assert_eq!(mqtt.status, "offline");
assert_eq!(mqtt.source, NotificationEndpointSource::Config);
let webhook = merged
.iter()
.find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
.expect("webhook-a should be present");
assert_eq!(webhook.status, "online");
assert_eq!(webhook.source, NotificationEndpointSource::Config);
}
#[test]
fn merge_notification_endpoints_skips_disabled_and_default_entries() {
let mut webhook_targets = HashMap::new();
webhook_targets.insert(DEFAULT_DELIMITER.to_string(), enabled_kvs("on"));
webhook_targets.insert("webhook-disabled".to_string(), enabled_kvs("off"));
webhook_targets.insert("webhook-enabled".to_string(), enabled_kvs("on"));
let config = Config(HashMap::from([(NOTIFY_WEBHOOK_SUB_SYS.to_string(), webhook_targets)]));
let runtime = HashMap::from([
(("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only" && entry.service == "mqtt")
.expect("env-only should be present");
assert_eq!(env_only.status, "offline");
assert_eq!(env_only.source, NotificationEndpointSource::Runtime);
let enabled = merged
.iter()
.find(|entry| entry.account_id == "webhook-enabled" && entry.service == "webhook")
.expect("webhook-enabled should be present");
assert_eq!(enabled.status, "online");
assert_eq!(enabled.source, NotificationEndpointSource::Config);
}
#[test]
fn merge_notification_endpoints_marks_env_and_mixed_sources() {
let config = Config(HashMap::from([
(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("mixed-target".to_string(), enabled_kvs("on"))]),
),
(
NOTIFY_MQTT_SUB_SYS.to_string(),
HashMap::from([("config-target".to_string(), enabled_kvs("on"))]),
),
]));
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-TARGET", Some("https://example.com/hook")),
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let runtime = HashMap::from([
(("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
(("env-only".to_string(), "webhook".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-target")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, NotificationEndpointSource::Env);
let config_only = merged
.iter()
.find(|entry| entry.account_id == "config-target")
.expect("config target should be present");
assert_eq!(config_only.source, NotificationEndpointSource::Config);
},
);
}
#[test]
fn target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", Some("on")),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook")),
],
|| {
let config = Config(HashMap::new());
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
assert!(reason.is_some());
assert!(reason.unwrap().contains("managed by environment variables"));
},
);
}
#[test]
fn target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
)]));
let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
assert!(reason.is_some());
assert!(reason.unwrap().contains("both persisted config and environment variables"));
});
}
#[test]
fn target_mutation_block_reason_allows_config_only_target() {
let target_name = "config-only-target";
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
)]));
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
}
#[test]
fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("mixed-disabled".to_string(), enabled_kvs("off"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_MIXED-DISABLED", Some("on")),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
],
|| {
let merged = merge_notification_endpoints(&config, HashMap::new());
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
assert_eq!(mixed.status, "offline");
},
);
}
#[test]
fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
],
|| {
let merged = merge_notification_endpoints(&config, HashMap::new());
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, NotificationEndpointSource::Env);
assert_eq!(env_only.status, "offline");
},
);
}
#[test]
fn collect_validated_key_values_rejects_duplicate_keys() {
let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
let key_values = vec![
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/one".to_string(),
},
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/two".to_string(),
},
];
let err = collect_validated_key_values(&key_values, &allowed_keys, NOTIFY_WEBHOOK_SUB_SYS).unwrap_err();
assert!(err.to_string().contains("duplicate key"));
}
#[test]
fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", Some("on")),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", Some("https://example.com/hook")),
],
|| {
let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
let merged = merge_notification_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
},
);
}
#[test]
fn collect_online_target_arns_filters_offline_targets() {
let arns = collect_online_target_arns(
"us-east-1",
vec![
(TargetID::new("webhook-a".to_string(), "webhook".to_string()), "online".to_string()),
(TargetID::new("mqtt-a".to_string(), "mqtt".to_string()), "offline".to_string()),
],
);
assert_eq!(arns, vec!["arn:rustfs:sqs:us-east-1:webhook-a:webhook".to_string()]);
}
#[test]
fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
HashMap::from([("PrimaryCase".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARYCASE", None::<&str>),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
],
|| {
assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
},
);
}
}
rustfs/src/admin/handlers/mod.rs
+2
View File
@@ -13,6 +13,7 @@
// limitations under the License.
pub mod account_info;
pub mod audit;
pub mod bucket_meta;
pub mod event;
pub mod group;
@@ -51,6 +52,7 @@ mod tests {
fn test_handler_struct_creation() {
// Test that handler structs can be created
let _account_handler = account_info::AccountInfoHandler {};
let _list_audit_targets = audit::ListAuditTargets {};
let _service_handler = system::ServiceHandle {};
let _server_info_handler = system::ServerInfoHandler {};
let _inspect_data_handler = system::InspectDataHandler {};
rustfs/src/admin/mod.rs
+3 -2
View File
@@ -25,8 +25,8 @@ mod console_test;
mod route_registration_test;
use handlers::{
bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
tier, user,
audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts,
system, tier, user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
@@ -55,6 +55,7 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
quota::register_quota_route(&mut r)?;
bucket_meta::register_bucket_meta_route(&mut r)?;
audit::register_audit_target_route(&mut r)?;
replication::register_replication_route(&mut r)?;
site_replication::register_site_replication_route(&mut r)?;
rustfs/src/admin/route_registration_test.rs
+6 -4
View File
@@ -14,8 +14,8 @@
use crate::admin::{
handlers::{
bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts, system,
tier, user,
audit, bucket_meta, heal, health, kms, oidc, pools, profile_admin, quota, rebalance, replication, site_replication, sts,
system, tier, user,
},
router::{AdminOperation, S3Router},
};
@@ -50,6 +50,7 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
tier::register_tier_route(router).expect("register tier route");
quota::register_quota_route(router).expect("register quota route");
bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route");
audit::register_audit_target_route(router).expect("register audit target route");
replication::register_replication_route(router).expect("register replication route");
site_replication::register_site_replication_route(router).expect("register site replication route");
profile_admin::register_profiling_route(router).expect("register profile route");
@@ -60,7 +61,6 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
#[test]
fn test_register_routes_cover_representative_admin_paths() {
let mut router: S3Router<AdminOperation> = S3Router::new(false);
register_admin_routes(&mut router);
assert_route(&router, Method::GET, HEALTH_PREFIX);
assert_route(&router, Method::HEAD, HEALTH_PREFIX);
@@ -91,6 +91,9 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::POST, &admin_path("/v3/idp/builtin/policy/detach"));
assert_route(&router, Method::GET, &admin_path("/v3/idp/builtin/policy-entities"));
assert_route(&router, Method::GET, &admin_path("/v3/target/list"));
assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit"));
assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset"));
assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));
assert_route(&router, Method::POST, &admin_path("/v3/service"));
@@ -165,7 +168,6 @@ fn test_register_routes_cover_representative_admin_paths() {
#[test]
fn test_admin_alias_paths_match_existing_admin_routes() {
let mut router: S3Router<AdminOperation> = S3Router::new(false);
register_admin_routes(&mut router);
for (method, path) in [
rustfs/src/server/audit.rs
+18 -8
View File
@@ -14,13 +14,27 @@
use crate::app::context::resolve_server_config;
use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState};
use rustfs_config::DEFAULT_DELIMITER;
use tracing::{info, warn};
fn server_config_from_context() -> Option<rustfs_ecstore::config::Config> {
resolve_server_config()
}
fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool {
for subsystem in [
rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
] {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
if targets.keys().any(|key| key != rustfs_config::DEFAULT_DELIMITER) {
return true;
}
}
false
}
/// Start the audit system.
/// This function checks if the audit subsystem is configured in the global server configuration.
/// If configured, it initializes and starts the audit system.
@@ -55,10 +69,8 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> {
"The global server configuration is loaded"
);
// 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
let mqtt_config = server_config.get_value(rustfs_config::audit::AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER);
let webhook_config = server_config.get_value(rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, DEFAULT_DELIMITER);
if mqtt_config.is_none() && webhook_config.is_none() {
let has_targets = has_any_audit_targets(&server_config);
if !has_targets {
info!(
target: "rustfs::main::start_audit_system",
"Audit subsystem (MQTT/Webhook) is not configured, and audit system initialization is skipped."
@@ -68,9 +80,7 @@ pub(crate) async fn start_audit_system() -> AuditResult<()> {
info!(
target: "rustfs::main::start_audit_system",
"Audit subsystem configuration detected (MQTT: {}, Webhook: {}) and started initializing the audit system.",
mqtt_config.is_some(),
webhook_config.is_some()
"Audit subsystem configuration detected and started initializing the audit system."
);
// 3. Initialize and start the audit system
let system = init_audit_system();
scripts/run.sh
+1 -1
View File
@@ -71,7 +71,7 @@ export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name
export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version
export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production
export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error
#export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging
export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory
export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily"
export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep