gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-09 06:39:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(admin): add audit target APIs and harden target source handling (#2350)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-04 09:07:22 +08:00
committed by GitHub
parent 67863630b2
commit d2901fd78c
29 changed files with 3534 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/audit/src/registry.rs
+10 -86
View File
@@ -109,9 +109,6 @@ impl AuditRegistry {
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
// A collection of asynchronous tasks for concurrently executing target creation
let mut tasks = FuturesUnordered::new();
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
// Record the defaults for each segment so that the segment can eventually be rebuilt
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
// 1. Traverse all registered plants and process them by target type
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
@@ -125,9 +122,6 @@ impl AuditRegistry {
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
debug!(?default_cfg, "Get the default configuration");
// Save defaults for eventual write back
section_defaults.insert(section_name.clone(), default_cfg.clone());
// *** Optimization point 1: Get all legitimate fields of the current target type ***
let valid_fields = factory.get_valid_fields();
debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -235,103 +229,28 @@ impl AuditRegistry {
if enabled {
info!(instance_id = %id, "Target is enabled, ready to create a task");
// 5.3. Create asynchronous tasks for enabled instances
let target_type_clone = target_type.clone();
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(target_type_clone, tid, result, Arc::clone(&merged_config_arc))
(tid, result)
});
} else {
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
// Remove disabled target from final configuration
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
info!(instance_id = %id, "Skip disabled target");
}
}
}
// 6. Concurrently execute all creation tasks and collect results
let mut successful_targets = Vec::new();
let mut successful_configs = Vec::new();
while let Some((target_type, id, result, final_config)) = tasks.next().await {
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
successful_targets.push(target);
successful_configs.push((target_type, id, final_config));
}
Err(e) => {
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
}
}
}
// 7. Aggregate new configuration and write back to system configuration
if !successful_configs.is_empty() || !section_defaults.is_empty() {
info!(
"Prepare to update {} successfully created target configurations to the system configuration...",
successful_configs.len()
);
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
for (target_type, id, kvs) in successful_configs {
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
successes_by_section
.entry(section_name)
.or_default()
.insert(id.to_lowercase(), (*kvs).clone());
}
let mut new_config = config.clone();
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
let mut sections: HashSet<String> = HashSet::new();
sections.extend(section_defaults.keys().cloned());
sections.extend(successes_by_section.keys().cloned());
for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item
if let Some(default_kvs) = section_defaults.get(&section)
&& !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
}
// Add successful instance item
if let Some(instances) = successes_by_section.get(&section) {
for (id, kvs) in instances {
section_map.insert(id.clone(), kvs.clone());
}
}
// Empty breaks are removed and non-empty breaks are replaced entirely.
if section_map.is_empty() {
new_config.0.remove(&section);
} else {
new_config.0.insert(section, section_map);
}
}
if &new_config == config {
info!("Audit target configuration unchanged, skip persisting server config");
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Err(AuditError::StorageNotAvailable(
"Failed to save target configuration: server storage not initialized".to_string(),
));
};
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
Ok(_) => {
info!("The new configuration was saved to the system successfully.")
}
Err(e) => {
error!("Failed to save the new configuration: {}", e);
return Err(AuditError::SaveConfig(Box::new(e)));
error!(instance_id = %id, error = %e, "Failed to create a target");
}
}
}
@@ -371,6 +290,11 @@ impl AuditRegistry {
self.targets.get(id).map(|t| t.as_ref())
}
/// Lists cloned target values for runtime inspection without exposing mutable registry access.
pub fn list_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
self.targets.values().map(|target| target.clone_dyn()).collect()
}
/// Lists all target IDs
///
/// # Returns
crates/audit/src/system.rs
+160 -107
View File
@@ -13,14 +13,15 @@
// limitations under the License.
use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
use hashbrown::HashMap;
use rustfs_ecstore::config::Config;
use rustfs_targets::{
StoreError, Target, TargetError,
store::{Key, Store},
target::EntityTarget,
target::{EntityTarget, QueuedPayload},
};
use std::sync::Arc;
use tokio::sync::{Mutex, RwLock};
use tokio::sync::{Mutex, RwLock, mpsc};
use tracing::{error, info, warn};
/// State of the audit system
@@ -39,6 +40,8 @@ pub struct AuditSystem {
registry: Arc<Mutex<AuditRegistry>>,
state: Arc<RwLock<AuditSystemState>>,
config: Arc<RwLock<Option<Config>>>,
/// Cancellation senders for active audit stream tasks (target_id -> cancel tx)
stream_cancellers: Arc<RwLock<HashMap<String, mpsc::Sender<()>>>>,
}
impl Default for AuditSystem {
@@ -54,6 +57,7 @@ impl AuditSystem {
registry: Arc::new(Mutex::new(AuditRegistry::new())),
state: Arc::new(RwLock::new(AuditSystemState::Stopped)),
config: Arc::new(RwLock::new(None)),
stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
}
}
@@ -110,27 +114,7 @@ impl AuditSystem {
// Initialize all targets
for target in targets {
let target_id = target.id().to_string();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
} else {
// After successful initialization, if enabled and there is a store, start the send from storage task
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target");
let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
self.start_audit_stream_with_batching(store_clone, target_arc);
info!(target_id = %target_id, "Audit stream processing started");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing");
}
registry.add_target(target_id, target);
}
self.init_and_register_target(target, &mut registry).await;
}
// Update state to running
@@ -214,6 +198,9 @@ impl AuditSystem {
info!("Stopping audit system");
// Stop all stream tasks first
self.stop_all_streams().await;
// Close all targets
let mut registry = self.registry.lock().await;
if let Err(e) = registry.close_all().await {
@@ -258,52 +245,49 @@ impl AuditSystem {
let state = self.state.read().await;
match *state {
AuditSystemState::Running => {
// Continue with dispatch
info!("Dispatching audit log entry");
}
AuditSystemState::Running => {}
AuditSystemState::Paused => {
// Skip dispatch when paused
return Ok(());
}
_ => {
// Don't dispatch when not running
return Err(AuditError::NotInitialized("Audit system is not running".to_string()));
}
}
drop(state);
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
// Collect cloned targets under lock, then dispatch without holding it
let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
if target_keys.is_empty() {
warn!("No audit targets configured for dispatch");
return Ok(());
}
if target_keys.is_empty() {
warn!("No audit targets configured for dispatch");
return Ok(());
}
// Dispatch to all targets concurrently
target_keys
.into_iter()
.filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
.collect()
};
// Dispatch to all targets concurrently (no lock held)
let mut tasks = Vec::new();
for target_key in target_keys {
if let Some(target) = registry.get_target(&target_key) {
let entry_clone = Arc::clone(&entry);
let target_key_clone = target_key.clone();
for (target_key, target) in targets {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
// Create EntityTarget for the audit log entry
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event, // Default, should be derived from entry
data: (*entry_clone).clone(),
};
let task = async move {
let result = target.save(Arc::new(entity_target)).await;
(target_key, result)
};
let task = async move {
let result = target.save(Arc::new(entity_target)).await;
(target_key_clone, result)
};
tasks.push(task);
}
tasks.push(task);
}
// Execute all dispatch tasks
@@ -359,39 +343,45 @@ impl AuditSystem {
}
drop(state);
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
// Collect targets under lock, then dispatch without holding it
let targets: Vec<(String, Box<dyn Target<AuditEntry> + Send + Sync>)> = {
let registry = self.registry.lock().await;
let target_keys = registry.list_targets();
if target_keys.is_empty() {
warn!("No audit targets configured for batch dispatch");
return Ok(());
}
if target_keys.is_empty() {
warn!("No audit targets configured for batch dispatch");
return Ok(());
}
target_keys
.into_iter()
.filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn())))
.collect()
};
let mut tasks = Vec::new();
for target_key in target_keys {
if let Some(target) = registry.get_target(&target_key) {
let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
let target_key_clone = target_key.clone();
for (target_key, target) in targets {
let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect();
let target_key_clone = target_key.clone();
let task = async move {
let mut success_count = 0;
let mut errors = Vec::new();
for entry in entries_clone {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
match target.save(Arc::new(entity_target)).await {
Ok(_) => success_count += 1,
Err(e) => errors.push(e),
}
let task = async move {
let mut success_count = 0;
let mut errors = Vec::new();
for entry in entries_clone {
let entity_target = EntityTarget {
object_name: entry.api.name.clone().unwrap_or_default(),
bucket_name: entry.api.bucket.clone().unwrap_or_default(),
event_name: entry.event,
data: (*entry).clone(),
};
match target.save(Arc::new(entity_target)).await {
Ok(_) => success_count += 1,
Err(e) => errors.push(e),
}
(target_key_clone, success_count, errors)
};
tasks.push(task);
}
}
(target_key_clone, success_count, errors)
};
tasks.push(task);
}
let results = futures::future::join_all(tasks).await;
@@ -417,6 +407,60 @@ impl AuditSystem {
Ok(())
}
/// Stops all active audit stream tasks by sending cancellation signals.
async fn stop_all_streams(&self) {
let mut cancellers = self.stream_cancellers.write().await;
for (target_id, cancel_tx) in cancellers.drain() {
info!(target_id = %target_id, "Stopping audit stream");
let _ = cancel_tx.send(()).await;
}
}
/// Initializes a single target: runs init(), starts stream if store is present,
/// and adds it to the registry. For store-backed targets, registration and stream
/// startup proceed even if init() fails so queued entries can be drained later.
async fn init_and_register_target(
&self,
target: Box<dyn Target<AuditEntry> + Send + Sync>,
registry: &mut AuditRegistry,
) -> Option<String> {
let target_id = target.id().to_string();
let has_store = target.store().is_some();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize audit target");
// Non-store targets: init failure is fatal.
if !has_store {
return None;
}
// Store-backed targets: still register and start the stream so queued
// entries can be drained when connectivity recovers.
warn!(
target_id = %target_id,
"Proceeding with store-backed audit target despite init failure"
);
}
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target");
let store_clone: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send> = store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc);
self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx);
info!(target_id = %target_id, "Audit stream processing started");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing");
}
registry.add_target(target_id.clone(), target);
Some(target_id)
}
/// Starts the audit stream processing for a target with batching and retry logic
///
/// # Arguments
@@ -427,9 +471,10 @@ impl AuditSystem {
/// and attempts to send them to the specified target. It implements retry logic with exponential backoff
fn start_audit_stream_with_batching(
&self,
store: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send>,
store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<AuditEntry> + Send + Sync>,
) {
) -> mpsc::Sender<()> {
let (cancel_tx, mut cancel_rx) = mpsc::channel(1);
let state = self.state.clone();
tokio::spawn(async move {
@@ -442,6 +487,12 @@ impl AuditSystem {
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
loop {
// Check for cancellation signal
if cancel_rx.try_recv().is_ok() {
info!("Audit stream cancelled for target: {}", target.id());
break;
}
match *state.read().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {}
_ => {
@@ -452,11 +503,22 @@ impl AuditSystem {
let keys: Vec<Key> = store.list();
if keys.is_empty() {
sleep(Duration::from_millis(500)).await;
tokio::select! {
_ = sleep(Duration::from_millis(500)) => {},
_ = cancel_rx.recv() => {
info!("Audit stream cancelled during idle for target: {}", target.id());
return;
}
}
continue;
}
for key in keys {
if cancel_rx.try_recv().is_ok() {
info!("Audit stream cancelled during processing for target: {}", target.id());
return;
}
let mut retries = 0usize;
let mut success = false;
@@ -497,6 +559,8 @@ impl AuditSystem {
sleep(Duration::from_millis(100)).await;
}
});
cancel_tx
}
/// Enables a specific target
@@ -594,6 +658,12 @@ impl AuditSystem {
registry.list_targets()
}
/// Returns cloned target values for read-only runtime inspection.
pub async fn get_target_values(&self) -> Vec<Box<dyn Target<AuditEntry> + Send + Sync>> {
let registry = self.registry.lock().await;
registry.list_target_values()
}
/// Gets information about a specific target
///
/// # Arguments
@@ -616,9 +686,11 @@ impl AuditSystem {
pub async fn reload_config(&self, new_config: Config) -> AuditResult<()> {
info!("Reloading audit system configuration");
// Record config reload
observability::record_config_reload();
// Stop all existing stream tasks first
self.stop_all_streams().await;
// Store new configuration
{
let mut config_guard = self.config.write().await;
@@ -636,28 +708,9 @@ impl AuditSystem {
Ok(targets) => {
info!(target_count = targets.len(), "Reloaded audit targets successfully");
// Initialize all new targets
for target in targets {
let target_id = target.id().to_string();
if let Err(e) = target.init().await {
error!(target_id = %target_id, error = %e, "Failed to initialize reloaded audit target");
} else {
// Same starts the storage stream after a heavy load
if target.is_enabled() {
if let Some(store) = target.store() {
info!(target_id = %target_id, "Start audit stream processing for target (reload)");
let store_clone: Box<dyn Store<EntityTarget<AuditEntry>, Error = StoreError, Key = Key> + Send> =
store.boxed_clone();
let target_arc: Arc<dyn Target<AuditEntry> + Send + Sync> = Arc::from(target.clone_dyn());
self.start_audit_stream_with_batching(store_clone, target_arc);
info!(target_id = %target_id, "Audit stream processing started (reload)");
} else {
info!(target_id = %target_id, "No store configured, skip audit stream processing (reload)");
}
} else {
info!(target_id = %target_id, "Target disabled, skip audit stream processing (reload)");
}
registry.add_target(target.id().to_string(), target);
if let Some(target_id) = self.init_and_register_target(target, &mut registry).await {
info!(target_id = %target_id, "Target initialized (reload)");
}
}