mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 23:26:53 +00:00
feat(admin): add audit target APIs and harden target source handling (#2350)
Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
@@ -18,12 +18,14 @@ use crate::{
|
||||
Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
|
||||
};
|
||||
use hashbrown::HashMap;
|
||||
use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
|
||||
use rustfs_config::notify::{
|
||||
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
|
||||
};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::store::{Key, Store};
|
||||
use rustfs_targets::target::EntityTarget;
|
||||
use rustfs_targets::target::QueuedPayload;
|
||||
use rustfs_targets::{StoreError, Target};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
@@ -34,6 +36,21 @@ use tracing::{debug, error, info, warn};
|
||||
|
||||
const MAX_RECENT_LIVE_EVENTS: usize = 1024;
|
||||
|
||||
fn subsystem_target_type(target_type: &str) -> &str {
|
||||
match target_type {
|
||||
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
|
||||
NOTIFY_MQTT_SUB_SYS => "mqtt",
|
||||
_ => target_type,
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
|
||||
TargetID {
|
||||
id: target_name.to_lowercase(),
|
||||
name: subsystem_target_type(target_type).to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct LiveEventBatch {
|
||||
pub events: Vec<Arc<Event>>,
|
||||
@@ -183,58 +200,84 @@ impl NotificationSystem {
|
||||
}
|
||||
}
|
||||
|
||||
/// Initializes targets and starts event streams for those with stores.
|
||||
/// Returns a map of (target_id -> cancel_sender) for streams that were started.
|
||||
async fn init_targets_and_start_streams(
|
||||
&self,
|
||||
targets: &[Box<dyn Target<Event> + Send + Sync>],
|
||||
) -> HashMap<TargetID, mpsc::Sender<()>> {
|
||||
let mut cancellers = HashMap::new();
|
||||
for target in targets {
|
||||
let target_id = target.id();
|
||||
info!("Initializing target: {}", target_id);
|
||||
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
if let Err(e) = target.init().await {
|
||||
warn!("Target {} Initialization failed: {}", target_id, e);
|
||||
// For targets without a store, init failure is fatal — skip.
|
||||
// For store-backed targets, still start the stream so queued events
|
||||
// can be drained when connectivity recovers (send_from_store retries).
|
||||
if !has_store {
|
||||
continue;
|
||||
}
|
||||
warn!(
|
||||
"Target {} has a store, starting stream despite init failure — \
|
||||
connectivity will be retried by send_from_store",
|
||||
target_id
|
||||
);
|
||||
} else {
|
||||
debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
|
||||
}
|
||||
|
||||
if !target.is_enabled() {
|
||||
info!("Target {} is not enabled, event stream processing is skipped", target_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(store) = target.store() {
|
||||
info!("Start event stream processing for target {}", target_id);
|
||||
|
||||
let store_clone = store.boxed_clone();
|
||||
let target_arc = Arc::from(target.clone_dyn());
|
||||
|
||||
let cancel_tx = self.enhanced_start_event_stream(
|
||||
store_clone,
|
||||
target_arc,
|
||||
self.metrics.clone(),
|
||||
self.concurrency_limiter.clone(),
|
||||
);
|
||||
|
||||
let target_id_clone = target_id.clone();
|
||||
cancellers.insert(target_id, cancel_tx);
|
||||
info!("Event stream processing for target {} is started successfully", target_id_clone);
|
||||
} else {
|
||||
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
|
||||
}
|
||||
}
|
||||
cancellers
|
||||
}
|
||||
|
||||
/// Initializes the notification system
|
||||
pub async fn init(&self) -> Result<(), NotificationError> {
|
||||
info!("Initialize notification system...");
|
||||
|
||||
let config = self.config.read().await;
|
||||
debug!("Initializing notification system with config: {:?}", *config);
|
||||
let config = {
|
||||
let guard = self.config.read().await;
|
||||
debug!("Initializing notification system with config: {:?}", *guard);
|
||||
guard.clone()
|
||||
};
|
||||
|
||||
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
|
||||
|
||||
info!("{} notification targets were created", targets.len());
|
||||
|
||||
// Initiate event stream processing for each storage enabled target
|
||||
let mut cancellers = HashMap::new();
|
||||
for target in &targets {
|
||||
let target_id = target.id();
|
||||
info!("Initializing target: {}", target.id());
|
||||
// Initialize the target
|
||||
if let Err(e) = target.init().await {
|
||||
warn!("Target {} Initialization failed:{}", target.id(), e);
|
||||
continue;
|
||||
}
|
||||
debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled());
|
||||
// Check if the target is enabled and has storage
|
||||
if target.is_enabled() {
|
||||
if let Some(store) = target.store() {
|
||||
info!("Start event stream processing for target {}", target.id());
|
||||
// Initialize targets and start event streams
|
||||
let cancellers = self.init_targets_and_start_streams(&targets).await;
|
||||
|
||||
// The storage of the cloned target and the target itself
|
||||
let store_clone = store.boxed_clone();
|
||||
let target_box = target.clone_dyn();
|
||||
let target_arc = Arc::from(target_box);
|
||||
|
||||
// Add a reference to the monitoring metrics
|
||||
let metrics = self.metrics.clone();
|
||||
let semaphore = self.concurrency_limiter.clone();
|
||||
|
||||
// Encapsulated enhanced version of start_event_stream
|
||||
let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
|
||||
|
||||
// Start event stream processing and save cancel sender
|
||||
let target_id_clone = target_id.clone();
|
||||
cancellers.insert(target_id, cancel_tx);
|
||||
info!("Event stream processing for target {} is started successfully", target_id_clone);
|
||||
} else {
|
||||
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
|
||||
}
|
||||
} else {
|
||||
info!("Target {} is not enabled, event stream processing is skipped", target_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Update canceler collection
|
||||
// Update canceller collection
|
||||
*self.stream_cancellers.write().await = cancellers;
|
||||
|
||||
// Initialize the bucket target
|
||||
self.notifier.init_bucket_targets(targets).await?;
|
||||
info!("Notification system initialized");
|
||||
@@ -333,7 +376,7 @@ impl NotificationSystem {
|
||||
info!("Attempting to remove target: {}", target_id);
|
||||
|
||||
let ttype = target_type.to_lowercase();
|
||||
let tname = target_id.name.to_lowercase();
|
||||
let tname = target_id.id.to_lowercase();
|
||||
|
||||
self.update_config_and_reload(|config| {
|
||||
let mut changed = false;
|
||||
@@ -405,11 +448,7 @@ impl NotificationSystem {
|
||||
|
||||
let ttype = target_type.to_lowercase();
|
||||
let tname = target_name.to_lowercase();
|
||||
|
||||
let target_id = TargetID {
|
||||
id: tname.clone(),
|
||||
name: ttype.clone(),
|
||||
};
|
||||
let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
|
||||
|
||||
// Deletion is prohibited if bucket rules refer to it
|
||||
if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
|
||||
@@ -451,7 +490,7 @@ impl NotificationSystem {
|
||||
/// Enhanced event stream startup function, including monitoring and concurrency control
|
||||
fn enhanced_start_event_stream(
|
||||
&self,
|
||||
store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
|
||||
store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
|
||||
target: Arc<dyn Target<Event> + Send + Sync>,
|
||||
metrics: Arc<NotificationMetrics>,
|
||||
semaphore: Arc<Semaphore>,
|
||||
@@ -476,14 +515,13 @@ impl NotificationSystem {
|
||||
let _ = cancel_tx.send(()).await;
|
||||
}
|
||||
|
||||
// Clear the target_list and ensure that reload is a replacement reconstruction (solve the target_list len unchanged/residual problem)
|
||||
// Clear the target_list and ensure that reload is a replacement reconstruction
|
||||
self.notifier.remove_all_bucket_targets().await;
|
||||
|
||||
// Update the config
|
||||
self.update_config(new_config.clone()).await;
|
||||
|
||||
// Create a new target from configuration
|
||||
// This function will now be responsible for merging env, creating and persisting the final configuration.
|
||||
// Create new targets from configuration
|
||||
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
|
||||
.registry
|
||||
.create_targets_from_config(&new_config)
|
||||
@@ -492,46 +530,8 @@ impl NotificationSystem {
|
||||
|
||||
info!("{} notification targets were created from the new configuration", targets.len());
|
||||
|
||||
// Start new event stream processing for each storage enabled target
|
||||
let mut new_cancellers = HashMap::new();
|
||||
for target in &targets {
|
||||
let target_id = target.id();
|
||||
|
||||
// Initialize the target
|
||||
if let Err(e) = target.init().await {
|
||||
error!("Target {} Initialization failed:{}", target_id, e);
|
||||
continue;
|
||||
}
|
||||
// Check if the target is enabled and has storage
|
||||
if target.is_enabled() {
|
||||
if let Some(store) = target.store() {
|
||||
info!("Start new event stream processing for target {}", target_id);
|
||||
|
||||
// The storage of the cloned target and the target itself
|
||||
let store_clone = store.boxed_clone();
|
||||
// let target_box = target.clone_dyn();
|
||||
let target_arc = Arc::from(target.clone_dyn());
|
||||
|
||||
// Encapsulated enhanced version of start_event_stream
|
||||
let cancel_tx = self.enhanced_start_event_stream(
|
||||
store_clone,
|
||||
target_arc,
|
||||
self.metrics.clone(),
|
||||
self.concurrency_limiter.clone(),
|
||||
);
|
||||
|
||||
// Start event stream processing and save cancel sender
|
||||
// let cancel_tx = start_event_stream(store_clone, target_clone);
|
||||
let target_id_clone = target_id.clone();
|
||||
new_cancellers.insert(target_id, cancel_tx);
|
||||
info!("Event stream processing of target {} is restarted successfully", target_id_clone);
|
||||
} else {
|
||||
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
|
||||
}
|
||||
} else {
|
||||
info!("Target {} disabled, event stream processing is skipped", target_id);
|
||||
}
|
||||
}
|
||||
// Initialize targets and start event streams using shared helper
|
||||
let new_cancellers = self.init_targets_and_start_streams(&targets).await;
|
||||
|
||||
// Update canceler collection
|
||||
*cancellers = new_cancellers;
|
||||
@@ -665,4 +665,18 @@ mod tests {
|
||||
assert_eq!(batch.events.len(), 1);
|
||||
assert_eq!(batch.events[0].s3.object.key, "one");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
|
||||
let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
|
||||
assert_eq!(target_id.id, "primary");
|
||||
assert_eq!(target_id.name, "webhook");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
|
||||
let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
|
||||
assert_eq!(target_id.id, "analytics");
|
||||
assert_eq!(target_id.name, "mqtt");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user