gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 19:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): reduce runtime noise (#3363)

Browse Source
This commit is contained in:
houseme
2026-06-11 19:49:01 +08:00
committed by GitHub
parent a0b6636b61
commit 0a987d870b
19 changed files with 1974 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/config_manager.rs
+139 -17
View File
@@ -23,7 +23,12 @@ use rustfs_config::server_config::{Config, KVS};
use rustfs_targets::{Target, arn::TargetID};
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, info};
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_CONFIG: &str = "config";
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
pub(crate) fn notify_configuration_hint() -> String {
let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE);
@@ -81,7 +86,13 @@ impl NotifyConfigManager {
}
pub async fn init(&self) -> Result<(), NotificationError> {
info!("Initialize notification system...");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initializing",
"Initializing notification system"
);
let config = {
let guard = self.config.read().await;
@@ -94,19 +105,48 @@ impl NotifyConfigManager {
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
info!("{} notification targets were created", targets.len());
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets"
);
if targets.is_empty() {
warn!("{}", notify_configuration_hint());
debug!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets"
);
}
let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
self.runtime_facade.replace_targets(activation).await?;
info!("Notification system initialized");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "initialized",
"Initialized notification system"
);
Ok(())
}
pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
info!("Attempting to remove target: {}", target_id);
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target",
target_id = %target_id,
target_type,
"Attempting to remove notification target"
);
let ttype = target_type.to_lowercase();
let tname = target_id.id.to_lowercase();
@@ -115,7 +155,15 @@ impl NotifyConfigManager {
let mut changed = false;
if let Some(targets_of_type) = config.0.get_mut(&ttype) {
if targets_of_type.remove(&tname).is_some() {
info!("Removed target {} from configuration", target_id);
info!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target",
target_id = %target_id,
result = "removed",
"Removed notification target from configuration"
);
changed = true;
}
if targets_of_type.is_empty() {
@@ -123,7 +171,15 @@ impl NotifyConfigManager {
}
}
if !changed {
warn!("Target {} not found in configuration", target_id);
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target",
target_id = %target_id,
result = "not_found",
"Notification target not found in configuration"
);
}
changed
})
@@ -131,7 +187,15 @@ impl NotifyConfigManager {
}
pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
info!("Setting config for target {} of type {}", target_name, target_type);
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "set_target_config",
target_type,
target_name,
"Setting notification target configuration"
);
let ttype = target_type.to_lowercase();
let tname = target_name.to_lowercase();
self.update_config_and_reload(|config| {
@@ -142,7 +206,15 @@ impl NotifyConfigManager {
}
pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
info!("Removing config for target {} of type {}", target_name, target_type);
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target_config",
target_type,
target_name,
"Removing notification target configuration"
);
let ttype = target_type.to_lowercase();
let tname = target_name.to_lowercase();
@@ -166,7 +238,16 @@ impl NotifyConfigManager {
}
}
if !changed {
info!("Target {} of type {} not found, no changes made.", target_name, target_type);
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "remove_target_config",
target_type = %target_type,
target_name = %target_name,
result = "not_found",
"Notification target configuration not found"
);
}
debug!(
subsystem_count = config.0.len(),
@@ -178,7 +259,13 @@ impl NotifyConfigManager {
}
pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
info!("Reload notification configuration starts");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloading",
"Reloading notification configuration"
);
self.update_config(new_config.clone()).await;
@@ -188,14 +275,35 @@ impl NotifyConfigManager {
.await
.map_err(NotificationError::Target)?;
info!("{} notification targets were created from the new configuration", targets.len());
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "targets_created",
target_count = targets.len(),
"Created notification targets from reloaded configuration"
);
if targets.is_empty() {
warn!("{}", notify_configuration_hint());
debug!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "idle",
reason = "no_targets_configured",
hint = %notify_configuration_hint(),
"Notification runtime has no configured targets after reload"
);
}
let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
self.runtime_facade.replace_targets(activation).await?;
info!("Configuration reloaded end");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
state = "reloaded",
"Reloaded notification configuration"
);
Ok(())
}
@@ -219,7 +327,14 @@ impl NotifyConfigManager {
.map_err(|e| NotificationError::ReadConfig(e.to_string()))?;
if !modifier(&mut new_config) {
info!("Configuration not changed, skipping save and reload.");
debug!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "unchanged",
"Notification configuration unchanged; skipping reload"
);
return Ok(());
}
@@ -227,7 +342,14 @@ impl NotifyConfigManager {
.await
.map_err(|e| NotificationError::SaveConfig(e.to_string()))?;
info!("Configuration updated. Reloading system...");
info!(
event = EVENT_NOTIFY_CONFIG_UPDATE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_CONFIG,
action = "reload_if_changed",
result = "updated",
"Notification configuration updated; reloading runtime"
);
self.reload_config(new_config).await
}
}
crates/notify/src/notifier.rs
+121 -16
View File
@@ -21,6 +21,14 @@ use std::sync::Arc;
use tokio::sync::{RwLock, Semaphore};
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_DISPATCH: &str = "dispatch";
const EVENT_NOTIFY_DISPATCH_SKIPPED: &str = "notify_dispatch_skipped";
const EVENT_NOTIFY_DISPATCH_FAILED: &str = "notify_dispatch_failed";
const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
/// Manages event notification to targets based on rules
@@ -83,7 +91,13 @@ impl EventNotifier {
// The logic for sending cancel signals via stream_cancel_senders would be removed.
// TargetList::clear_targets_only already handles calling target.close().
target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
info!("Removed all targets and their streams");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_cleared",
"Removed all notify targets"
);
}
/// Sends an event to the appropriate targets based on the bucket rules
@@ -98,7 +112,15 @@ impl EventNotifier {
let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await;
if target_ids.is_empty() {
debug!("No matching targets for event in bucket: {}", bucket_name);
debug!(
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name,
object = %object_key,
reason = "no_matching_targets",
"Skipped notify dispatch"
);
self.metrics.increment_skipped();
return;
}
@@ -107,7 +129,15 @@ impl EventNotifier {
// Use scope to limit the borrow scope of target_list
let target_list_guard = self.target_list.read().await;
info!("Sending event to targets: {:?}", target_ids);
debug!(
event = EVENT_NOTIFY_DISPATCH_STARTED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name,
object = %object_key,
target_count = target_ids_len,
"Dispatching notify event"
);
for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
if let Some(target_arc) = target_list_guard.get(&target_id) {
@@ -115,7 +145,14 @@ impl EventNotifier {
// target_arc is already Arc, clone it for the async task
let target_for_task = target_arc.clone();
if !target_for_task.is_enabled() {
debug!("Skipping disabled target: {}", target_for_task.name());
debug!(
event = EVENT_NOTIFY_DISPATCH_SKIPPED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_for_task.id(),
reason = "target_disabled",
"Skipped notify dispatch target"
);
continue;
}
let limiter = self.send_limiter.clone();
@@ -123,7 +160,14 @@ impl EventNotifier {
let event_clone = event.clone();
let is_deferred = target_for_task.store().is_some();
let target_name_for_task = target_for_task.name(); // Get the name before generating the task
debug!("Preparing to send event to target: {}", target_name_for_task);
debug!(
event = EVENT_NOTIFY_DISPATCH_STARTED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_for_task.id(),
deferred = is_deferred,
"Prepared notify target dispatch"
);
// Use cloned data in closures to avoid borrowing conflicts
// Create an EntityTarget from the event
let entity_target: Arc<EntityTarget<Event>> = Arc::new(EntityTarget {
@@ -137,26 +181,56 @@ impl EventNotifier {
let _permit = match limiter.acquire_owned().await {
Ok(p) => p,
Err(e) => {
error!("Failed to acquire send permit for target {}: {}", target_name_for_task, e);
error!(
event = EVENT_NOTIFY_DISPATCH_FAILED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task,
error = %e,
reason = "permit_acquire_failed",
"Failed to acquire notify send permit"
);
metrics.increment_failed();
return;
}
};
if let Err(e) = target_for_task.save(entity_target.clone()).await {
metrics.increment_failed();
error!("Failed to send event to target {}: {}", target_name_for_task, e);
error!(
event = EVENT_NOTIFY_DISPATCH_FAILED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task,
error = %e,
reason = "send_failed",
"Failed to dispatch notify event"
);
} else {
if is_deferred {
metrics.decrement_processing();
} else {
metrics.increment_processed();
}
debug!("Successfully saved event to target {}", target_name_for_task);
debug!(
event = EVENT_NOTIFY_DISPATCH_COMPLETED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_name_for_task,
deferred = is_deferred,
"Completed notify target dispatch"
);
}
});
handles.push(handle);
} else {
warn!("Target ID {:?} found in rules but not in target list.", target_id);
warn!(
event = EVENT_NOTIFY_DISPATCH_FAILED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
target_id = %target_id,
reason = "target_missing_from_runtime",
"Matched notify target is missing from runtime"
);
self.metrics.increment_skipped();
}
}
@@ -166,10 +240,24 @@ impl EventNotifier {
// Wait for all tasks to be completed
for handle in handles {
if let Err(e) = handle.await {
error!("Task for sending/saving event failed: {}", e);
error!(
event = EVENT_NOTIFY_DISPATCH_FAILED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
error = %e,
reason = "join_failed",
"Notify dispatch task failed"
);
}
}
info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name);
debug!(
event = EVENT_NOTIFY_DISPATCH_COMPLETED,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
bucket = %bucket_name,
target_count = target_ids_len,
"Finished notify dispatch fan-out"
);
}
/// Initializes the targets for buckets from shared target handles.
@@ -179,14 +267,24 @@ impl EventNotifier {
target_list_guard.clear();
for target in targets_to_init {
debug!("init bucket target: {}", target.name());
debug!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "target_init",
target_id = %target.id(),
"Initializing notify runtime target"
);
target_list_guard.add(target)?;
}
info!(
"Initialized {} shared targets, list size: {}",
target_list_guard.len(),
target_list_guard.len()
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "targets_initialized",
target_count = target_list_guard.len(),
"Initialized notify runtime targets"
);
Ok(())
}
@@ -223,7 +321,14 @@ impl TargetList {
let id = target.id();
if self.runtime.get_by_target_id(&id).is_some() {
// Potentially update or log a warning/error if replacing an existing target.
warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id);
warn!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_DISPATCH,
state = "target_overwrite",
target_id = %id,
"Overwriting existing notify runtime target"
);
}
self.runtime.add_arc(target);
Ok(())
crates/notify/src/runtime_facade.rs
+44 -6
View File
@@ -19,7 +19,11 @@ use rustfs_targets::{
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{RwLock, Semaphore};
use tracing::info;
use tracing::{debug, info};
const LOG_COMPONENT_NOTIFY: &str = "notify";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
#[derive(Clone)]
pub struct NotifyRuntimeFacade {
@@ -55,9 +59,24 @@ impl NotifyRuntimeFacade {
}),
Arc::new(|target_id, has_replay| {
if has_replay {
info!("Event stream processing for target {} is started successfully", target_id);
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
target_id = %target_id,
state = "replay_started",
"Started notify replay worker"
);
} else {
info!("Target {} has no replay worker to start", target_id);
debug!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
target_id = %target_id,
state = "replay_skipped",
reason = "no_store_configured",
"Skipped notify replay worker startup"
);
}
}),
Some(concurrency_limiter),
@@ -97,10 +116,23 @@ impl NotifyRuntimeFacade {
}
pub async fn shutdown(&self) {
info!("Turn off the notification system");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopping",
"Stopping notification runtime"
);
let active_targets = self.replay_workers.read().await.len();
info!("Stops {} active event stream processing tasks", active_targets);
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "replay_stopping",
active_targets,
"Stopping notify replay workers"
);
{
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
@@ -116,7 +148,13 @@ impl NotifyRuntimeFacade {
}
tokio::time::sleep(Duration::from_millis(500)).await;
info!("Notify the system to be shut down completed");
info!(
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
component = LOG_COMPONENT_NOTIFY,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "stopped",
"Stopped notification runtime"
);
}
}