gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 05:43:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(admin): add audit target APIs and harden target source handling (#2350)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
2026-04-04 09:07:22 +08:00
committed by GitHub
parent 67863630b2
commit d2901fd78c
29 changed files with 3534 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/notify/src/integration.rs
+108 -94
View File
@@ -18,12 +18,14 @@ use crate::{
Event, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream,
};
use hashbrown::HashMap;
use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY};
use rustfs_config::notify::{
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_ecstore::config::{Config, KVS};
use rustfs_s3_common::EventName;
use rustfs_targets::arn::TargetID;
use rustfs_targets::store::{Key, Store};
use rustfs_targets::target::EntityTarget;
use rustfs_targets::target::QueuedPayload;
use rustfs_targets::{StoreError, Target};
use std::collections::VecDeque;
use std::sync::Arc;
@@ -34,6 +36,21 @@ use tracing::{debug, error, info, warn};
const MAX_RECENT_LIVE_EVENTS: usize = 1024;
fn subsystem_target_type(target_type: &str) -> &str {
match target_type {
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_MQTT_SUB_SYS => "mqtt",
_ => target_type,
}
}
fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID {
TargetID {
id: target_name.to_lowercase(),
name: subsystem_target_type(target_type).to_string(),
}
}
#[derive(Clone)]
pub struct LiveEventBatch {
pub events: Vec<Arc<Event>>,
@@ -183,58 +200,84 @@ impl NotificationSystem {
}
}
/// Initializes targets and starts event streams for those with stores.
/// Returns a map of (target_id -> cancel_sender) for streams that were started.
async fn init_targets_and_start_streams(
&self,
targets: &[Box<dyn Target<Event> + Send + Sync>],
) -> HashMap<TargetID, mpsc::Sender<()>> {
let mut cancellers = HashMap::new();
for target in targets {
let target_id = target.id();
info!("Initializing target: {}", target_id);
let has_store = target.store().is_some();
if let Err(e) = target.init().await {
warn!("Target {} Initialization failed: {}", target_id, e);
// For targets without a store, init failure is fatal — skip.
// For store-backed targets, still start the stream so queued events
// can be drained when connectivity recovers (send_from_store retries).
if !has_store {
continue;
}
warn!(
"Target {} has a store, starting stream despite init failure — \
connectivity will be retried by send_from_store",
target_id
);
} else {
debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled());
}
if !target.is_enabled() {
info!("Target {} is not enabled, event stream processing is skipped", target_id);
continue;
}
if let Some(store) = target.store() {
info!("Start event stream processing for target {}", target_id);
let store_clone = store.boxed_clone();
let target_arc = Arc::from(target.clone_dyn());
let cancel_tx = self.enhanced_start_event_stream(
store_clone,
target_arc,
self.metrics.clone(),
self.concurrency_limiter.clone(),
);
let target_id_clone = target_id.clone();
cancellers.insert(target_id, cancel_tx);
info!("Event stream processing for target {} is started successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
}
cancellers
}
/// Initializes the notification system
pub async fn init(&self) -> Result<(), NotificationError> {
info!("Initialize notification system...");
let config = self.config.read().await;
debug!("Initializing notification system with config: {:?}", *config);
let config = {
let guard = self.config.read().await;
debug!("Initializing notification system with config: {:?}", *guard);
guard.clone()
};
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
info!("{} notification targets were created", targets.len());
// Initiate event stream processing for each storage enabled target
let mut cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
info!("Initializing target: {}", target.id());
// Initialize the target
if let Err(e) = target.init().await {
warn!("Target {} Initialization failed:{}", target.id(), e);
continue;
}
debug!("Target {} initialized successfully,enabled:{}", target_id, target.is_enabled());
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start event stream processing for target {}", target.id());
// Initialize targets and start event streams
let cancellers = self.init_targets_and_start_streams(&targets).await;
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
let target_box = target.clone_dyn();
let target_arc = Arc::from(target_box);
// Add a reference to the monitoring metrics
let metrics = self.metrics.clone();
let semaphore = self.concurrency_limiter.clone();
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(store_clone, target_arc, metrics, semaphore);
// Start event stream processing and save cancel sender
let target_id_clone = target_id.clone();
cancellers.insert(target_id, cancel_tx);
info!("Event stream processing for target {} is started successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} is not enabled, event stream processing is skipped", target_id);
}
}
// Update canceler collection
// Update canceller collection
*self.stream_cancellers.write().await = cancellers;
// Initialize the bucket target
self.notifier.init_bucket_targets(targets).await?;
info!("Notification system initialized");
@@ -333,7 +376,7 @@ impl NotificationSystem {
info!("Attempting to remove target: {}", target_id);
let ttype = target_type.to_lowercase();
let tname = target_id.name.to_lowercase();
let tname = target_id.id.to_lowercase();
self.update_config_and_reload(|config| {
let mut changed = false;
@@ -405,11 +448,7 @@ impl NotificationSystem {
let ttype = target_type.to_lowercase();
let tname = target_name.to_lowercase();
let target_id = TargetID {
id: tname.clone(),
name: ttype.clone(),
};
let target_id = runtime_target_id_for_subsystem(&ttype, &tname);
// Deletion is prohibited if bucket rules refer to it
if self.notifier.is_target_bound_to_any_bucket(&target_id).await {
@@ -451,7 +490,7 @@ impl NotificationSystem {
/// Enhanced event stream startup function, including monitoring and concurrency control
fn enhanced_start_event_stream(
&self,
store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
@@ -476,14 +515,13 @@ impl NotificationSystem {
let _ = cancel_tx.send(()).await;
}
// Clear the target_list and ensure that reload is a replacement reconstruction (solve the target_list len unchanged/residual problem)
// Clear the target_list and ensure that reload is a replacement reconstruction
self.notifier.remove_all_bucket_targets().await;
// Update the config
self.update_config(new_config.clone()).await;
// Create a new target from configuration
// This function will now be responsible for merging env, creating and persisting the final configuration.
// Create new targets from configuration
let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
.registry
.create_targets_from_config(&new_config)
@@ -492,46 +530,8 @@ impl NotificationSystem {
info!("{} notification targets were created from the new configuration", targets.len());
// Start new event stream processing for each storage enabled target
let mut new_cancellers = HashMap::new();
for target in &targets {
let target_id = target.id();
// Initialize the target
if let Err(e) = target.init().await {
error!("Target {} Initialization failed:{}", target_id, e);
continue;
}
// Check if the target is enabled and has storage
if target.is_enabled() {
if let Some(store) = target.store() {
info!("Start new event stream processing for target {}", target_id);
// The storage of the cloned target and the target itself
let store_clone = store.boxed_clone();
// let target_box = target.clone_dyn();
let target_arc = Arc::from(target.clone_dyn());
// Encapsulated enhanced version of start_event_stream
let cancel_tx = self.enhanced_start_event_stream(
store_clone,
target_arc,
self.metrics.clone(),
self.concurrency_limiter.clone(),
);
// Start event stream processing and save cancel sender
// let cancel_tx = start_event_stream(store_clone, target_clone);
let target_id_clone = target_id.clone();
new_cancellers.insert(target_id, cancel_tx);
info!("Event stream processing of target {} is restarted successfully", target_id_clone);
} else {
info!("Target {} No storage is configured, event stream processing is skipped", target_id);
}
} else {
info!("Target {} disabled, event stream processing is skipped", target_id);
}
}
// Initialize targets and start event streams using shared helper
let new_cancellers = self.init_targets_and_start_streams(&targets).await;
// Update canceler collection
*cancellers = new_cancellers;
@@ -665,4 +665,18 @@ mod tests {
assert_eq!(batch.events.len(), 1);
assert_eq!(batch.events[0].s3.object.key, "one");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary");
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "webhook");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
assert_eq!(target_id.id, "analytics");
assert_eq!(target_id.name, "mqtt");
}
}
crates/notify/src/notifier.rs
+3 -3
View File
@@ -399,7 +399,7 @@ mod tests {
use rustfs_targets::{
TargetError,
store::{Key, Store},
target::EntityTarget,
target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
};
use serde::{Serialize, de::DeserializeOwned};
use std::sync::{
@@ -442,7 +442,7 @@ mod tests {
Ok(())
}
async fn send_from_store(&self, _key: Key) -> Result<(), TargetError> {
async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
Ok(())
}
@@ -450,7 +450,7 @@ mod tests {
Ok(())
}
fn store(&self) -> Option<&(dyn Store<EntityTarget<E>, Error = StoreError, Key = Key> + Send + Sync)> {
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
None
}
crates/notify/src/registry.rs
+5 -93
View File
@@ -89,9 +89,6 @@ impl TargetRegistry {
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
// A collection of asynchronous tasks for concurrently executing target creation
let mut tasks = FuturesUnordered::new();
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
// Record the defaults for each segment so that the segment can eventually be rebuilt
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
// 1. Traverse all registered plants and process them by target type
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
@@ -105,9 +102,6 @@ impl TargetRegistry {
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
debug!(?default_cfg, "Get the default configuration");
// Save defaults for eventual write back
section_defaults.insert(section_name.clone(), default_cfg.clone());
// *** Optimization point 1: Get all legitimate fields of the current target type ***
let valid_fields = factory.get_valid_fields();
debug!(?valid_fields, "Get the legitimate configuration fields");
@@ -215,110 +209,28 @@ impl TargetRegistry {
if enabled {
info!(instance_id = %id, "Target is enabled, ready to create a task");
// 5.3. Create asynchronous tasks for enabled instances
let target_type_clone = target_type.clone();
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(target_type_clone, tid, result, Arc::clone(&merged_config_arc))
(tid, result)
});
} else {
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
// Remove disabled target from final configuration
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
info!(instance_id = %id, "Skip disabled target");
}
}
}
// 6. Concurrently execute all creation tasks and collect results
let mut successful_targets = Vec::new();
let mut successful_configs = Vec::new();
while let Some((target_type, id, result, final_config)) = tasks.next().await {
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(target_type = %target_type, instance_id = %id, "Create a target successfully");
info!(instance_id = %id, "Create target successfully");
successful_targets.push(target);
successful_configs.push((target_type, id, final_config));
}
Err(e) => {
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create a target");
}
}
}
// 7. Aggregate new configuration and write back to system configuration
if !successful_configs.is_empty() || !section_defaults.is_empty() {
info!(
"Prepare to update {} successfully created target configurations to the system configuration...",
successful_configs.len()
);
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
for (target_type, id, kvs) in successful_configs {
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
successes_by_section
.entry(section_name)
.or_default()
.insert(id.to_lowercase(), (*kvs).clone());
}
let mut new_config = config.clone();
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
let mut sections: HashSet<String> = HashSet::new();
sections.extend(section_defaults.keys().cloned());
sections.extend(successes_by_section.keys().cloned());
for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item
if let Some(default_kvs) = section_defaults.get(&section)
&& !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
}
// Add successful instance item
if let Some(instances) = successes_by_section.get(&section) {
for (id, kvs) in instances {
section_map.insert(id.clone(), kvs.clone());
}
}
// Empty breaks are removed and non-empty breaks are replaced entirely.
if section_map.is_empty() {
new_config.0.remove(&section);
} else {
new_config.0.insert(section, section_map);
}
}
if &new_config == config {
info!("Notification target configuration unchanged, skip persisting server config");
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
let store = match rustfs_ecstore::global::new_object_layer_fn() {
Some(s) => s,
None => {
warn!(
"Object store not available at notification init; skipping config persistence. \
{} target(s) active in memory.",
successful_targets.len()
);
info!(count = successful_targets.len(), "All target processing completed");
return Ok(successful_targets);
}
};
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
Ok(_) => {
info!("The new configuration was saved to the system successfully.")
}
Err(e) => {
error!("Failed to save the new configuration: {}", e);
return Err(TargetError::SaveConfig(e.to_string()));
error!(instance_id = %id, error = %e, "Failed to create target");
}
}
}
crates/notify/src/stream.rs
+51 -66
View File
@@ -15,8 +15,8 @@
use crate::{Event, integration::NotificationMetrics};
use rustfs_targets::{
StoreError, Target, TargetError,
store::{Key, Store},
target::EntityTarget,
store::{Key, Store, ensure_store_entry_raw_readable},
target::QueuedPayload,
};
use rustfs_utils::get_env_usize;
use std::sync::Arc;
@@ -32,7 +32,7 @@ use tracing::{debug, error, info, warn};
/// - `target`: The target to send events to
/// - `cancel_rx`: Receiver to listen for cancellation signals
pub async fn stream_events(
store: &mut (dyn Store<Event, Error = StoreError, Key = Key> + Send),
store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
target: &dyn Target<Event>,
mut cancel_rx: mpsc::Receiver<()>,
) {
@@ -119,7 +119,7 @@ pub async fn stream_events(
/// # Returns
/// A sender to signal cancellation of the event stream
pub fn start_event_stream(
mut store: Box<dyn Store<Event, Error = StoreError, Key = Key> + Send>,
mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
) -> mpsc::Sender<()> {
let (cancel_tx, cancel_rx) = mpsc::channel(1);
@@ -143,7 +143,7 @@ pub fn start_event_stream(
/// # Returns
/// A sender to signal cancellation of the event stream
pub fn start_event_stream_with_batching(
mut store: Box<dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send>,
mut store: Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send>,
target: Arc<dyn Target<Event> + Send + Sync>,
metrics: Arc<NotificationMetrics>,
semaphore: Arc<Semaphore>,
@@ -170,7 +170,7 @@ pub fn start_event_stream_with_batching(
/// # Notes
/// This function processes events in batches to improve efficiency.
pub async fn stream_events_with_batching(
store: &mut (dyn Store<EntityTarget<Event>, Error = StoreError, Key = Key> + Send),
store: &mut (dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send),
target: &dyn Target<Event>,
mut cancel_rx: mpsc::Receiver<()>,
metrics: Arc<NotificationMetrics>,
@@ -185,7 +185,6 @@ pub async fn stream_events_with_batching(
const MAX_RETRIES: usize = 5;
const BASE_RETRY_DELAY: Duration = Duration::from_secs(2);
let mut batch: Vec<EntityTarget<Event>> = Vec::with_capacity(batch_size);
let mut batch_keys = Vec::with_capacity(batch_size);
let mut last_flush = Instant::now();
@@ -201,8 +200,8 @@ pub async fn stream_events_with_batching(
debug!("Found {} keys in store for target: {}", keys.len(), target.name());
if keys.is_empty() {
// If there is data in the batch and timeout, refresh the batch
if !batch.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
@@ -218,41 +217,31 @@ pub async fn stream_events_with_batching(
info!("Cancellation received during processing for target: {}", target.name());
// Processing collected batches before exiting
if !batch.is_empty() {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
if !batch_keys.is_empty() {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
}
return;
}
// Try to get events from storage
match store.get(&key) {
Ok(event) => {
// Add to batch
batch.push(event);
batch_keys.push(key);
metrics.increment_processing();
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch, &mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore)
.await;
last_flush = Instant::now();
}
// Skip unreadable entries so a single corrupt file cannot stall the stream.
// ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to
// remove the corrupt entry before returning Err, so no cleanup is needed here.
match ensure_store_entry_raw_readable(&*store, &key) {
Ok(true) => {} // entry is readable, proceed
Ok(false) => continue, // entry not found (already removed), skip
Err(err) => {
warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err);
continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable
}
Err(e) => {
error!("Failed to target: {}, get event {} from store: {}", target.name(), key.to_string(), e);
// Consider deleting unreadable events to prevent infinite loops from trying to read
match store.del(&key) {
Ok(_) => {
info!("Deleted corrupted event {} from store", key.to_string());
}
Err(del_err) => {
error!("Failed to delete corrupted event {}: {}", key.to_string(), del_err);
}
}
}
metrics.increment_failed();
}
batch_keys.push(key);
metrics.increment_processing();
// If the batch is full or enough time has passed since the last refresh, the batch will be processed
if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT {
process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await;
last_flush = Instant::now();
}
}
@@ -273,7 +262,6 @@ pub async fn stream_events_with_batching(
/// # Notes
/// This function processes a batch of events, sending each event to the target with retry
async fn process_batch(
batch: &mut Vec<EntityTarget<Event>>,
batch_keys: &mut Vec<Key>,
target: &dyn Target<Event>,
max_retries: usize,
@@ -281,8 +269,8 @@ async fn process_batch(
metrics: &Arc<NotificationMetrics>,
semaphore: &Arc<Semaphore>,
) {
debug!("Processing batch of {} events for target: {}", batch.len(), target.name());
if batch.is_empty() {
debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name());
if batch_keys.is_empty() {
return;
}
@@ -296,44 +284,42 @@ async fn process_batch(
};
// Handle every event in the batch
for (_event, key) in batch.iter().zip(batch_keys.iter()) {
for key in batch_keys.iter() {
let mut retry_count = 0;
let mut success = false;
// Retry logic
while retry_count < max_retries && !success {
// After sending successfully, the event in the storage is deleted synchronously.
match target.send_from_store(key.clone()).await {
Ok(_) => {
info!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string());
success = true;
metrics.increment_processed();
}
Err(e) => {
// Different retry strategies are adopted according to the error type
match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await; // Exponential backoff
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
tokio::time::sleep(base_delay * (1 << retry_count)).await;
}
_ => {
// Permanent error, skip this event
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
}
Err(e) => match &e {
TargetError::NotConnected => {
warn!("Target {} not connected, retrying...", target.name());
retry_count += 1;
let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
let backoff = 1u32 << retry_count as u32;
tokio::time::sleep(base_delay * backoff + jitter).await;
}
}
TargetError::Timeout(_) => {
warn!("Timeout for target {}, retrying...", target.name());
retry_count += 1;
let jitter = Duration::from_millis(key.to_string().len() as u64 % 500);
let backoff = 1u32 << retry_count as u32;
tokio::time::sleep(base_delay * backoff + jitter).await;
}
_ => {
error!("Permanent error for target {}: {}", target.name(), e);
metrics.increment_failed();
break;
}
},
}
}
// Handle the situation where the maximum number of retry exhaustion is exhausted
if retry_count >= max_retries && !success {
warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name());
metrics.increment_failed();
@@ -341,7 +327,6 @@ async fn process_batch(
}
// Clear processed batches
batch.clear();
batch_keys.clear();
// Release semaphore permission (via drop)