fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

crates/targets/src/target/kafka.rs

@@ -24,7 +24,7 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
         open_target_queue_store, persist_queued_payload_to_store,
     },
 };
@@ -32,13 +32,84 @@ use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
 use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
-use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};
 
 pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
+const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct KafkaDeliveryAttempt<'a> {
+    armed: bool,
+    poisoned: &'a AtomicBool,
+}
+
+impl KafkaDeliveryAttempt<'_> {
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for KafkaDeliveryAttempt<'_> {
+    fn drop(&mut self) {
+        if self.armed {
+            self.poisoned.store(true, Ordering::Release);
+        }
+    }
+}
+
+fn kafka_delivery_timeout() -> TargetError {
+    TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
+}
+
+async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
+    delivery_lock: &Mutex<()>,
+    delivery_poisoned: &AtomicBool,
+    select_producer: Select,
+    deliver: Deliver,
+    invalidate: Invalidate,
+) -> Result<T, TargetError>
+where
+    P: Send,
+    T: Send,
+    Select: FnOnce() -> SelectFuture + Send,
+    SelectFuture: Future<Output = Result<P, TargetError>> + Send,
+    Deliver: FnOnce(P) -> DeliveryFuture + Send,
+    DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
+    Invalidate: Fn() -> InvalidateFuture + Send,
+    InvalidateFuture: Future<Output = ()> + Send,
+{
+    let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
+    let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    let mut attempt = KafkaDeliveryAttempt {
+        armed: true,
+        poisoned: delivery_poisoned,
+    };
+
+    if delivery_poisoned.load(Ordering::Acquire) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+
+    let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    if result.as_ref().is_err_and(is_connectivity_error) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+    attempt.disarm();
+    result
+}
 
 /// Arguments for configuring a Kafka target
 #[derive(Clone)]
@@ -233,6 +304,8 @@ where
     args: KafkaArgs,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_lock: Arc<Mutex<()>>,
+    delivery_poisoned: Arc<AtomicBool>,
     tls_state: Arc<Mutex<TargetTlsState>>,
     /// Adapter that bridges this target to the TLS reload coordinator.
     /// When `Some`, the target uses coordinator-managed material; when `None`,
@@ -291,6 +364,8 @@ where
             args,
             store: queue_store,
             producer: Arc::new(Mutex::new(None)),
+            delivery_lock: Arc::new(Mutex::new(())),
+            delivery_poisoned: Arc::new(AtomicBool::new(false)),
             tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
             tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -307,7 +382,7 @@ where
         };
 
         let mut config = AsyncProducerConfig::new()
-            .with_ack_timeout(Duration::from_secs(30))
+            .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
             .with_required_acks(acks);
 
         if let Some(security) = self.args.security_config(true)? {
@@ -388,20 +463,26 @@ where
             "Sending Kafka payload"
         );
 
-        let producer = self.get_or_build_producer().await?;
-
-        // Use "<bucket>/<object>" as the message key so all events for the same
-        // object hash to the same partition and preserve per-object ordering
-        // across multiple partitions (backlog#983).
-        let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
-        if let Err(err) = producer
-            .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
-            .await
-        {
-            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
-            return Err(mapped);
-        }
+        // rustfs-kafka-async does not validate response correlation IDs. Keep
+        // producer selection, send, and timeout invalidation serialized so a
+        // waiter cannot reuse a connection with an unread timed-out response.
+        with_serialized_kafka_delivery(
+            &self.delivery_lock,
+            &self.delivery_poisoned,
+            || self.get_or_build_producer(),
+            |producer| async move {
+                // Use "<bucket>/<object>" as the message key so all events for the same
+                // object hash to the same partition and preserve per-object ordering
+                // across multiple partitions (backlog#983).
+                let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
+                producer
+                    .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
+                    .await
+                    .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
+            },
+            || self.invalidate_cached_producer(),
+        )
+        .await?;
 
         debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
         self.delivery_counters.record_success();
@@ -415,6 +496,8 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             producer: Arc::clone(&self.producer),
+            delivery_lock: Arc::clone(&self.delivery_lock),
+            delivery_poisoned: Arc::clone(&self.delivery_poisoned),
             tls_state: Arc::clone(&self.tls_state),
             tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -559,6 +642,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::atomic::AtomicUsize;
+    use tokio::sync::Notify;
 
     fn base_args() -> KafkaArgs {
         KafkaArgs {
@@ -580,6 +665,158 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    move |selected| async move {
+                        assert_eq!(selected, 1);
+                        first_entered.notify_one();
+                        std::future::pending::<Result<usize, TargetError>>().await
+                    },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        first_entered.notified().await;
+        tokio::time::advance(Duration::from_secs(1)).await;
+        let second = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    |selected| async move { Ok(selected) },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        assert!(matches!(
+            first.await.expect("first delivery task should not panic"),
+            Err(TargetError::Timeout(_))
+        ));
+        assert_eq!(
+            second
+                .await
+                .expect("second delivery task should not panic")
+                .expect("second delivery should succeed"),
+            2,
+            "the waiter must select a fresh producer generation after timeout invalidation"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = AtomicBool::new(false);
+        let selected = Arc::new(AtomicBool::new(false));
+        let _held = delivery_lock.lock().await;
+
+        let error = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let selected = Arc::clone(&selected);
+                move || async move {
+                    selected.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            },
+            |()| async { Ok(()) },
+            || async {},
+        )
+        .await
+        .expect_err("lock admission must share the absolute delivery deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
+        assert!(!delivery_poisoned.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    || async { Ok(1usize) },
+                    move |_| async move {
+                        first_entered.notify_one();
+                        std::future::pending::<Result<(), TargetError>>().await
+                    },
+                    || async {},
+                )
+                .await
+            })
+        };
+        first_entered.notified().await;
+        first.abort();
+        assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
+        assert!(delivery_poisoned.load(Ordering::Acquire));
+
+        let selected = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let generation = Arc::clone(&generation);
+                move || async move { Ok(generation.load(Ordering::SeqCst)) }
+            },
+            |selected| async move { Ok(selected) },
+            {
+                let generation = Arc::clone(&generation);
+                move || {
+                    let generation = Arc::clone(&generation);
+                    async move { generation.store(2, Ordering::SeqCst) }
+                }
+            },
+        )
+        .await
+        .expect("the next delivery should recover from cancellation poisoning");
+
+        assert_eq!(selected, 2);
+        assert!(!delivery_poisoned.load(Ordering::Acquire));
+    }
+
     #[test]
     fn test_validate_empty_brokers() {
         let args = KafkaArgs {

crates/targets/src/target/mod.rs

@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
+use std::cell::Cell;
 use std::fmt::Formatter;
 use std::future::Future;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::thread_local;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};
 
 pub mod amqp;
@@ -111,7 +113,12 @@ where
     /// Checks if the target is active and reachable
     async fn is_active(&self) -> Result<bool, TargetError>;
 
-    /// Saves an event (either sends it immediately or stores it for later)
+    /// Saves an event (either sends it immediately or stores it for later).
+    ///
+    /// A target whose [`Self::store`] returns `Some` must only persist the event
+    /// here; network delivery belongs to its replay worker. Runtime lifecycle
+    /// handoff drains these durable enqueues while allowing a direct network
+    /// send to finish against a detached target.
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
 
     /// Sends an event from the store using the queued raw body and metadata.
@@ -605,6 +612,24 @@ pub(crate) fn open_target_queue_store(
     Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
 }
 
+thread_local! {
+    static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
+}
+
+pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
+    struct Reset(bool);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
+        }
+    }
+
+    let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
+    let _reset = Reset(previous);
+    operation()
+}
+
 /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
 /// failed-store capability holds it directly rather than through the type-erased Store handle.
 pub(crate) fn open_target_queue_store_typed(
@@ -625,9 +650,11 @@ pub(crate) fn open_target_queue_store_typed(
         TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
     };
     let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
-    store
-        .open()
-        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
+        store
+            .open()
+            .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    }
 
     Ok(Some(store))
 }
@@ -649,6 +676,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
     matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
 }
 
+/// Applies an absolute deadline to one protocol delivery attempt.
+///
+/// Target clients expose different timeout controls, and several of them only
+/// place a timeout value in the wire request without bounding the local socket
+/// future. Keeping the outer deadline here gives every caller the same typed,
+/// retryable timeout without changing the target-specific error mapping.
+pub(crate) async fn with_delivery_deadline<T, F>(
+    deadline: Duration,
+    operation: &'static str,
+    delivery: F,
+) -> Result<T, TargetError>
+where
+    F: Future<Output = Result<T, TargetError>>,
+{
+    match tokio::time::timeout(deadline, delivery).await {
+        Ok(result) => result,
+        Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
+    }
+}
+
 pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
 where
     F: FnOnce() -> Fut,
@@ -1137,6 +1184,29 @@ mod tests {
         let _ = fs::remove_file(base);
     }
 
+    #[test]
+    fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let store = with_deferred_queue_store_open(|| {
+            open_target_queue_store(
+                base.to_str().unwrap(),
+                100,
+                TargetType::NotifyEvent,
+                ChannelTargetType::Kafka.as_str(),
+                &target_id,
+                "deferred open",
+            )
+        })
+        .expect("deferred construction must not open the queue directory")
+        .expect("non-empty queue directory should create a dormant store");
+
+        assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
+        let _ = fs::remove_file(base);
+    }
+
     #[test]
     fn persist_queued_payload_to_store_writes_encoded_payload() {
         let store = MockQueuedStore::new(false);
@@ -1182,6 +1252,19 @@ mod tests {
         assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
+        let error = with_delivery_deadline(
+            Duration::from_secs(30),
+            "test delivery",
+            std::future::pending::<Result<(), TargetError>>(),
+        )
+        .await
+        .expect_err("a stalled delivery must hit its hard deadline");
+
+        assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
+    }
+
     #[tokio::test]
     async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
         let marker = Arc::new(AtomicBool::new(false));

crates/targets/src/target/mqtt.rs

@@ -765,11 +765,6 @@ where
 
     #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
     async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
         debug!(
             event = EVENT_MQTT_DELIVERY_STATE,
             component = LOG_COMPONENT_TARGETS,
@@ -790,10 +785,22 @@ where
         // silently dropped the event while its durable copy was already deleted
         // (backlog#971). Error classification now matches on the typed error
         // instead of substring matching on the display string.
-        let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
-            Ok(notice) => notice,
-            Err(e) => {
-                let err = classify_mqtt_client_error(&e);
+        let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
+            let client_guard = self.client.lock().await;
+            let client = client_guard
+                .as_ref()
+                .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+            let notice = client
+                .publish_tracked(&self.args.topic, self.args.qos, false, body)
+                .await
+                .map_err(|error| classify_mqtt_client_error(&error))?;
+            drop(client_guard);
+            Ok(notice)
+        })
+        .await
+        {
+            Ok(Ok(notice)) => notice,
+            Ok(Err(err)) => {
                 warn!(
                     event = EVENT_MQTT_DELIVERY_STATE,
                     component = LOG_COMPONENT_TARGETS,
@@ -801,18 +808,29 @@ where
                     target_id = %self.id,
                     state = "publish_failed",
                     reason = "enqueue_error",
-                    error = %e,
+                    error = %err,
                     "mqtt delivery state"
                 );
                 mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                 return Err(err);
             }
+            Err(_) => {
+                warn!(
+                    event = EVENT_MQTT_DELIVERY_STATE,
+                    component = LOG_COMPONENT_TARGETS,
+                    subsystem = LOG_SUBSYSTEM_MQTT,
+                    target_id = %self.id,
+                    state = "publish_failed",
+                    reason = "enqueue_timeout",
+                    "mqtt delivery state"
+                );
+                // Admission can time out because the local bounded request
+                // channel is full while the MQTT session remains connected.
+                // Only protocol/client failures are evidence of disconnect.
+                return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
+            }
         };
 
-        // Release the client lock before awaiting the broker acknowledgement so a
-        // slow/hung broker never blocks other senders from queueing publishes.
-        drop(client_guard);
-
         match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
             Ok(Ok(())) => {
                 debug!(
@@ -1708,9 +1726,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
-        classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
-        validate_mqtt_broker_url,
+        AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
+        MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
+        next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
     };
     use crate::error::TargetError;
     use crate::target::{REDACTED_SECRET, TargetType};
@@ -1720,6 +1738,23 @@ mod tests {
     use tokio::sync::mpsc;
     use url::Url;
 
+    fn base_mqtt_args() -> MQTTArgs {
+        MQTTArgs {
+            enable: true,
+            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
+            topic: "rustfs/events".to_string(),
+            qos: QoS::AtLeastOnce,
+            username: String::new(),
+            password: String::new(),
+            tls: MQTTTlsConfig::default(),
+            max_reconnect_interval: Duration::from_secs(1),
+            keep_alive: Duration::from_secs(30),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
     #[test]
     fn mqtt_client_error_classified_as_not_connected() {
         // A publish that cannot be handed to the event loop means the client is
@@ -1752,6 +1787,38 @@ mod tests {
         assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn enqueue_timeout_keeps_a_live_session_connected() {
+        let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
+        let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
+            .capacity(1)
+            .build();
+        client
+            .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
+            .await
+            .expect("first publish should fill the local channel");
+        *target.client.lock().await = Some(client);
+        target.connected.store(true, Ordering::SeqCst);
+        let meta = QueuedPayloadMeta::new(
+            rustfs_s3_types::EventName::ObjectCreatedPut,
+            "bucket".to_string(),
+            "object".to_string(),
+            "application/json",
+            2,
+        );
+
+        let error = target
+            .send_body(b"{}".to_vec(), &meta)
+            .await
+            .expect_err("a full local request channel should hit the enqueue deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(
+            target.connected.load(Ordering::SeqCst),
+            "local admission pressure is not evidence that the MQTT session disconnected"
+        );
+    }
+
     #[test]
     fn next_reconnect_backoff_doubles_until_capped() {
         let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
@@ -1864,21 +1931,13 @@ mod tests {
     #[test]
     fn debug_redacts_mqtt_secret_fields() {
         let args = MQTTArgs {
-            enable: true,
-            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
-            topic: "rustfs/events".to_string(),
-            qos: QoS::AtLeastOnce,
             username: "mqtt-user".to_string(),
             password: "mqtt-password".to_string(),
             tls: MQTTTlsConfig {
                 client_key_path: "/etc/rustfs/mqtt.key".to_string(),
                 ..MQTTTlsConfig::default()
             },
-            max_reconnect_interval: Duration::from_secs(1),
-            keep_alive: Duration::from_secs(30),
-            queue_dir: String::new(),
-            queue_limit: 0,
-            target_type: TargetType::NotifyEvent,
+            ..base_mqtt_args()
         };
 
         let rendered = format!("{args:?}");

crates/targets/src/target/mysql.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -47,6 +47,8 @@ use uuid::Uuid;
 /// `TargetError::Timeout`, a connectivity error, so the payload stays queued
 /// for replay.
 const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
+/// Absolute ceiling for one INSERT, including pool checkout and server execution.
+const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Name of the optional idempotency-key column / primary key. Present on tables
 /// created by this target; absent on legacy two-column tables.
@@ -784,29 +786,33 @@ where
             "Inserting MySQL event"
         );
 
-        let pool = self.get_or_init_pool().await?;
-        // At this point the pool has already been initialized (get_or_init_pool
-        // succeeded above), so get_conn() failures are always transient: the
-        // connection was lost or the pool is temporarily exhausted.
-        let mut conn = checkout_conn(&pool).await?;
-
         let event_time = extract_event_time(body)?;
         let event_data =
             std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;
 
         let quoted_table = quote_table_name(&self.args.table)?;
+        with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
+            let pool = self.get_or_init_pool().await?;
+            // At this point the pool has already been initialized (get_or_init_pool
+            // succeeded above), so get_conn() failures are always transient: the
+            // connection was lost or the pool is temporarily exhausted.
+            let mut conn = checkout_conn(&pool).await?;
 
-        if self.idempotency_supported.load(Ordering::Relaxed) {
-            let sql = mysql_insert_sql_with_event_id(&quoted_table);
-            conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        } else {
-            let sql = mysql_insert_sql_legacy(&quoted_table);
-            conn.exec_drop(sql, (event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        }
+            if self.idempotency_supported.load(Ordering::Relaxed) {
+                let sql = mysql_insert_sql_with_event_id(&quoted_table);
+                conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            } else {
+                let sql = mysql_insert_sql_legacy(&quoted_table);
+                conn.exec_drop(sql, (event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            }
+
+            Ok(())
+        })
+        .await?;
 
         self.delivery_counters.record_success();
         debug!(target_id = %self.id, "MySQL event inserted");

crates/targets/src/target/nats/mod.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
-        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
+        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
 pub(crate) use jetstream::resolve_dedup_id;
 pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
 
+const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
 #[derive(Clone)]
 pub struct NATSArgs {
     pub enable: bool,
@@ -397,9 +399,21 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        let client = self.get_or_connect().await?;
-        if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
-            let err = classify_nats_publish_error(&e);
+        let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
+            let client = self.get_or_connect().await?;
+            client
+                .publish(self.args.subject.clone(), body.into())
+                .await
+                .map_err(|err| classify_nats_publish_error(&err))?;
+
+            // publish only enqueues the message on the client's outbound channel. Flush to confirm the
+            // message reached the server before delivery is treated as successful (backlog#971).
+            client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = result {
             if is_connectivity_error(&err) {
                 self.invalidate_cached_client_connection().await;
                 self.connected.store(false, Ordering::SeqCst);
@@ -407,15 +421,6 @@ where
             return Err(err);
         }
 
-        // publish only enqueues the message on the client's outbound channel. Flush to confirm the
-        // message reached the server before delivery is treated as successful (backlog#971).
-        if let Err(e) = client.flush().await {
-            let err = classify_nats_flush_error(&e);
-            self.invalidate_cached_client_connection().await;
-            self.connected.store(false, Ordering::SeqCst);
-            return Err(err);
-        }
-
         self.delivery_counters.record_success();
         Ok(())
     }

crates/targets/src/target/postgres.rs

@@ -38,7 +38,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
-        redacted_secret,
+        redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
 /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
 /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
 const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
+/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
+const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Returns `true` for any `s3:ObjectRemoved:*` event.
 ///
@@ -730,30 +732,29 @@ where
 
         let key = resolve_payload_key(&payload, meta);
 
-        let result = match self.args.format {
-            // For the single-row `namespace` format, an object removal must
-            // delete the row rather than UPSERT it, otherwise stale state
-            // lingers in the table after the object is gone.
-            PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
-                client.execute(&self.namespace_delete_sql, &[&key]).await
+        with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
+            match self.args.format {
+                // For the single-row `namespace` format, an object removal must
+                // delete the row rather than UPSERT it, otherwise stale state
+                // lingers in the table after the object is gone.
+                PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
+                    client.execute(&self.namespace_delete_sql, &[&key]).await
+                }
+                PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
+                PostgresFormat::Access => {
+                    let event_name_str = meta.event_name.to_string();
+                    let queued_at_ms = meta.queued_at_unix_ms as i64;
+                    client
+                        .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
+                        .await
+                }
             }
-            PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
-            PostgresFormat::Access => {
-                let event_name_str = meta.event_name.to_string();
-                let queued_at_ms = meta.queued_at_unix_ms as i64;
-                client
-                    .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
-                    .await
-            }
-        };
+            .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
+        })
+        .await?;
 
-        match result {
-            Ok(_) => {
-                self.delivery_counters.record_success();
-                Ok(())
-            }
-            Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
-        }
+        self.delivery_counters.record_success();
+        Ok(())
     }
 
     /// Probes the table from `init()`. Failure is non-fatal when a queue is

crates/targets/src/target/pulsar.rs

@@ -24,8 +24,9 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -39,11 +40,15 @@ use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;
 
+const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
+
 #[derive(Clone)]
 pub struct PulsarArgs {
     pub enable: bool,
@@ -276,6 +281,23 @@ where
         self.tls_state.lock().reset();
     }
 
+    async fn clear_failed_delivery_state(&self) {
+        match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
+            Ok(mut producer) => {
+                producer.take();
+            }
+            Err(_) => {
+                warn!(
+                    target_id = %self.id,
+                    reason = "producer_cleanup_lock_timeout",
+                    "Timed out clearing the Pulsar producer after a failed delivery"
+                );
+            }
+        }
+        self.clear_cached_client();
+        self.connected.store(false, Ordering::SeqCst);
+    }
+
     async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
         // When a TLS reload adapter is attached, it drives client rebuilds
         // in the background. The inline per-send fingerprint check is skipped.
@@ -334,20 +356,30 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        self.init_producer().await?;
-        let mut guard = self.producer.lock().await;
-        let producer = guard
-            .as_mut()
-            .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
-        let receipt = producer
-            .send_non_blocking(body)
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
-        receipt
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
-        self.delivery_counters.record_success();
-        Ok(())
+        let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
+            self.init_producer().await?;
+            let mut guard = self.producer.lock().await;
+            let producer = guard
+                .as_mut()
+                .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
+            let receipt = producer
+                .send_non_blocking(body)
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
+            receipt
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
+            self.delivery_counters.record_success();
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = &result
+            && is_connectivity_error(err)
+        {
+            self.clear_failed_delivery_state().await;
+        }
+        result
     }
 }
 
@@ -525,6 +557,22 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
+        let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
+        target.connected.store(true, Ordering::SeqCst);
+        let producer_guard = target.producer.lock().await;
+        let cleanup = {
+            let target = Arc::clone(&target);
+            tokio::spawn(async move { target.clear_failed_delivery_state().await })
+        };
+
+        cleanup.await.expect("cleanup task should not panic");
+
+        assert!(!target.connected.load(Ordering::SeqCst));
+        drop(producer_guard);
+    }
+
     #[test]
     fn debug_redacts_pulsar_secret_fields() {
         let args = PulsarArgs {

crates/targets/src/target/redis.rs

@@ -26,6 +26,7 @@ use crate::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
         mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
 use tracing::{debug, info, instrument, warn};
 use url::Url;
 
+const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+
+fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
+    let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
+    let per_attempt = args
+        .connection_timeout
+        .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
+        .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
+        .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
+    per_attempt.saturating_mul(attempts)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RedisTlsPolicy {
     SystemCa,
@@ -214,6 +228,16 @@ impl RedisArgs {
             ));
         }
 
+        if self.connection_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration(
+                "Redis connection_timeout must be greater than zero".to_string(),
+            ));
+        }
+
+        if self.response_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
+        }
+
         if self.pipeline_buffer_size == Some(0) {
             return Err(TargetError::Configuration(
                 "Redis pipeline_buffer_size must be greater than zero".to_string(),
@@ -464,71 +488,97 @@ where
             "Sending Redis payload"
         );
 
-        let mut attempt = 0usize;
-        let mut last_error = None;
-        while attempt < self.args.max_retry_attempts {
-            attempt += 1;
+        let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
+            let mut attempt = 0usize;
+            let mut last_error = None;
+            while attempt < self.args.max_retry_attempts {
+                attempt += 1;
 
-            let mut publisher = self.get_or_create_publisher().await?;
-            match publisher
-                .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
+                let mut publisher = match with_delivery_deadline(
+                    connection_timeout,
+                    "Redis connection",
+                    self.get_or_create_publisher(),
+                )
                 .await
-            {
-                Ok(receiver_count) => {
-                    // PUBLISH returns the number of subscribers that received the
-                    // message. Redis pub/sub is best-effort: with zero subscribers
-                    // the event is delivered to no one, yet the durable copy is
-                    // deleted. Warn so operators relying on reliable delivery are
-                    // not silently losing events (backlog#982).
-                    if receiver_count == 0 {
+                {
+                    Ok(publisher) => publisher,
+                    Err(err) => {
+                        invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                        return Err(err);
+                    }
+                };
+                let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
+                match with_delivery_deadline(response_timeout, "Redis publish response", async {
+                    publisher
+                        .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                        .await
+                        .map_err(map_redis_error)
+                })
+                .await
+                {
+                    Ok(receiver_count) => {
+                        // PUBLISH returns the number of subscribers that received the
+                        // message. Redis pub/sub is best-effort: with zero subscribers
+                        // the event is delivered to no one, yet the durable copy is
+                        // deleted. Warn so operators relying on reliable delivery are
+                        // not silently losing events (backlog#982).
+                        if receiver_count == 0 {
+                            warn!(
+                                target_id = %self.id,
+                                channel = %self.args.channel,
+                                "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            );
+                        }
+                        debug!(
+                            target_id = %self.id,
+                            channel = %self.args.channel,
+                            attempt,
+                            receiver_count,
+                            "Event published to Redis channel"
+                        );
+                        self.delivery_counters.record_success();
+                        return Ok(());
+                    }
+                    Err(mapped) => {
+                        invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
+
                         warn!(
                             target_id = %self.id,
                             channel = %self.args.channel,
-                            "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            attempt,
+                            max_attempts = self.args.max_retry_attempts,
+                            error = %mapped,
+                            "Redis publish attempt failed"
                         );
-                    }
-                    debug!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        receiver_count,
-                        "Event published to Redis channel"
-                    );
-                    self.delivery_counters.record_success();
-                    return Ok(());
-                }
-                Err(err) => {
-                    let mapped = map_redis_error(err);
-                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
 
-                    warn!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        max_attempts = self.args.max_retry_attempts,
-                        error = %mapped,
-                        "Redis publish attempt failed"
-                    );
+                        if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                            last_error = Some(mapped);
+                            break;
+                        }
 
-                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                         last_error = Some(mapped);
-                        break;
+                        tokio::time::sleep(compute_retry_delay(
+                            attempt,
+                            self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
+                            self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
+                        ))
+                        .await;
                     }
-
-                    last_error = Some(mapped);
-                    tokio::time::sleep(compute_retry_delay(
-                        attempt,
-                        self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
-                        self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
-                    ))
-                    .await;
                 }
             }
+
+            Err(last_error.unwrap_or(TargetError::Unknown(
+                "Redis publish failed without a captured error".to_string(),
+            )))
+        })
+        .await;
+
+        if let Err(err) = &result {
+            invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
+            self.connected.store(false, Ordering::SeqCst);
         }
-
-        self.connected.store(false, Ordering::SeqCst);
-
-        Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
+        result
     }
 }
 
@@ -551,7 +601,7 @@ where
         // thus a fresh TCP+TLS handshake — on every health check (backlog#982).
         // ensure_publisher_ready already invalidates the cached manager on a
         // connectivity error so the next attempt rebuilds it.
-        match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
+        match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
             Ok(Ok(())) => {
                 self.connected.store(true, Ordering::SeqCst);
                 Ok(true)
@@ -917,6 +967,26 @@ mod tests {
         assert!(args.validate().is_err());
     }
 
+    #[test]
+    fn validate_rejects_zero_connection_timeout() {
+        let args = RedisArgs {
+            connection_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_response_timeout() {
+        let args = RedisArgs {
+            response_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
     #[test]
     fn validate_accepts_custom_ca_tls_policy() {
         let args = RedisArgs {
@@ -1237,6 +1307,23 @@ mod tests {
         assert_eq!(target.delivery_snapshot().total_messages, 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
+        let mut args = base_args();
+        args.max_retry_attempts = 1;
+        args.response_timeout = Some(Duration::from_secs(90));
+        let manager_config = build_redis_connection_manager_config(&args);
+
+        assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
+        assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
+        with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
+            tokio::time::sleep(Duration::from_secs(70)).await;
+            Ok::<_, TargetError>(())
+        })
+        .await
+        .expect("the configured delivery budget must not impose a fixed sixty-second cap");
+    }
+
     #[tokio::test]
     async fn send_body_sets_connected_false_after_retry_exhaustion() {
         let listener = match TcpListener::bind("127.0.0.1:0").await {

crates/targets/src/target/webhook.rs

@@ -952,7 +952,7 @@ mod tests {
             .expect("https webhook probe should trust configured ca");
 
         assert_eq!(resp.status(), reqwest::StatusCode::OK);
-        assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
+        assert!(resp.bytes().await.expect("read response body").is_empty());
         handle.join().expect("tls server thread");
     }
 }