fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

crates/targets/src/config/common.rs

@@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<St
         .max_by_key(|(field, _)| field.len())
 }
 
-pub(super) fn is_target_enabled(config: &KVS) -> bool {
-    config
-        .lookup(ENABLE_KEY)
-        .map(|v| {
-            EnableState::from_str(v.as_str())
-                .ok()
-                .map(|s| s.is_enabled())
-                .unwrap_or(false)
-        })
-        .unwrap_or(false)
+pub(super) fn is_target_enabled(config: &KVS) -> Result<bool, TargetError> {
+    let Some(value) = config.lookup(ENABLE_KEY) else {
+        return Ok(false);
+    };
+    EnableState::from_str(value.as_str())
+        .map(EnableState::is_enabled)
+        .map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'")))
 }
 
 pub(super) fn parse_target_bool(value: Option<&str>) -> Option<bool> {

crates/targets/src/config/instance.rs

@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::loader::collect_merged_target_configs_from_env;
+use super::loader::{
+    MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env,
+};
+use crate::TargetError;
 use crate::domain::TargetDomain;
 use rustfs_config::server_config::{Config, KVS};
 use std::collections::HashSet;
@@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances(
     normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
 }
 
+pub fn try_normalize_target_plugin_instances(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError> {
+    try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
+}
+
 pub fn normalize_target_plugin_instances_from_env<I>(
     config: &Config,
     descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
@@ -100,7 +110,7 @@ where
         .map(|field| (*field).to_string())
         .collect::<HashSet<_>>();
 
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         descriptor.subsystem,
         descriptor.route_prefix,
@@ -109,7 +119,42 @@ where
         env_vars,
     )
     .into_iter()
-    .map(|record| TargetPluginInstanceRecord {
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect()
+}
+
+pub fn try_normalize_target_plugin_instances_from_env<I>(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    env_vars: I,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let valid_fields = descriptor
+        .valid_fields
+        .iter()
+        .map(|field| (*field).to_string())
+        .collect::<HashSet<_>>();
+
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        descriptor.subsystem,
+        descriptor.route_prefix,
+        descriptor.target_type,
+        &valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect())
+}
+
+fn target_plugin_instance_record(
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    record: MergedTargetConfigRecord,
+) -> TargetPluginInstanceRecord {
+    TargetPluginInstanceRecord {
         domain: descriptor.domain,
         plugin_id: descriptor.plugin_id.to_string(),
         target_type: descriptor.target_type.to_string(),
@@ -123,8 +168,7 @@ where
             has_env_instance: record.has_env_instance,
         },
         effective_config: record.effective_config,
-    })
-    .collect()
+    }
 }
 
 pub fn normalize_legacy_target_instances(
@@ -149,8 +193,9 @@ where
 mod tests {
     use super::{
         TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
-        normalize_target_plugin_instances_from_env,
+        try_normalize_target_plugin_instances_from_env,
     };
+    use crate::TargetError;
     use crate::domain::TargetDomain;
     use crate::manifest::builtin_target_manifest;
     use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -338,9 +383,50 @@ mod tests {
         let descriptor = notify_webhook_descriptor();
         let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];
 
-        let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
+        let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone())
+            .expect("canonical normalization should succeed");
         let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);
 
         assert_eq!(canonical, compatibility);
     }
+
+    #[test]
+    fn normalize_instances_rejects_invalid_enable_value() {
+        let error = try_normalize_target_plugin_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must be propagated by the public normalizer");
+
+        match error {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() {
+        let instances = normalize_legacy_target_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(instances.len(), 2);
+        let good = instances
+            .iter()
+            .find(|instance| instance.instance_id == "good")
+            .expect("valid instance should remain present");
+        let bad = instances
+            .iter()
+            .find(|instance| instance.instance_id == "bad")
+            .expect("invalid legacy instance should remain visible");
+        assert!(good.enabled);
+        assert!(!bad.enabled);
+    }
 }

crates/targets/src/config/loader.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::common::{is_target_enabled, split_env_field_and_instance};
+use crate::TargetError;
 use rustfs_config::server_config::{Config, KVS};
 use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
 use std::collections::{HashMap, HashSet};
@@ -27,6 +28,15 @@ pub fn collect_target_configs(
     collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
 }
 
+pub fn try_collect_target_configs(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+) -> Result<Vec<(String, KVS)>, TargetError> {
+    try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
+}
+
 fn is_sensitive_target_field(field_name: &str) -> bool {
     let field_name = field_name.to_ascii_lowercase();
     field_name.contains("password")
@@ -123,7 +133,7 @@ pub fn collect_target_configs_from_env<I>(
 where
     I: IntoIterator<Item = (String, String)>,
 {
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         &format!("{route_prefix}{target_type}").to_lowercase(),
         route_prefix,
@@ -137,6 +147,30 @@ where
     .collect()
 }
 
+pub fn try_collect_target_configs_from_env<I>(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Result<Vec<(String, KVS)>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .filter(|record| record.enabled)
+    .map(|record| (record.instance_id, record.effective_config))
+    .collect())
+}
+
 pub(crate) fn collect_merged_target_configs_from_env<I>(
     config: &Config,
     section_name: &str,
@@ -144,7 +178,52 @@ pub(crate) fn collect_merged_target_configs_from_env<I>(
     target_type: &str,
     valid_fields: &HashSet<String>,
     env_vars: I,
+) -> Result<Vec<MergedTargetConfigRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| result.map_err(|(_, err)| err))
+        .collect()
+}
+
+pub(crate) fn collect_merged_target_configs_compat_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
 ) -> Vec<MergedTargetConfigRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| match result {
+            Ok(record) => record,
+            Err((record, err)) => {
+                warn!(
+                    target_type,
+                    instance_id = %record.instance_id,
+                    error = %err,
+                    "Treating target instance with invalid enable configuration as disabled"
+                );
+                record
+            }
+        })
+        .collect()
+}
+
+fn collect_merged_target_config_results_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Vec<Result<MergedTargetConfigRecord, (MergedTargetConfigRecord, TargetError)>>
 where
     I: IntoIterator<Item = (String, String)>,
 {
@@ -220,14 +299,28 @@ where
             let redacted_config = redacted_target_config(&merged_config);
             debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
         }
-        merged_configs.push(MergedTargetConfigRecord {
-            instance_id: id,
-            enabled: is_target_enabled(&merged_config),
-            effective_config: merged_config,
-            has_file_default,
-            has_file_instance,
-            has_env_default,
-            has_env_instance,
+        merged_configs.push(match is_target_enabled(&merged_config) {
+            Ok(enabled) => Ok(MergedTargetConfigRecord {
+                instance_id: id,
+                enabled,
+                effective_config: merged_config,
+                has_file_default,
+                has_file_instance,
+                has_env_default,
+                has_env_instance,
+            }),
+            Err(err) => Err((
+                MergedTargetConfigRecord {
+                    instance_id: id,
+                    enabled: false,
+                    effective_config: merged_config,
+                    has_file_default,
+                    has_file_instance,
+                    has_env_default,
+                    has_env_instance,
+                },
+                err,
+            )),
         });
     }
 
@@ -238,8 +331,9 @@ where
 mod tests {
     use super::{
         collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
-        redacted_target_config,
+        redacted_target_config, try_collect_target_configs_from_env,
     };
+    use crate::TargetError;
     use rustfs_config::notify::{
         ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
         ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX,
@@ -269,7 +363,7 @@ mod tests {
 
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -282,7 +376,8 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()),
                 ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
             ],
-        );
+        )
+        .expect("valid env target");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         assert_eq!(configs.len(), 2);
@@ -295,7 +390,7 @@ mod tests {
     #[test]
     fn collect_target_configs_discovers_enabled_instance_from_env() {
         let cfg = Config(HashMap::new());
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -307,7 +402,8 @@ mod tests {
                     "https://example.com/from-env".to_string(),
                 ),
             ],
-        );
+        )
+        .expect("valid target configs");
 
         assert_eq!(configs.len(), 1);
         assert_eq!(configs[0].0, "primary");
@@ -323,7 +419,7 @@ mod tests {
         subsystem.insert("_".to_string(), default_kvs);
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -332,7 +428,8 @@ mod tests {
                 "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
                 "https://example.com/secondary".to_string(),
             )],
-        );
+        )
+        .expect("valid target configs");
 
         assert!(configs.is_empty());
     }
@@ -361,7 +458,7 @@ mod tests {
         let cfg = Config(HashMap::new());
         let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect();
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "redis",
@@ -372,7 +469,8 @@ mod tests {
                 (format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()),
                 (format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()),
             ],
-        );
+        )
+        .expect("valid redis target config");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         let redis_config = configs.get("primary").expect("redis env target should be discovered");
@@ -383,6 +481,57 @@ mod tests {
         assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off"));
     }
 
+    #[test]
+    fn collect_target_configs_rejects_invalid_instance_enable_value() {
+        let err = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must not look like a disabled target");
+
+        match err {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() {
+        let configs = collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "good");
+    }
+
+    #[test]
+    fn collect_target_configs_preserves_whitespace_padded_legacy_value() {
+        let configs = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())],
+        )
+        .expect("the shared enable parser accepts surrounding whitespace");
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "primary");
+        assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on "));
+    }
+
     #[test]
     fn redact_target_field_value_redacts_sensitive_fields() {
         assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");

crates/targets/src/config/mod.rs

@@ -21,10 +21,11 @@ pub use instance::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use loader::{
     collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
-    collect_target_configs_from_env,
+    collect_target_configs_from_env, try_collect_target_configs, try_collect_target_configs_from_env,
 };
 pub use target_args::{
     build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,

crates/targets/src/lib.rs

@@ -43,6 +43,7 @@ pub use config::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use control_plane::{
     TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError,
@@ -65,8 +66,9 @@ pub use plugin::{
     TargetPluginRegistry, TargetRequestValidator, boxed_target,
 };
 pub use runtime::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
+    OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot,
+    RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager,
+    activate_targets_with_replay,
     adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
     init_target_and_optionally_start_replay,
     ops_diagnostics::{

crates/targets/src/plugin.rs

@@ -14,8 +14,9 @@
 
 use crate::{
     PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
-    config::collect_target_configs,
+    config::try_collect_target_configs,
     manifest::{TargetPluginManifest, builtin_target_manifest},
+    target::with_deferred_queue_store_open,
 };
 use hashbrown::HashMap;
 use rustfs_config::server_config::{Config, KVS};
@@ -317,39 +318,68 @@ where
         config: &Config,
         route_prefix: &str,
     ) -> Result<Vec<BoxedTarget<E>>, TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, false)
+            .await
+            .map(|(targets, _)| targets)
+    }
+
+    /// Creates targets while deferring queue-store open until runtime handoff.
+    /// Unlike the compatibility activation API, lifecycle preparation reports
+    /// any invalid or unconstructable configured instance so the originating
+    /// Admin request cannot report a false success.
+    pub async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, true)
+            .await
+    }
+
+    async fn create_targets_from_config_with_store_mode(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+        defer_store_open: bool,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
         let mut successful_targets = Vec::new();
-        let mut failed_targets = 0usize;
+        let mut failures = Vec::new();
 
         for (target_type, plugin) in &self.plugins {
             info!(target_type = %target_type, "Start working on target type");
-            for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
+            for (id, merged_config) in try_collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set())? {
                 info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
-                match self.create_target(target_type, id.clone(), &merged_config) {
+                let created = if defer_store_open {
+                    with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config))
+                } else {
+                    self.create_target(target_type, id.clone(), &merged_config)
+                };
+                match created {
                     Ok(target) => {
                         info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
                         successful_targets.push(target);
                     }
-                    Err(err) => {
-                        failed_targets += 1;
-                        error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
+                    Err(_) => {
+                        failures.push(format!("{target_type}/{id}: target construction failed"));
+                        error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target");
                     }
                 }
             }
         }
 
-        if failed_targets > 0 {
+        if !failures.is_empty() {
             warn!(
                 created = successful_targets.len(),
-                failed = failed_targets,
+                failed = failures.len(),
                 "Some configured targets failed to create and were skipped"
             );
         }
         info!(
             count = successful_targets.len(),
-            failed = failed_targets,
+            failed = failures.len(),
             "All target processing completed"
         );
-        Ok(successful_targets)
+        Ok((successful_targets, failures))
     }
 
     pub async fn create_activation_from_config<A>(

crates/targets/src/runtime/adapter.rs

@@ -13,21 +13,53 @@
 // limitations under the License.
 
 use super::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+    OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
+    RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
+    start_replay_worker,
 };
 use crate::plugin::PluginEvent;
-use crate::{Target, TargetError};
+use crate::{SharedTarget, Target, TargetError};
 use async_trait::async_trait;
+use rayon::prelude::*;
 use std::future::Future;
+use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
 
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
 
+const MAX_PARALLEL_STORE_OPENS: usize = 4;
+
+static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(MAX_PARALLEL_STORE_OPENS)
+        .thread_name(|index| format!("rustfs-target-store-open-{index}"))
+        .build()
+});
+
+enum StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    Accepted(SharedTarget<E>),
+    Rejected { panicked: bool, target: SharedTarget<E> },
+}
+
+fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
+        Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
+        Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
+        Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
+    }
+}
+
 /// Shared runtime contract for target plugins.
 #[async_trait]
 pub trait PluginRuntimeAdapter<E>: Send + Sync
@@ -96,6 +128,234 @@ where
             stop_log_prefix: stop_log_prefix.into(),
         }
     }
+
+    pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, None).await
+    }
+
+    pub async fn prepare_targets_cancellable(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, Some(cancellation)).await
+    }
+
+    async fn prepare_targets_inner(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: Option<&CancellationToken>,
+    ) -> PreparedActivation<E> {
+        let mut prepared = Vec::with_capacity(targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        let mut targets = targets.into_iter();
+        while let Some(target) = targets.next() {
+            match prepare_target(target, cancellation).await {
+                PrepareTargetResult::Ready(target) => prepared.push(target),
+                PrepareTargetResult::Degraded { error, target } => {
+                    drop(error);
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: initialization failed", target.id()),
+                    });
+                    prepared.push(target);
+                }
+                PrepareTargetResult::Failed { error, target } => {
+                    drop(error);
+                    let target_id = target.id().to_string();
+                    tracing::warn!(
+                        target_id,
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: initialization failed"),
+                    });
+                    rejected_targets.push(Arc::from(target));
+                }
+                PrepareTargetResult::Cancelled(target) => {
+                    prepared.push(Arc::from(target));
+                    prepared.extend(targets.map(Arc::from));
+                    break;
+                }
+            }
+        }
+        PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: prepared,
+        }
+    }
+
+    /// Opens queue stores only after the previous runtime generation has been
+    /// quiesced. Targets whose stores cannot be opened retain the established
+    /// fault-isolation behavior and are returned for lock-free shutdown.
+    pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
+        let mut accepted = Vec::with_capacity(prepared.targets.len());
+        let mut failures = prepared.failures;
+        let mut rejected = prepared.rejected_targets;
+        let outcomes = if prepared.targets.len() < 2 {
+            prepared.targets.into_iter().map(open_target_store).collect()
+        } else {
+            match STORE_OPEN_POOL.as_ref() {
+                // Vec's indexed parallel iterator preserves configuration
+                // order in collect, keeping failure summaries deterministic.
+                Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
+                Err(err) => {
+                    tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
+                    prepared.targets.into_iter().map(open_target_store).collect()
+                }
+            }
+        };
+        for outcome in outcomes {
+            match outcome {
+                StoreOpenOutcome::Accepted(target) => accepted.push(target),
+                StoreOpenOutcome::Rejected { panicked, target } => {
+                    if panicked {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_panicked",
+                            "Target queue store panicked while opening during runtime handoff"
+                        );
+                    } else {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_failed",
+                            "Failed to open target queue store during runtime handoff"
+                        );
+                    }
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: queue store open failed", target.id()),
+                    });
+                    rejected.push(target);
+                }
+            }
+        }
+        (
+            OpenedActivation { targets: accepted },
+            PreparedActivation {
+                failures,
+                rejected_targets: rejected,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
+        let mut replay_workers = ReplayWorkerManager::new();
+        let mut accepted = Vec::with_capacity(opened.targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        for target in opened.targets {
+            let target_id = target.id().to_string();
+            let replay = catch_unwind(AssertUnwindSafe(|| {
+                target.store().filter(|_| target.is_enabled()).map(|store| {
+                    start_replay_worker(
+                        store.boxed_clone(),
+                        Arc::clone(&target),
+                        Arc::clone(&self.replay_hook),
+                        self.replay_semaphore.clone(),
+                        self.batch_timeout,
+                        self.idle_sleep,
+                    )
+                })
+            }));
+            let replay = match replay {
+                Ok(replay) => replay,
+                Err(_) => {
+                    tracing::error!(
+                        target_id,
+                        reason = "replay_activation_panicked",
+                        "Target replay activation panicked during runtime handoff"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: replay activation failed"),
+                    });
+                    rejected_targets.push(target);
+                    continue;
+                }
+            };
+            (self.replay_start_observer)(&target_id, replay.is_some());
+            if let Some((cancel_tx, join)) = replay {
+                replay_workers.insert_with_handle(target_id, cancel_tx, join);
+            }
+            accepted.push(target);
+        }
+
+        (
+            RuntimeActivation {
+                replay_workers,
+                targets: accepted,
+            },
+            PreparedActivation {
+                failures,
+                rejected_targets,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    #[doc(hidden)]
+    pub async fn prepare_dormant_compat_activation(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    ) -> RuntimeActivation<E> {
+        let PreparedActivation {
+            failures,
+            rejected_targets,
+            targets,
+        } = self.prepare_targets(targets).await;
+        let rejected = PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: Vec::new(),
+        };
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
+        }
+        RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn start_dormant_compat_activation(
+        &self,
+        activation: RuntimeActivation<E>,
+    ) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
+        let prepared = PreparedActivation {
+            failures: Vec::new(),
+            rejected_targets: Vec::new(),
+            targets: activation.targets,
+        };
+        let (opened, open_rejected) = self.open_prepared_stores(prepared);
+        let (activation, activation_rejected) = self.try_activate_prepared(opened);
+        (activation, open_rejected, activation_rejected)
+    }
+
+    #[doc(hidden)]
+    pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+        self.shutdown(&mut runtime, &mut activation.replay_workers).await
+    }
+
+    pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
+            runtime.add_arc(target);
+        }
+        let mut replay_workers = ReplayWorkerManager::new();
+        self.shutdown(&mut runtime, &mut replay_workers).await
+    }
 }
 
 #[async_trait]
@@ -104,36 +364,16 @@ where
     E: PluginEvent,
 {
     async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
-        let replay_hook = Arc::clone(&self.replay_hook);
-        let replay_start_observer = Arc::clone(&self.replay_start_observer);
-        let replay_semaphore = self.replay_semaphore.clone();
-        let batch_timeout = self.batch_timeout;
-        let idle_sleep = self.idle_sleep;
-
-        activate_targets_with_replay(targets, move |target| {
-            let replay_hook = Arc::clone(&replay_hook);
-            let replay_start_observer = Arc::clone(&replay_start_observer);
-            let replay_semaphore = replay_semaphore.clone();
-
-            async move {
-                init_target_and_optionally_start_replay(
-                    target,
-                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
-                    move |store, target| {
-                        start_replay_worker(
-                            store,
-                            target,
-                            Arc::clone(&replay_hook),
-                            replay_semaphore.clone(),
-                            batch_timeout,
-                            idle_sleep,
-                        )
-                    },
-                )
-                .await
-            }
-        })
-        .await
+        let prepared = self.prepare_targets(targets).await;
+        let (opened, rejected) = self.open_prepared_stores(prepared);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
+        }
+        let (activation, rejected) = self.try_activate_prepared(opened);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
+        }
+        activation
     }
 
     async fn replace_runtime_targets(
@@ -189,7 +429,7 @@ where
         if !close_errors.is_empty() {
             let detail = close_errors
                 .into_iter()
-                .map(|(target_id, err)| format!("{target_id}: {err}"))
+                .map(|(target_id, _)| target_id)
                 .collect::<Vec<_>>()
                 .join("; ");
             return Err(TargetError::Storage(format!("Failed to close {detail}")));
@@ -200,24 +440,143 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
     use crate::PluginEvent;
     use crate::arn::TargetID;
     use crate::store::{Key, QueueStore, Store};
     use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use crate::{StoreError, Target, TargetError};
     use async_trait::async_trait;
-    use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Condvar, Mutex};
     use std::time::Duration;
     use tempfile::tempdir;
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
+
+    type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
+
+    type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
+
+    #[derive(Clone)]
+    struct TestOpenStore {
+        before_clone: BeforeOpen,
+        before_open: BeforeOpen,
+        store: QueueStore<QueuedPayload>,
+    }
+
+    impl Store<QueuedPayload> for TestOpenStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            (self.before_open)();
+            self.store.open()
+        }
+
+        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put(item)
+        }
+
+        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put_multiple(items)
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            self.store.put_raw(data)
+        }
+
+        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            self.store.get(key)
+        }
+
+        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            self.store.get_multiple(key)
+        }
+
+        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            self.store.get_raw(key)
+        }
+
+        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+            self.store.del(key)
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            self.store.delete()
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            self.store.list()
+        }
+
+        fn len(&self) -> usize {
+            self.store.len()
+        }
+
+        fn is_empty(&self) -> bool {
+            self.store.is_empty()
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            (self.before_clone)();
+            Box::new(self.clone())
+        }
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGate {
+        changed: Condvar,
+        state: Mutex<StoreOpenGateState>,
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGateState {
+        active: usize,
+        max_active: usize,
+        released: bool,
+    }
+
+    impl StoreOpenGate {
+        fn enter(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.active += 1;
+            state.max_active = state.max_active.max(state.active);
+            self.changed.notify_all();
+            while !state.released {
+                state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
+            }
+            state.active -= 1;
+        }
+
+        fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
+            let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            let (state, _) = self
+                .changed
+                .wait_timeout_while(state, timeout, |state| state.max_active < expected)
+                .unwrap_or_else(|err| err.into_inner());
+            state.max_active >= expected
+        }
+
+        fn release(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.released = true;
+            self.changed.notify_all();
+        }
+
+        fn max_active(&self) -> usize {
+            self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
+        }
+    }
 
     #[derive(Clone)]
     struct TestTarget {
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        init_calls: Arc<AtomicUsize>,
+        init_entered: Option<Arc<Notify>>,
         init_fails: bool,
-        store: Option<QueueStore<QueuedPayload>>,
+        store: Option<Arc<TestStore>>,
     }
 
     impl TestTarget {
@@ -225,6 +584,8 @@ mod tests {
             Self {
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                init_calls: Arc::new(AtomicUsize::new(0)),
+                init_entered: None,
                 init_fails: false,
                 store: None,
             }
@@ -235,11 +596,16 @@ mod tests {
             self
         }
 
+        fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
+            self.init_entered = Some(init_entered);
+            self
+        }
+
         fn with_store(mut self) -> Self {
             let dir = tempdir().expect("tempdir should be created for queue store tests");
             let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
             store.open().expect("queue store should open");
-            self.store = Some(store);
+            self.store = Some(Arc::new(store));
             self
         }
     }
@@ -271,9 +637,7 @@ mod tests {
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            self.store
-                .as_ref()
-                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+            self.store.as_deref()
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -281,6 +645,11 @@ mod tests {
         }
 
         async fn init(&self) -> Result<(), TargetError> {
+            self.init_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(init_entered) = &self.init_entered {
+                init_entered.notify_one();
+                return std::future::pending().await;
+            }
             if self.init_fails {
                 return Err(TargetError::Configuration("forced init failure".to_string()));
             }
@@ -334,6 +703,212 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 1);
     }
 
+    #[tokio::test]
+    async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert_eq!(prepared.targets.len(), 1);
+        assert!(
+            prepared
+                .failure_summary()
+                .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
+        );
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.failure_summary().is_some());
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation.replay_workers.stop_all("stop degraded target replay worker").await;
+    }
+
+    #[tokio::test]
+    async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
+        let adapter = builtin_adapter();
+        let init_entered = Arc::new(Notify::new());
+        let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
+        let first_close_calls = first.close_calls.clone();
+        let second = TestTarget::new("second", "webhook");
+        let second_close_calls = second.close_calls.clone();
+        let second_init_calls = second.init_calls.clone();
+        let cancellation = CancellationToken::new();
+        let prepare_adapter = adapter.clone();
+        let prepare_cancellation = cancellation.clone();
+        let prepare = tokio::spawn(async move {
+            prepare_adapter
+                .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
+                .await
+        });
+
+        init_entered.notified().await;
+        cancellation.cancel();
+        let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
+            .await
+            .expect("cancellation should interrupt target initialization")
+            .expect("preparation task should finish");
+        assert_eq!(prepared.targets.len(), 2);
+
+        adapter
+            .close_prepared(prepared)
+            .await
+            .expect("cancelled targets should close");
+        assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn prepared_activation_opens_store_before_starting_replay() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let queue_path = dir.path().join("queue");
+        let mut target = TestTarget::new("primary", "webhook");
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.targets.is_empty());
+        assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation
+            .replay_workers
+            .stop_all("stop prepared activation test worker")
+            .await;
+    }
+
+    #[tokio::test]
+    async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let invalid_base = dir.path().join("not-a-directory");
+        std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
+        let mut target = TestTarget::new("primary", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
+        const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
+
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let gate = Arc::new(StoreOpenGate::default());
+        let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
+        let mut expected_ids = Vec::with_capacity(TARGETS);
+        for index in 0..TARGETS {
+            let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
+            expected_ids.push(target.id.to_string());
+            let open_gate = gate.clone();
+            target.store = Some(Arc::new(TestOpenStore {
+                before_clone: Arc::new(|| {}),
+                before_open: Arc::new(move || open_gate.enter()),
+                store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
+            }));
+            targets.push(Box::new(target));
+        }
+
+        let prepared = adapter.prepare_targets(targets).await;
+        let open_adapter = adapter.clone();
+        let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
+        let wait_gate = gate.clone();
+        let reached_bound =
+            tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
+                .await
+                .expect("store-open observer should not panic");
+        gate.release();
+        let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
+        let opened_ids = opened
+            .targets
+            .iter()
+            .map(|target| target.id().to_string())
+            .collect::<Vec<_>>();
+
+        assert!(reached_bound, "store opens did not use the configured parallelism");
+        assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
+        assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
+        assert!(rejected.targets.is_empty());
+        assert!(rejected.failure_summary().is_none());
+    }
+
+    #[tokio::test]
+    async fn panicking_store_open_rejects_and_closes_only_that_target() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| {}),
+            before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store should report a generic activation failure");
+        let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
+
+        assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
+        assert!(
+            activation.replay_workers.is_empty(),
+            "a rejected target must not publish without a replay worker"
+        );
+        assert!(activation_rejected.failure_summary().is_none());
+        assert!(summary.contains("queue store open failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected after a store panic should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking-clone", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
+            before_open: Arc::new(|| {}),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
+        assert!(open_rejected.failure_summary().is_none());
+        let (activation, rejected) = adapter.try_activate_prepared(opened);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store clone should report a generic activation failure");
+
+        assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
+        assert!(activation.replay_workers.is_empty());
+        assert!(summary.contains("replay activation failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected during replay activation should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[tokio::test]
     async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
         let adapter = builtin_adapter();

crates/targets/src/runtime/mod.rs

@@ -27,11 +27,21 @@ use crate::store::{Key, Store, ensure_store_entry_raw_readable};
 use crate::target::QueuedPayload;
 use crate::target::TargetDeliverySnapshot;
 use crate::{StoreError, TargetError};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::sync::Arc;
 use std::{collections::HashMap, fmt::Debug};
 use std::{future::Future, pin::Pin, time::Duration};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}
 
 /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
 /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
@@ -68,11 +78,21 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
 pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 
-/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
-/// replay worker to observe its cancel signal and exit before it is forcibly
-/// aborted. Workers observe cancellation promptly (including during retry
-/// backoff), so this only guards against a wedged task.
-const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
+pub(crate) enum PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    Ready(SharedTarget<E>),
+    Degraded {
+        error: TargetError,
+        target: SharedTarget<E>,
+    },
+    Failed {
+        error: TargetError,
+        target: Box<dyn Target<E> + Send + Sync>,
+    },
+    Cancelled(Box<dyn Target<E> + Send + Sync>),
+}
 
 /// Tracks a running replay worker: its cancel channel and, when the worker was
 /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
@@ -113,7 +133,7 @@ impl ReplayWorkerManager {
     }
 
     /// Registers a cancel channel together with the worker's join handle so
-    /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
+    /// `stop_all` can await the worker's exit.
     pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
         self.cancellers.insert(
             target_id,
@@ -140,37 +160,36 @@ impl ReplayWorkerManager {
     }
 
     /// Stops every replay worker: it first signals cancellation to all of them,
-    /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
-    /// which the task is aborted). Signalling before joining lets all workers
-    /// wind down concurrently, and joining guarantees no worker keeps draining
-    /// the shared store after this returns — preventing duplicate delivery and
-    /// orphaned tasks across reloads and shutdown.
+    /// then strictly awaits each worker's exit. A worker already awaiting a
+    /// delivery acknowledgement is allowed to finish; aborting it at an
+    /// arbitrary deadline could leave an acknowledged queue entry undeleted and
+    /// make the replacement worker deliver it again. Signalling before joining
+    /// lets all workers wind down concurrently. Legacy joinless registrations
+    /// can only be signalled.
     pub async fn stop_all(&mut self, log_prefix: &str) {
-        let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let mut joins = std::collections::VecDeque::new();
 
         // Phase 1: signal cancellation to all workers.
-        for (target_id, handle) in &handles {
+        for (target_id, handle) in handles {
             tracing::info!(target_id = %target_id, "{log_prefix}");
-            let _ = handle.cancel_tx.send(()).await;
+            let _ = handle.cancel_tx.try_send(());
+            if let Some(join) = handle.join {
+                joins.push_back((target_id, join));
+            } else {
+                tracing::warn!(
+                    target_id = %target_id,
+                    "Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
+                );
+            }
         }
 
-        // Phase 2: await each worker's exit, forcibly aborting any that overrun.
-        for (target_id, handle) in handles.drain(..) {
-            let Some(mut join) = handle.join else {
-                continue;
-            };
-            match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
-                Ok(Ok(())) => {}
-                Ok(Err(err)) => {
-                    tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
-                }
-                Err(_) => {
-                    join.abort();
-                    tracing::warn!(
-                        target_id = %target_id,
-                        "Timed out awaiting replay worker exit; task aborted"
-                    );
-                }
+        // Phase 2: strict join. Delivery operations own their own protocol
+        // deadlines; lifecycle must not invent a shorter deadline that turns an
+        // acknowledgement race into duplicate delivery.
+        while let Some((target_id, join)) = joins.pop_front() {
+            if let Err(err) = join.await {
+                tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
             }
         }
     }
@@ -184,6 +203,55 @@ where
     pub targets: Vec<SharedTarget<E>>,
 }
 
+/// Targets whose persistent queue stores are open and are ready to start
+/// replay. This distinct stage prevents activation from skipping store open.
+pub struct OpenedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+struct TargetActivationFailure {
+    detail: String,
+}
+
+/// Targets that have completed initialization but have not started replay
+/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
+/// the previous workers before the replacement workers are spawned.
+pub struct PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    failures: Vec<TargetActivationFailure>,
+    rejected_targets: Vec<SharedTarget<E>>,
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+impl<E> PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub fn failure_summary(&self) -> Option<String> {
+        if self.failures.is_empty() {
+            return None;
+        }
+
+        Some(
+            self.failures
+                .iter()
+                .map(|failure| failure.detail.clone())
+                .collect::<Vec<_>>()
+                .join("; "),
+        )
+    }
+
+    pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
+        self.failures
+            .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct RuntimeStatusSnapshot {
     pub replay_worker_count: usize,
@@ -339,17 +407,19 @@ where
     /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
     /// runtime.
     pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
-        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        let targets = std::mem::take(&mut self.targets);
+        let mut closes = FuturesUnordered::new();
+        for (target_id, target) in targets {
+            closes.push(async move { (target_id, target.close().await) });
+        }
+
         let mut errors = Vec::new();
-        for target_id in target_ids {
-            if let Some(target) = self.targets.remove(&target_id)
-                && let Err(err) = target.close().await
-            {
+        while let Some((target_id, result)) = closes.next().await {
+            if let Err(err) = result {
                 tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
                 errors.push((target_id, err));
             }
         }
-        self.targets.clear();
         errors
     }
 
@@ -440,21 +510,16 @@ where
         SharedTarget<E>,
     ) -> (mpsc::Sender<()>, JoinHandle<()>),
 {
-    let target_id = target.id().to_string();
-    let has_store = target.store().is_some();
-
-    if let Err(err) = target.init().await {
-        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
-        if !has_store {
+    let shared = match prepare_target(target, None).await {
+        PrepareTargetResult::Ready(target) => target,
+        PrepareTargetResult::Degraded { target, .. } => target,
+        PrepareTargetResult::Failed { target, .. } => {
+            let _ = target.close().await;
             return None;
         }
-        tracing::warn!(
-            target_id = %target_id,
-            "Proceeding with store-backed target despite init failure"
-        );
-    }
-
-    let shared: SharedTarget<E> = Arc::from(target);
+        PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
+    };
+    let target_id = shared.id().to_string();
     if !shared.is_enabled() {
         on_replay_start(&target_id, false);
         return Some((shared, None));
@@ -467,6 +532,45 @@ where
     Some((shared, cancel))
 }
 
+pub(crate) async fn prepare_target<E>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    cancellation: Option<&CancellationToken>,
+) -> PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    let init_result = match cancellation {
+        Some(cancellation) => {
+            tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
+                result = target.init() => result,
+            }
+        }
+        None => target.init().await,
+    };
+
+    if let Err(err) = init_result {
+        tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
+        if !has_store {
+            return PrepareTargetResult::Failed { error: err, target };
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+        return PrepareTargetResult::Degraded {
+            error: err,
+            target: Arc::from(target),
+        };
+    }
+
+    PrepareTargetResult::Ready(Arc::from(target))
+}
+
 type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);
 
 pub async fn activate_targets_with_replay<E, F, Fut>(
@@ -584,7 +688,11 @@ async fn stream_replay_worker<E>(
                 }
                 Ok(Ok(_)) => {}
                 Err(join_err) => {
-                    tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = join_failure_reason(&join_err),
+                        "The failed-events maintenance task failed to join"
+                    );
                 }
             }
             last_prune = tokio::time::Instant::now();
@@ -835,7 +943,8 @@ mod tests {
     use crate::{Target, TargetError};
     use async_trait::async_trait;
     use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use tokio::sync::{Notify, Semaphore};
 
     #[tokio::test(start_paused = true)]
     async fn seed_interval_start_backdates_by_one_interval() {
@@ -900,14 +1009,20 @@ mod tests {
     #[derive(Clone)]
     struct TestTarget {
         id: TargetID,
+        block_on_close: Arc<AtomicBool>,
+        close_gate: Arc<Semaphore>,
         close_calls: Arc<AtomicUsize>,
+        close_started: Arc<Notify>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
                 id: TargetID::new(id.to_string(), name.to_string()),
+                block_on_close: Arc::new(AtomicBool::new(false)),
+                close_gate: Arc::new(Semaphore::new(0)),
                 close_calls: Arc::new(AtomicUsize::new(0)),
+                close_started: Arc::new(Notify::new()),
             }
         }
     }
@@ -935,6 +1050,10 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            self.close_started.notify_one();
+            if self.block_on_close.load(Ordering::SeqCst) {
+                let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
+            }
             Ok(())
         }
 
@@ -966,6 +1085,45 @@ mod tests {
         assert_eq!(close_calls.load(Ordering::SeqCst), 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let first = TestTarget::new("first", "webhook");
+        let second = TestTarget::new("second", "webhook");
+        let first_observer = first.clone();
+        let second_observer = second.clone();
+        manager.add_boxed(Box::new(first));
+        manager.add_boxed(Box::new(second));
+
+        let first_close_key = manager
+            .keys()
+            .into_iter()
+            .next()
+            .expect("two targets should have a first close key");
+        let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
+            (first_observer, second_observer)
+        } else {
+            (second_observer, first_observer)
+        };
+        blocked.block_on_close.store(true, Ordering::SeqCst);
+
+        let close_task = tokio::spawn(async move { manager.clear_and_close().await });
+
+        tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
+            .await
+            .expect("the first target close should start");
+        tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
+            .await
+            .expect("a blocked first close must not prevent the second close from starting");
+        assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
+
+        blocked.close_gate.add_permits(1);
+        let errors = close_task.await.expect("clear_and_close task should join");
+        assert!(errors.is_empty());
+        assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[test]
     fn runtime_manager_snapshots_targets() {
         let mut manager = TargetRuntimeManager::<String>::new();
@@ -1030,6 +1188,51 @@ mod tests {
         assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
+        use super::ReplayWorkerManager;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use tokio::sync::Notify;
+
+        let mut manager = ReplayWorkerManager::new();
+        let acknowledgement = Arc::new(Notify::new());
+        let worker_started = Arc::new(Notify::new());
+        let exited = Arc::new(AtomicBool::new(false));
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
+        let worker_acknowledgement = acknowledgement.clone();
+        let worker_started_signal = worker_started.clone();
+        let worker_exited = exited.clone();
+        let join = tokio::spawn(async move {
+            worker_started_signal.notify_one();
+            let _ = cancel_rx.recv().await;
+            // Model a protocol operation that has accepted the request but has
+            // not returned its acknowledgement yet. Lifecycle must not abort
+            // this future or the same durable entry can be sent twice.
+            worker_acknowledgement.notified().await;
+            worker_exited.store(true, Ordering::SeqCst);
+        });
+        manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
+        worker_started.notified().await;
+
+        let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
+            _ = std::future::ready(()) => {}
+        }
+        tokio::time::advance(std::time::Duration::from_secs(60)).await;
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
+            _ = std::future::ready(()) => {}
+        }
+
+        acknowledgement.notify_one();
+        stop.await;
+        assert!(exited.load(Ordering::SeqCst));
+        assert!(manager.is_empty());
+    }
+
     mod classifier {
         use super::super::{ReplayEvent, stream_replay_worker};
         use crate::arn::TargetID;

crates/targets/src/target/kafka.rs

@@ -24,7 +24,7 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
         open_target_queue_store, persist_queued_payload_to_store,
     },
 };
@@ -32,13 +32,84 @@ use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
 use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
-use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};
 
 pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
+const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct KafkaDeliveryAttempt<'a> {
+    armed: bool,
+    poisoned: &'a AtomicBool,
+}
+
+impl KafkaDeliveryAttempt<'_> {
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for KafkaDeliveryAttempt<'_> {
+    fn drop(&mut self) {
+        if self.armed {
+            self.poisoned.store(true, Ordering::Release);
+        }
+    }
+}
+
+fn kafka_delivery_timeout() -> TargetError {
+    TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
+}
+
+async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
+    delivery_lock: &Mutex<()>,
+    delivery_poisoned: &AtomicBool,
+    select_producer: Select,
+    deliver: Deliver,
+    invalidate: Invalidate,
+) -> Result<T, TargetError>
+where
+    P: Send,
+    T: Send,
+    Select: FnOnce() -> SelectFuture + Send,
+    SelectFuture: Future<Output = Result<P, TargetError>> + Send,
+    Deliver: FnOnce(P) -> DeliveryFuture + Send,
+    DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
+    Invalidate: Fn() -> InvalidateFuture + Send,
+    InvalidateFuture: Future<Output = ()> + Send,
+{
+    let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
+    let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    let mut attempt = KafkaDeliveryAttempt {
+        armed: true,
+        poisoned: delivery_poisoned,
+    };
+
+    if delivery_poisoned.load(Ordering::Acquire) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+
+    let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    if result.as_ref().is_err_and(is_connectivity_error) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+    attempt.disarm();
+    result
+}
 
 /// Arguments for configuring a Kafka target
 #[derive(Clone)]
@@ -233,6 +304,8 @@ where
     args: KafkaArgs,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_lock: Arc<Mutex<()>>,
+    delivery_poisoned: Arc<AtomicBool>,
     tls_state: Arc<Mutex<TargetTlsState>>,
     /// Adapter that bridges this target to the TLS reload coordinator.
     /// When `Some`, the target uses coordinator-managed material; when `None`,
@@ -291,6 +364,8 @@ where
             args,
             store: queue_store,
             producer: Arc::new(Mutex::new(None)),
+            delivery_lock: Arc::new(Mutex::new(())),
+            delivery_poisoned: Arc::new(AtomicBool::new(false)),
             tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
             tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -307,7 +382,7 @@ where
         };
 
         let mut config = AsyncProducerConfig::new()
-            .with_ack_timeout(Duration::from_secs(30))
+            .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
             .with_required_acks(acks);
 
         if let Some(security) = self.args.security_config(true)? {
@@ -388,20 +463,26 @@ where
             "Sending Kafka payload"
         );
 
-        let producer = self.get_or_build_producer().await?;
-
-        // Use "<bucket>/<object>" as the message key so all events for the same
-        // object hash to the same partition and preserve per-object ordering
-        // across multiple partitions (backlog#983).
-        let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
-        if let Err(err) = producer
-            .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
-            .await
-        {
-            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
-            return Err(mapped);
-        }
+        // rustfs-kafka-async does not validate response correlation IDs. Keep
+        // producer selection, send, and timeout invalidation serialized so a
+        // waiter cannot reuse a connection with an unread timed-out response.
+        with_serialized_kafka_delivery(
+            &self.delivery_lock,
+            &self.delivery_poisoned,
+            || self.get_or_build_producer(),
+            |producer| async move {
+                // Use "<bucket>/<object>" as the message key so all events for the same
+                // object hash to the same partition and preserve per-object ordering
+                // across multiple partitions (backlog#983).
+                let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
+                producer
+                    .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
+                    .await
+                    .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
+            },
+            || self.invalidate_cached_producer(),
+        )
+        .await?;
 
         debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
         self.delivery_counters.record_success();
@@ -415,6 +496,8 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             producer: Arc::clone(&self.producer),
+            delivery_lock: Arc::clone(&self.delivery_lock),
+            delivery_poisoned: Arc::clone(&self.delivery_poisoned),
             tls_state: Arc::clone(&self.tls_state),
             tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -559,6 +642,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::atomic::AtomicUsize;
+    use tokio::sync::Notify;
 
     fn base_args() -> KafkaArgs {
         KafkaArgs {
@@ -580,6 +665,158 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    move |selected| async move {
+                        assert_eq!(selected, 1);
+                        first_entered.notify_one();
+                        std::future::pending::<Result<usize, TargetError>>().await
+                    },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        first_entered.notified().await;
+        tokio::time::advance(Duration::from_secs(1)).await;
+        let second = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    |selected| async move { Ok(selected) },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        assert!(matches!(
+            first.await.expect("first delivery task should not panic"),
+            Err(TargetError::Timeout(_))
+        ));
+        assert_eq!(
+            second
+                .await
+                .expect("second delivery task should not panic")
+                .expect("second delivery should succeed"),
+            2,
+            "the waiter must select a fresh producer generation after timeout invalidation"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = AtomicBool::new(false);
+        let selected = Arc::new(AtomicBool::new(false));
+        let _held = delivery_lock.lock().await;
+
+        let error = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let selected = Arc::clone(&selected);
+                move || async move {
+                    selected.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            },
+            |()| async { Ok(()) },
+            || async {},
+        )
+        .await
+        .expect_err("lock admission must share the absolute delivery deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
+        assert!(!delivery_poisoned.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    || async { Ok(1usize) },
+                    move |_| async move {
+                        first_entered.notify_one();
+                        std::future::pending::<Result<(), TargetError>>().await
+                    },
+                    || async {},
+                )
+                .await
+            })
+        };
+        first_entered.notified().await;
+        first.abort();
+        assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
+        assert!(delivery_poisoned.load(Ordering::Acquire));
+
+        let selected = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let generation = Arc::clone(&generation);
+                move || async move { Ok(generation.load(Ordering::SeqCst)) }
+            },
+            |selected| async move { Ok(selected) },
+            {
+                let generation = Arc::clone(&generation);
+                move || {
+                    let generation = Arc::clone(&generation);
+                    async move { generation.store(2, Ordering::SeqCst) }
+                }
+            },
+        )
+        .await
+        .expect("the next delivery should recover from cancellation poisoning");
+
+        assert_eq!(selected, 2);
+        assert!(!delivery_poisoned.load(Ordering::Acquire));
+    }
+
     #[test]
     fn test_validate_empty_brokers() {
         let args = KafkaArgs {

crates/targets/src/target/mod.rs

@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
+use std::cell::Cell;
 use std::fmt::Formatter;
 use std::future::Future;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::thread_local;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};
 
 pub mod amqp;
@@ -111,7 +113,12 @@ where
     /// Checks if the target is active and reachable
     async fn is_active(&self) -> Result<bool, TargetError>;
 
-    /// Saves an event (either sends it immediately or stores it for later)
+    /// Saves an event (either sends it immediately or stores it for later).
+    ///
+    /// A target whose [`Self::store`] returns `Some` must only persist the event
+    /// here; network delivery belongs to its replay worker. Runtime lifecycle
+    /// handoff drains these durable enqueues while allowing a direct network
+    /// send to finish against a detached target.
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
 
     /// Sends an event from the store using the queued raw body and metadata.
@@ -605,6 +612,24 @@ pub(crate) fn open_target_queue_store(
     Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
 }
 
+thread_local! {
+    static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
+}
+
+pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
+    struct Reset(bool);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
+        }
+    }
+
+    let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
+    let _reset = Reset(previous);
+    operation()
+}
+
 /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
 /// failed-store capability holds it directly rather than through the type-erased Store handle.
 pub(crate) fn open_target_queue_store_typed(
@@ -625,9 +650,11 @@ pub(crate) fn open_target_queue_store_typed(
         TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
     };
     let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
-    store
-        .open()
-        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
+        store
+            .open()
+            .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    }
 
     Ok(Some(store))
 }
@@ -649,6 +676,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
     matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
 }
 
+/// Applies an absolute deadline to one protocol delivery attempt.
+///
+/// Target clients expose different timeout controls, and several of them only
+/// place a timeout value in the wire request without bounding the local socket
+/// future. Keeping the outer deadline here gives every caller the same typed,
+/// retryable timeout without changing the target-specific error mapping.
+pub(crate) async fn with_delivery_deadline<T, F>(
+    deadline: Duration,
+    operation: &'static str,
+    delivery: F,
+) -> Result<T, TargetError>
+where
+    F: Future<Output = Result<T, TargetError>>,
+{
+    match tokio::time::timeout(deadline, delivery).await {
+        Ok(result) => result,
+        Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
+    }
+}
+
 pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
 where
     F: FnOnce() -> Fut,
@@ -1137,6 +1184,29 @@ mod tests {
         let _ = fs::remove_file(base);
     }
 
+    #[test]
+    fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let store = with_deferred_queue_store_open(|| {
+            open_target_queue_store(
+                base.to_str().unwrap(),
+                100,
+                TargetType::NotifyEvent,
+                ChannelTargetType::Kafka.as_str(),
+                &target_id,
+                "deferred open",
+            )
+        })
+        .expect("deferred construction must not open the queue directory")
+        .expect("non-empty queue directory should create a dormant store");
+
+        assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
+        let _ = fs::remove_file(base);
+    }
+
     #[test]
     fn persist_queued_payload_to_store_writes_encoded_payload() {
         let store = MockQueuedStore::new(false);
@@ -1182,6 +1252,19 @@ mod tests {
         assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
+        let error = with_delivery_deadline(
+            Duration::from_secs(30),
+            "test delivery",
+            std::future::pending::<Result<(), TargetError>>(),
+        )
+        .await
+        .expect_err("a stalled delivery must hit its hard deadline");
+
+        assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
+    }
+
     #[tokio::test]
     async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
         let marker = Arc::new(AtomicBool::new(false));

crates/targets/src/target/mqtt.rs

@@ -765,11 +765,6 @@ where
 
     #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
     async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
         debug!(
             event = EVENT_MQTT_DELIVERY_STATE,
             component = LOG_COMPONENT_TARGETS,
@@ -790,10 +785,22 @@ where
         // silently dropped the event while its durable copy was already deleted
         // (backlog#971). Error classification now matches on the typed error
         // instead of substring matching on the display string.
-        let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
-            Ok(notice) => notice,
-            Err(e) => {
-                let err = classify_mqtt_client_error(&e);
+        let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
+            let client_guard = self.client.lock().await;
+            let client = client_guard
+                .as_ref()
+                .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+            let notice = client
+                .publish_tracked(&self.args.topic, self.args.qos, false, body)
+                .await
+                .map_err(|error| classify_mqtt_client_error(&error))?;
+            drop(client_guard);
+            Ok(notice)
+        })
+        .await
+        {
+            Ok(Ok(notice)) => notice,
+            Ok(Err(err)) => {
                 warn!(
                     event = EVENT_MQTT_DELIVERY_STATE,
                     component = LOG_COMPONENT_TARGETS,
@@ -801,18 +808,29 @@ where
                     target_id = %self.id,
                     state = "publish_failed",
                     reason = "enqueue_error",
-                    error = %e,
+                    error = %err,
                     "mqtt delivery state"
                 );
                 mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                 return Err(err);
             }
+            Err(_) => {
+                warn!(
+                    event = EVENT_MQTT_DELIVERY_STATE,
+                    component = LOG_COMPONENT_TARGETS,
+                    subsystem = LOG_SUBSYSTEM_MQTT,
+                    target_id = %self.id,
+                    state = "publish_failed",
+                    reason = "enqueue_timeout",
+                    "mqtt delivery state"
+                );
+                // Admission can time out because the local bounded request
+                // channel is full while the MQTT session remains connected.
+                // Only protocol/client failures are evidence of disconnect.
+                return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
+            }
         };
 
-        // Release the client lock before awaiting the broker acknowledgement so a
-        // slow/hung broker never blocks other senders from queueing publishes.
-        drop(client_guard);
-
         match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
             Ok(Ok(())) => {
                 debug!(
@@ -1708,9 +1726,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
-        classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
-        validate_mqtt_broker_url,
+        AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
+        MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
+        next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
     };
     use crate::error::TargetError;
     use crate::target::{REDACTED_SECRET, TargetType};
@@ -1720,6 +1738,23 @@ mod tests {
     use tokio::sync::mpsc;
     use url::Url;
 
+    fn base_mqtt_args() -> MQTTArgs {
+        MQTTArgs {
+            enable: true,
+            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
+            topic: "rustfs/events".to_string(),
+            qos: QoS::AtLeastOnce,
+            username: String::new(),
+            password: String::new(),
+            tls: MQTTTlsConfig::default(),
+            max_reconnect_interval: Duration::from_secs(1),
+            keep_alive: Duration::from_secs(30),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
     #[test]
     fn mqtt_client_error_classified_as_not_connected() {
         // A publish that cannot be handed to the event loop means the client is
@@ -1752,6 +1787,38 @@ mod tests {
         assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn enqueue_timeout_keeps_a_live_session_connected() {
+        let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
+        let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
+            .capacity(1)
+            .build();
+        client
+            .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
+            .await
+            .expect("first publish should fill the local channel");
+        *target.client.lock().await = Some(client);
+        target.connected.store(true, Ordering::SeqCst);
+        let meta = QueuedPayloadMeta::new(
+            rustfs_s3_types::EventName::ObjectCreatedPut,
+            "bucket".to_string(),
+            "object".to_string(),
+            "application/json",
+            2,
+        );
+
+        let error = target
+            .send_body(b"{}".to_vec(), &meta)
+            .await
+            .expect_err("a full local request channel should hit the enqueue deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(
+            target.connected.load(Ordering::SeqCst),
+            "local admission pressure is not evidence that the MQTT session disconnected"
+        );
+    }
+
     #[test]
     fn next_reconnect_backoff_doubles_until_capped() {
         let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
@@ -1864,21 +1931,13 @@ mod tests {
     #[test]
     fn debug_redacts_mqtt_secret_fields() {
         let args = MQTTArgs {
-            enable: true,
-            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
-            topic: "rustfs/events".to_string(),
-            qos: QoS::AtLeastOnce,
             username: "mqtt-user".to_string(),
             password: "mqtt-password".to_string(),
             tls: MQTTTlsConfig {
                 client_key_path: "/etc/rustfs/mqtt.key".to_string(),
                 ..MQTTTlsConfig::default()
             },
-            max_reconnect_interval: Duration::from_secs(1),
-            keep_alive: Duration::from_secs(30),
-            queue_dir: String::new(),
-            queue_limit: 0,
-            target_type: TargetType::NotifyEvent,
+            ..base_mqtt_args()
         };
 
         let rendered = format!("{args:?}");

crates/targets/src/target/mysql.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -47,6 +47,8 @@ use uuid::Uuid;
 /// `TargetError::Timeout`, a connectivity error, so the payload stays queued
 /// for replay.
 const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
+/// Absolute ceiling for one INSERT, including pool checkout and server execution.
+const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Name of the optional idempotency-key column / primary key. Present on tables
 /// created by this target; absent on legacy two-column tables.
@@ -784,29 +786,33 @@ where
             "Inserting MySQL event"
         );
 
-        let pool = self.get_or_init_pool().await?;
-        // At this point the pool has already been initialized (get_or_init_pool
-        // succeeded above), so get_conn() failures are always transient: the
-        // connection was lost or the pool is temporarily exhausted.
-        let mut conn = checkout_conn(&pool).await?;
-
         let event_time = extract_event_time(body)?;
         let event_data =
             std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;
 
         let quoted_table = quote_table_name(&self.args.table)?;
+        with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
+            let pool = self.get_or_init_pool().await?;
+            // At this point the pool has already been initialized (get_or_init_pool
+            // succeeded above), so get_conn() failures are always transient: the
+            // connection was lost or the pool is temporarily exhausted.
+            let mut conn = checkout_conn(&pool).await?;
 
-        if self.idempotency_supported.load(Ordering::Relaxed) {
-            let sql = mysql_insert_sql_with_event_id(&quoted_table);
-            conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        } else {
-            let sql = mysql_insert_sql_legacy(&quoted_table);
-            conn.exec_drop(sql, (event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        }
+            if self.idempotency_supported.load(Ordering::Relaxed) {
+                let sql = mysql_insert_sql_with_event_id(&quoted_table);
+                conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            } else {
+                let sql = mysql_insert_sql_legacy(&quoted_table);
+                conn.exec_drop(sql, (event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            }
+
+            Ok(())
+        })
+        .await?;
 
         self.delivery_counters.record_success();
         debug!(target_id = %self.id, "MySQL event inserted");

crates/targets/src/target/nats/mod.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
-        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
+        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
 pub(crate) use jetstream::resolve_dedup_id;
 pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
 
+const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
 #[derive(Clone)]
 pub struct NATSArgs {
     pub enable: bool,
@@ -397,9 +399,21 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        let client = self.get_or_connect().await?;
-        if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
-            let err = classify_nats_publish_error(&e);
+        let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
+            let client = self.get_or_connect().await?;
+            client
+                .publish(self.args.subject.clone(), body.into())
+                .await
+                .map_err(|err| classify_nats_publish_error(&err))?;
+
+            // publish only enqueues the message on the client's outbound channel. Flush to confirm the
+            // message reached the server before delivery is treated as successful (backlog#971).
+            client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = result {
             if is_connectivity_error(&err) {
                 self.invalidate_cached_client_connection().await;
                 self.connected.store(false, Ordering::SeqCst);
@@ -407,15 +421,6 @@ where
             return Err(err);
         }
 
-        // publish only enqueues the message on the client's outbound channel. Flush to confirm the
-        // message reached the server before delivery is treated as successful (backlog#971).
-        if let Err(e) = client.flush().await {
-            let err = classify_nats_flush_error(&e);
-            self.invalidate_cached_client_connection().await;
-            self.connected.store(false, Ordering::SeqCst);
-            return Err(err);
-        }
-
         self.delivery_counters.record_success();
         Ok(())
     }

crates/targets/src/target/postgres.rs

@@ -38,7 +38,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
-        redacted_secret,
+        redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
 /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
 /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
 const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
+/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
+const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Returns `true` for any `s3:ObjectRemoved:*` event.
 ///
@@ -730,30 +732,29 @@ where
 
         let key = resolve_payload_key(&payload, meta);
 
-        let result = match self.args.format {
-            // For the single-row `namespace` format, an object removal must
-            // delete the row rather than UPSERT it, otherwise stale state
-            // lingers in the table after the object is gone.
-            PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
-                client.execute(&self.namespace_delete_sql, &[&key]).await
+        with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
+            match self.args.format {
+                // For the single-row `namespace` format, an object removal must
+                // delete the row rather than UPSERT it, otherwise stale state
+                // lingers in the table after the object is gone.
+                PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
+                    client.execute(&self.namespace_delete_sql, &[&key]).await
+                }
+                PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
+                PostgresFormat::Access => {
+                    let event_name_str = meta.event_name.to_string();
+                    let queued_at_ms = meta.queued_at_unix_ms as i64;
+                    client
+                        .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
+                        .await
+                }
             }
-            PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
-            PostgresFormat::Access => {
-                let event_name_str = meta.event_name.to_string();
-                let queued_at_ms = meta.queued_at_unix_ms as i64;
-                client
-                    .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
-                    .await
-            }
-        };
+            .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
+        })
+        .await?;
 
-        match result {
-            Ok(_) => {
-                self.delivery_counters.record_success();
-                Ok(())
-            }
-            Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
-        }
+        self.delivery_counters.record_success();
+        Ok(())
     }
 
     /// Probes the table from `init()`. Failure is non-fatal when a queue is

crates/targets/src/target/pulsar.rs

@@ -24,8 +24,9 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -39,11 +40,15 @@ use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;
 
+const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
+
 #[derive(Clone)]
 pub struct PulsarArgs {
     pub enable: bool,
@@ -276,6 +281,23 @@ where
         self.tls_state.lock().reset();
     }
 
+    async fn clear_failed_delivery_state(&self) {
+        match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
+            Ok(mut producer) => {
+                producer.take();
+            }
+            Err(_) => {
+                warn!(
+                    target_id = %self.id,
+                    reason = "producer_cleanup_lock_timeout",
+                    "Timed out clearing the Pulsar producer after a failed delivery"
+                );
+            }
+        }
+        self.clear_cached_client();
+        self.connected.store(false, Ordering::SeqCst);
+    }
+
     async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
         // When a TLS reload adapter is attached, it drives client rebuilds
         // in the background. The inline per-send fingerprint check is skipped.
@@ -334,20 +356,30 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        self.init_producer().await?;
-        let mut guard = self.producer.lock().await;
-        let producer = guard
-            .as_mut()
-            .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
-        let receipt = producer
-            .send_non_blocking(body)
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
-        receipt
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
-        self.delivery_counters.record_success();
-        Ok(())
+        let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
+            self.init_producer().await?;
+            let mut guard = self.producer.lock().await;
+            let producer = guard
+                .as_mut()
+                .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
+            let receipt = producer
+                .send_non_blocking(body)
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
+            receipt
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
+            self.delivery_counters.record_success();
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = &result
+            && is_connectivity_error(err)
+        {
+            self.clear_failed_delivery_state().await;
+        }
+        result
     }
 }
 
@@ -525,6 +557,22 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
+        let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
+        target.connected.store(true, Ordering::SeqCst);
+        let producer_guard = target.producer.lock().await;
+        let cleanup = {
+            let target = Arc::clone(&target);
+            tokio::spawn(async move { target.clear_failed_delivery_state().await })
+        };
+
+        cleanup.await.expect("cleanup task should not panic");
+
+        assert!(!target.connected.load(Ordering::SeqCst));
+        drop(producer_guard);
+    }
+
     #[test]
     fn debug_redacts_pulsar_secret_fields() {
         let args = PulsarArgs {

crates/targets/src/target/redis.rs

@@ -26,6 +26,7 @@ use crate::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
         mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
 use tracing::{debug, info, instrument, warn};
 use url::Url;
 
+const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+
+fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
+    let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
+    let per_attempt = args
+        .connection_timeout
+        .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
+        .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
+        .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
+    per_attempt.saturating_mul(attempts)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RedisTlsPolicy {
     SystemCa,
@@ -214,6 +228,16 @@ impl RedisArgs {
             ));
         }
 
+        if self.connection_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration(
+                "Redis connection_timeout must be greater than zero".to_string(),
+            ));
+        }
+
+        if self.response_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
+        }
+
         if self.pipeline_buffer_size == Some(0) {
             return Err(TargetError::Configuration(
                 "Redis pipeline_buffer_size must be greater than zero".to_string(),
@@ -464,71 +488,97 @@ where
             "Sending Redis payload"
         );
 
-        let mut attempt = 0usize;
-        let mut last_error = None;
-        while attempt < self.args.max_retry_attempts {
-            attempt += 1;
+        let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
+            let mut attempt = 0usize;
+            let mut last_error = None;
+            while attempt < self.args.max_retry_attempts {
+                attempt += 1;
 
-            let mut publisher = self.get_or_create_publisher().await?;
-            match publisher
-                .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
+                let mut publisher = match with_delivery_deadline(
+                    connection_timeout,
+                    "Redis connection",
+                    self.get_or_create_publisher(),
+                )
                 .await
-            {
-                Ok(receiver_count) => {
-                    // PUBLISH returns the number of subscribers that received the
-                    // message. Redis pub/sub is best-effort: with zero subscribers
-                    // the event is delivered to no one, yet the durable copy is
-                    // deleted. Warn so operators relying on reliable delivery are
-                    // not silently losing events (backlog#982).
-                    if receiver_count == 0 {
+                {
+                    Ok(publisher) => publisher,
+                    Err(err) => {
+                        invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                        return Err(err);
+                    }
+                };
+                let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
+                match with_delivery_deadline(response_timeout, "Redis publish response", async {
+                    publisher
+                        .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                        .await
+                        .map_err(map_redis_error)
+                })
+                .await
+                {
+                    Ok(receiver_count) => {
+                        // PUBLISH returns the number of subscribers that received the
+                        // message. Redis pub/sub is best-effort: with zero subscribers
+                        // the event is delivered to no one, yet the durable copy is
+                        // deleted. Warn so operators relying on reliable delivery are
+                        // not silently losing events (backlog#982).
+                        if receiver_count == 0 {
+                            warn!(
+                                target_id = %self.id,
+                                channel = %self.args.channel,
+                                "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            );
+                        }
+                        debug!(
+                            target_id = %self.id,
+                            channel = %self.args.channel,
+                            attempt,
+                            receiver_count,
+                            "Event published to Redis channel"
+                        );
+                        self.delivery_counters.record_success();
+                        return Ok(());
+                    }
+                    Err(mapped) => {
+                        invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
+
                         warn!(
                             target_id = %self.id,
                             channel = %self.args.channel,
-                            "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            attempt,
+                            max_attempts = self.args.max_retry_attempts,
+                            error = %mapped,
+                            "Redis publish attempt failed"
                         );
-                    }
-                    debug!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        receiver_count,
-                        "Event published to Redis channel"
-                    );
-                    self.delivery_counters.record_success();
-                    return Ok(());
-                }
-                Err(err) => {
-                    let mapped = map_redis_error(err);
-                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
 
-                    warn!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        max_attempts = self.args.max_retry_attempts,
-                        error = %mapped,
-                        "Redis publish attempt failed"
-                    );
+                        if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                            last_error = Some(mapped);
+                            break;
+                        }
 
-                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                         last_error = Some(mapped);
-                        break;
+                        tokio::time::sleep(compute_retry_delay(
+                            attempt,
+                            self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
+                            self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
+                        ))
+                        .await;
                     }
-
-                    last_error = Some(mapped);
-                    tokio::time::sleep(compute_retry_delay(
-                        attempt,
-                        self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
-                        self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
-                    ))
-                    .await;
                 }
             }
+
+            Err(last_error.unwrap_or(TargetError::Unknown(
+                "Redis publish failed without a captured error".to_string(),
+            )))
+        })
+        .await;
+
+        if let Err(err) = &result {
+            invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
+            self.connected.store(false, Ordering::SeqCst);
         }
-
-        self.connected.store(false, Ordering::SeqCst);
-
-        Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
+        result
     }
 }
 
@@ -551,7 +601,7 @@ where
         // thus a fresh TCP+TLS handshake — on every health check (backlog#982).
         // ensure_publisher_ready already invalidates the cached manager on a
         // connectivity error so the next attempt rebuilds it.
-        match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
+        match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
             Ok(Ok(())) => {
                 self.connected.store(true, Ordering::SeqCst);
                 Ok(true)
@@ -917,6 +967,26 @@ mod tests {
         assert!(args.validate().is_err());
     }
 
+    #[test]
+    fn validate_rejects_zero_connection_timeout() {
+        let args = RedisArgs {
+            connection_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_response_timeout() {
+        let args = RedisArgs {
+            response_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
     #[test]
     fn validate_accepts_custom_ca_tls_policy() {
         let args = RedisArgs {
@@ -1237,6 +1307,23 @@ mod tests {
         assert_eq!(target.delivery_snapshot().total_messages, 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
+        let mut args = base_args();
+        args.max_retry_attempts = 1;
+        args.response_timeout = Some(Duration::from_secs(90));
+        let manager_config = build_redis_connection_manager_config(&args);
+
+        assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
+        assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
+        with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
+            tokio::time::sleep(Duration::from_secs(70)).await;
+            Ok::<_, TargetError>(())
+        })
+        .await
+        .expect("the configured delivery budget must not impose a fixed sixty-second cap");
+    }
+
     #[tokio::test]
     async fn send_body_sets_connected_false_after_retry_exhaustion() {
         let listener = match TcpListener::bind("127.0.0.1:0").await {

crates/targets/src/target/webhook.rs

@@ -952,7 +952,7 @@ mod tests {
             .expect("https webhook probe should trust configured ca");
 
         assert_eq!(resp.status(), reqwest::StatusCode::OK);
-        assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
+        assert!(resp.bytes().await.expect("read response body").is_empty());
         handle.join().expect("tls server thread");
     }
 }