fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

crates/e2e_test/src/notification_webhook_test.rs

@@ -156,7 +156,43 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
     Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
 }
 
-fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool>, thread::JoinHandle<()>), BoxError> {
+struct HttpsEventCollector {
+    endpoint: String,
+    running: Arc<AtomicBool>,
+    handle: Option<thread::JoinHandle<()>>,
+    events: mpsc::UnboundedReceiver<Value>,
+}
+
+impl HttpsEventCollector {
+    fn endpoint(&self) -> &str {
+        &self.endpoint
+    }
+
+    fn events_mut(&mut self) -> &mut mpsc::UnboundedReceiver<Value> {
+        &mut self.events
+    }
+
+    fn shutdown(&mut self) -> TestResult {
+        self.running.store(false, Ordering::Relaxed);
+        if let Ok(parsed) = self.endpoint.parse::<reqwest::Url>()
+            && let Some(port) = parsed.port()
+        {
+            let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+        }
+        if let Some(handle) = self.handle.take() {
+            handle.join().map_err(|_| "https event collector thread panicked")?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for HttpsEventCollector {
+    fn drop(&mut self) {
+        let _ = self.shutdown();
+    }
+}
+
+fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, BoxError> {
     use rustls::{
         ServerConfig,
         pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
@@ -188,48 +224,103 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool
 
     let running = Arc::new(AtomicBool::new(true));
     let server_running = Arc::clone(&running);
+    let (tx, events) = mpsc::unbounded_channel();
     let handle = thread::spawn(move || {
+        let mut connections = Vec::new();
         while server_running.load(Ordering::Relaxed) {
             match listener.accept() {
                 Ok((stream, _)) => {
                     let config = Arc::clone(&server_config);
-                    handle_https_probe(stream, config);
+                    let tx = tx.clone();
+                    connections.push(thread::spawn(move || {
+                        let _ = handle_https_request(stream, config, tx);
+                    }));
                 }
                 Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)),
                 Err(_) => break,
             }
         }
+        for connection in connections {
+            let _ = connection.join();
+        }
     });
 
-    Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle))
+    Ok(HttpsEventCollector {
+        endpoint: format!("https://{endpoint_host}:{}/events", addr.port()),
+        running,
+        handle: Some(handle),
+        events,
+    })
 }
 
-fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc<rustls::ServerConfig>) {
-    use std::io::{Read, Write};
-
-    let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
-    let _ = stream.set_write_timeout(Some(Duration::from_secs(5)));
-    let Ok(connection) = rustls::ServerConnection::new(server_config) else {
-        return;
+fn read_sync_http_message<R: std::io::Read>(stream: &mut R) -> Result<(String, Vec<u8>), BoxError> {
+    let mut buffer = Vec::new();
+    let mut chunk = [0_u8; 4096];
+    let header_end = loop {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request headers were complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+        if let Some(pos) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
+            break pos;
+        }
     };
-    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
-    let mut buf = [0u8; 1024];
-    if tls_stream.read(&mut buf).is_err() {
-        return;
+
+    let header_text = std::str::from_utf8(&buffer[..header_end])?;
+    let mut lines = header_text.split("\r\n");
+    let method = lines
+        .next()
+        .and_then(|line| line.split_whitespace().next())
+        .ok_or("missing request method")?
+        .to_string();
+    let mut content_length = 0usize;
+    for line in lines {
+        if let Some((name, value)) = line.split_once(':')
+            && name.trim().eq_ignore_ascii_case("content-length")
+        {
+            content_length = value.trim().parse()?;
+        }
     }
-    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
-    let _ = tls_stream.write_all(response.as_bytes());
-    let _ = tls_stream.flush();
+
+    let body_offset = header_end + 4;
+    while buffer.len().saturating_sub(body_offset) < content_length {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request body was complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+    }
+    Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
 }
 
-fn stop_https_event_collector(endpoint: &str, running: Arc<AtomicBool>, handle: thread::JoinHandle<()>) -> TestResult {
-    running.store(false, Ordering::Relaxed);
-    if let Ok(parsed) = endpoint.parse::<reqwest::Url>()
-        && let Some(port) = parsed.port()
-    {
-        let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+fn handle_https_request(
+    stream: std::net::TcpStream,
+    server_config: Arc<rustls::ServerConfig>,
+    tx: mpsc::UnboundedSender<Value>,
+) -> Result<(), BoxError> {
+    use std::io::Write;
+
+    stream.set_nonblocking(false)?;
+    stream.set_read_timeout(Some(Duration::from_secs(5)))?;
+    stream.set_write_timeout(Some(Duration::from_secs(5)))?;
+    let connection = rustls::ServerConnection::new(server_config)?;
+    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
+    let (method, body) = read_sync_http_message(&mut tls_stream)?;
+    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
+    tls_stream.write_all(response.as_bytes())?;
+    tls_stream.flush()?;
+    tls_stream.conn.send_close_notify();
+    while tls_stream.conn.wants_write() {
+        tls_stream.conn.write_tls(&mut tls_stream.sock)?;
+    }
+    let _ = tls_stream.sock.shutdown(std::net::Shutdown::Write);
+    if method == "POST"
+        && !body.is_empty()
+        && let Ok(event) = serde_json::from_slice(&body)
+    {
+        let _ = tx.send(event);
     }
-    handle.join().map_err(|_| "https event collector thread panicked")?;
     Ok(())
 }
 
@@ -415,17 +506,20 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s
     Err(format!("target {target_name} was not registered in admin ARNs").into())
 }
 
-async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
+async fn wait_for_target_online(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
     let url = format!("{}/rustfs/admin/v3/target/list", env.url);
     for _ in 0..40 {
         let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
         if response.status() == StatusCode::OK {
             let body: Value = serde_json::from_slice(&response.bytes().await?)?;
+            if body["notify_enabled"].as_bool() != Some(true) {
+                return Err(format!("admin target list did not report notify_enabled=true: {body}").into());
+            }
             let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| {
                 endpoints.iter().any(|endpoint| {
                     endpoint["account_id"].as_str() == Some(target_name)
                         && endpoint["service"].as_str() == Some("webhook")
-                        && endpoint["status"].as_str().is_some()
+                        && endpoint["status"].as_str() == Some("online")
                 })
             });
             if listed {
@@ -434,7 +528,7 @@ async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str)
         }
         tokio::time::sleep(Duration::from_millis(250)).await;
     }
-    Err(format!("target {target_name} was not listed in admin targets").into())
+    Err(format!("target {target_name} did not become online in admin targets").into())
 }
 
 /// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
@@ -484,11 +578,11 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
 // ---------------------------------------------------------------------------
 
 /// Regression for rustfs#5052: with the notify module enabled through
-/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted
-/// and remain visible in the admin target list.
+/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become
+/// online and receive a real S3 event POST.
 #[tokio::test]
 #[serial]
-async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult {
+async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult {
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
@@ -496,22 +590,40 @@ async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult
         .await?;
 
     let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
-    let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?;
+    let mut collector = spawn_https_event_collector(&ca_path)?;
     let target = "peri1https";
+    let bucket = "peri1-https-events";
+    let key = "uploads/https.dat";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
 
     configure_webhook_target_with_key_values(
         &env,
         target,
         vec![
-            ("endpoint", endpoint.clone()),
+            ("endpoint", collector.endpoint().to_string()),
             ("client_ca", ca_path.to_string_lossy().into_owned()),
         ],
     )
     .await?;
-    wait_for_target_listed(&env, target).await?;
+    wait_for_target_online(&env, target).await?;
+    wait_for_target_registered(&env, target).await?;
+    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"https webhook event body"))
+        .send()
+        .await?;
+    let event = wait_for_event(collector.events_mut(), key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    assert_eq!(event["EventName"].as_str(), Some("s3:ObjectCreated:Put"));
+    assert_eq!(event["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
+    assert_eq!(event_key(&event).as_deref(), Some(key));
 
     env.stop_server();
-    stop_https_event_collector(&endpoint, running, handle)?;
+    collector.shutdown()?;
     Ok(())
 }
 

crates/ecstore/src/api/mod.rs

@@ -241,8 +241,10 @@ pub mod config {
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
             ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
-            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, save_config, save_config_with_opts,
-            save_server_config, try_migrate_server_config,
+            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
+            read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
+            save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
+            with_server_config_read_lock, with_server_config_write_lock,
         };
     }
 
@@ -383,9 +385,9 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
-        SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
+        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
+        gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
         node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
         sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
         verify_tonic_rpc_signature,

crates/ecstore/src/cluster/rpc/mod.rs

@@ -37,8 +37,8 @@ pub use http_auth::{
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub use peer_rest_client::{
-    PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-    ScannerPeerActivity,
+    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
 pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};

crates/ecstore/src/config/com.rs

@@ -17,12 +17,15 @@ use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::runtime::sources as runtime_sources;
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::{
     admin::StorageAdminApi,
     heal::HealOperations,
+    namespace::NamespaceLocking,
     object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete},
     range::HTTPRangeSpec,
 };
+use crate::store::ECStore;
 use http::HeaderMap;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_config::audit::{
@@ -45,10 +48,94 @@ use serde_json::{Map, Value};
 use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 use std::sync::{Arc, RwLock};
+use tokio::sync::RwLock as AsyncRwLock;
 use tracing::{debug, error, info, instrument, warn};
 
 pub const CONFIG_PREFIX: &str = "config";
-const CONFIG_FILE: &str = "config.json";
+const SERVER_CONFIG_OBJECT: &str = "config/config.json";
+
+// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
+// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
+static SERVER_CONFIG_LOCK: LazyLock<AsyncRwLock<()>> = LazyLock::new(|| AsyncRwLock::new(()));
+
+fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
+    let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    Error::other(format!("{operation} task {outcome}"))
+}
+
+/// Runs one complete server-config transaction while holding both the local
+/// process guard and the distributed namespace write lock for `config.json`.
+/// The operation must use the corresponding no-lock read/save functions.
+pub async fn with_server_config_write_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
+        let _local_guard = SERVER_CONFIG_LOCK.write().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config write", error))?
+}
+
+/// Reads and synchronously publishes one server-config snapshot while holding
+/// a shared namespace lock. Concurrent peer reloads may proceed together, but
+/// no writer can interleave between the snapshot read and generation accept.
+pub async fn with_server_config_read_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
+        let _local_guard = SERVER_CONFIG_LOCK.read().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config read", error))?
+}
+
+/// Runs a cancellation-safe transaction under the distributed write lock for
+/// one metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_write_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object write", error))?
+}
+
+/// Runs one read-and-publish transaction under the distributed read lock for
+/// a metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_read_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object read", error))?
+}
 
 /// Environment variable gating the startup fallback to the default server
 /// config when the persisted `config.json` object is corrupt beyond repair
@@ -393,6 +480,31 @@ where
     .await
 }
 
+pub async fn save_config_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_config_with_opts(
+        api,
+        file,
+        data,
+        &ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await
+}
+
 #[instrument(skip(api))]
 pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
 where
@@ -462,8 +574,17 @@ where
     Ok(cfg)
 }
 
+async fn new_and_save_server_config_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: EcstoreObjectIO + StorageAdminApi,
+{
+    let cfg = new_server_config();
+    save_server_config_no_lock(api, &cfg).await?;
+    Ok(cfg)
+}
+
 fn get_config_file() -> String {
-    format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
+    SERVER_CONFIG_OBJECT.to_string()
 }
 
 fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
@@ -1267,13 +1388,17 @@ where
 }
 
 /// Handle the situation where the configuration file does not exist, create and save a new configuration
-async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
+async fn handle_missing_config<S>(api: Arc<S>, context: &str, namespace_lock_held: bool) -> Result<Config>
 where
     S: EcstoreObjectIO + StorageAdminApi,
 {
     warn!("Configuration not found ({}): Start initializing new configuration", context);
     let cfg = if runtime_sources::first_cluster_node_is_local().await {
-        new_and_save_server_config(api.clone()).await?
+        if namespace_lock_held {
+            new_and_save_server_config_no_lock(api.clone()).await?
+        } else {
+            new_and_save_server_config(api.clone()).await?
+        }
     } else {
         new_server_config()
     };
@@ -1288,6 +1413,46 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
 }
 
 pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, false).await
+}
+
+/// Reads the server config while an upper layer holds the namespace write lock
+/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching
+/// no-lock save path and therefore cannot recursively acquire the same lock.
+pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, true).await
+}
+
+/// Reads an already-initialized server config while a caller owns a namespace
+/// read lock. This never initializes or migrates a missing object.
+pub async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<Config> {
+    let data = read_config_no_lock(api, SERVER_CONFIG_OBJECT).await?;
+    Ok(decode_persisted_server_config(&data)?.merge())
+}
+
+async fn read_config_without_migrate_inner<S>(api: Arc<S>, namespace_lock_held: bool) -> Result<Config>
 where
     S: ObjectIO<
             Error = Error,
@@ -1303,13 +1468,13 @@ where
 
     // Try to read the configuration file
     match read_config_no_lock(api.clone(), &config_file).await {
-        Ok(data) => read_server_config(api, &data).await,
-        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
+        Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
+        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
         Err(err) => handle_config_read_error(err, &config_file),
     }
 }
 
-async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
+async fn read_server_config<S>(api: Arc<S>, data: &[u8], namespace_lock_held: bool) -> Result<Config>
 where
     S: EcstoreObjectIO + StorageAdminApi,
 {
@@ -1324,7 +1489,9 @@ where
                 let cfg = decode_persisted_server_config(&cfg_data)?;
                 return Ok(cfg.merge());
             }
-            Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
+            Err(Error::ConfigNotFound) => {
+                return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await;
+            }
             Err(err) => return handle_config_read_error(err, &config_file),
         }
     }
@@ -1534,6 +1701,38 @@ fn fallback_server_config_after_corruption(err: Error, config_file: &str, recove
 }
 
 pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, false).await
+}
+
+/// Saves the server config while an upper layer holds the namespace write
+/// lock for [`SERVER_CONFIG_OBJECT`].
+pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, true).await
+}
+
+async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, no_lock: bool) -> Result<()>
 where
     S: ObjectIO<
             Error = Error,
@@ -1546,7 +1745,11 @@ where
         >,
 {
     let config_file = get_config_file();
-    let existing = match read_config(api.clone(), &config_file).await {
+    let existing = match if no_lock {
+        read_config_no_lock(api.clone(), &config_file).await
+    } else {
+        read_config(api.clone(), &config_file).await
+    } {
         Ok(v) => Some(v),
         Err(Error::ConfigNotFound) => None,
         Err(err) => {
@@ -1570,7 +1773,21 @@ where
         return Ok(());
     }
 
-    save_config(api, &config_file, data).await
+    if no_lock {
+        save_config_with_opts(
+            api,
+            &config_file,
+            data,
+            &ObjectOptions {
+                max_parity: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    } else {
+        save_config(api, &config_file, data).await
+    }
 }
 
 pub async fn lookup_configs<S>(cfg: &mut Config, api: Arc<S>) -> Result<()>
@@ -1630,9 +1847,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        apply_dynamic_config_for_sub_sys_with, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata,
-        storage_class_kvs_mut,
+        apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob,
+        encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty,
+        read_config_with_metadata, storage_class_kvs_mut,
     };
     use crate::config::{audit, notify, oidc};
     use crate::disk::endpoint::Endpoint;
@@ -1652,6 +1869,17 @@ mod tests {
     use rustfs_config::{
         DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
     };
+
+    #[tokio::test]
+    async fn config_task_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = config_task_join_error("server config write", join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
     use rustfs_lock::client::LockClient;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};

crates/ecstore/src/services/notification_sys.rs

@@ -253,7 +253,7 @@ impl NotificationSys {
         join_all(futures).await
     }
 
-    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+    async fn signal_dynamic_config(&self, sub_sys: &str, dry_run: bool) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {
             let sub_sys = sub_sys.to_string();
@@ -263,7 +263,7 @@ impl NotificationSys {
                         .signal_service(
                             crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC,
                             &sub_sys,
-                            false,
+                            dry_run,
                             SystemTime::UNIX_EPOCH,
                         )
                         .await
@@ -288,6 +288,14 @@ impl NotificationSys {
         join_all(futures).await
     }
 
+    pub async fn preflight_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, true).await
+    }
+
+    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, false).await
+    }
+
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/ecstore/src/store/object.rs

@@ -872,6 +872,8 @@ impl ECStore {
         let idx = if opts.data_movement && opts.version_id.is_some() {
             self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false)
                 .await?
+        } else if opts.no_lock {
+            self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
         } else {
             self.get_pool_idx(bucket, &object, data.size()).await?
         };

crates/notify/Cargo.toml

@@ -44,6 +44,7 @@ serde = { workspace = true, features = ["derive"] }
 starshard = { workspace = true, features = ["rayon", "async", "serde"] }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
+tokio-util = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
 metrics = { workspace = true }
@@ -62,6 +63,7 @@ axum = { workspace = true }
 rustfs-utils = { workspace = true, features = ["path"] }
 serde_json = { workspace = true, features = ["raw_value"] }
 criterion = { workspace = true, features = ["html_reports"] }
+tempfile = { workspace = true }
 
 [lints]
 workspace = true

crates/notify/src/config_manager.rs

@@ -13,89 +13,83 @@
 // limitations under the License.
 
 use crate::{
-    Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
+    NotificationError,
+    lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
+    registry::TargetRegistry,
+    resolve_notify_object_store_handle,
+    rule_engine::NotifyRuleEngine,
     runtime_facade::NotifyRuntimeFacade,
+    with_notify_server_config_read_lock, with_notify_server_config_write_lock,
 };
 use rustfs_config::notify::{
     NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
     NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::server_config::{Config, KVS};
-use rustfs_targets::{Target, arn::TargetID};
-use std::sync::{Arc, LazyLock};
-use tokio::sync::{Mutex, RwLock};
+use rustfs_targets::arn::TargetID;
+use std::sync::Arc;
+use tokio::sync::RwLock;
 use tracing::{debug, info};
 
-/// Serializes the read-modify-write sequence over the persisted notify server
-/// config. The persisted config is a single process-global resource (there is
-/// only one backing object store), so without this guard two concurrent updates
-/// can both read the same base config, apply disjoint changes, and race their
-/// full-config writes — the later write silently overwrites the earlier one,
-/// losing updates. Holding this mutex across the whole read→modify→write makes
-/// concurrent updates apply serially so every change is preserved.
-///
-/// The lock is only ever acquired inside `update_server_config`; it never nests
-/// with the per-manager `config` RwLock (the in-memory reload runs after this
-/// guard is released), so it introduces no lock-ordering risk.
-static NOTIFY_CONFIG_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
-
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_CONFIG: &str = "config";
-const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
 
 #[derive(Debug)]
 enum NotifyConfigStoreError {
+    Lock(String),
     StorageNotAvailable,
     Read(String),
     Save(String),
 }
 
-async fn update_server_config<F>(modifier: F) -> Result<Option<Config>, NotifyConfigStoreError>
+async fn update_server_config<F>(
+    modifier: F,
+    lifecycle: NotifyLifecycleCoordinator,
+) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
     let Some(store) = resolve_notify_object_store_handle() else {
         return Err(NotifyConfigStoreError::StorageNotAvailable);
     };
 
+    let store_for_read = store.clone();
     let store_for_save = store.clone();
-    serialized_read_modify_write(
-        modifier,
-        move || async move {
-            crate::read_notify_server_config_without_migrate(store)
-                .await
-                .map_err(NotifyConfigStoreError::Read)
-        },
-        move |config| async move {
-            crate::save_notify_server_config(store_for_save, &config)
-                .await
-                .map_err(NotifyConfigStoreError::Save)
-        },
-    )
+    with_notify_server_config_write_lock(store, move || {
+        read_modify_write(
+            modifier,
+            move || async move {
+                crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
+                    .await
+                    .map_err(NotifyConfigStoreError::Read)
+            },
+            move |config| async move {
+                crate::save_notify_server_config_no_lock(store_for_save, &config)
+                    .await
+                    .map_err(NotifyConfigStoreError::Save)
+            },
+            move |config| lifecycle.update_config(config),
+        )
+    })
     .await
+    .map_err(NotifyConfigStoreError::Lock)?
 }
 
-/// Runs a `read → modify → write` over the persisted notify config while holding
-/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber
-/// each other's changes (backlog#968). `read`/`save` are injected so the exact
-/// production serialization path can be exercised in tests without a live store.
-async fn serialized_read_modify_write<F, R, RFut, S, SFut>(
+async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
     mut modifier: F,
     read: R,
     save: S,
-) -> Result<Option<Config>, NotifyConfigStoreError>
+    publish: P,
+) -> Result<Option<T>, NotifyConfigStoreError>
 where
     F: FnMut(&mut Config) -> bool,
     R: FnOnce() -> RFut,
     RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
     S: FnOnce(Config) -> SFut,
     SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
+    P: FnOnce(Config) -> T,
 {
-    // Hold the RMW lock across the entire read→modify→write so concurrent
-    // updates serialize and cannot clobber each other's changes (backlog#968).
-    let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await;
-
     let mut new_config = read().await?;
 
     if !modifier(&mut new_config) {
@@ -104,7 +98,7 @@ where
 
     save(new_config.clone()).await?;
 
-    Ok(Some(new_config))
+    Ok(Some(publish(new_config)))
 }
 
 pub(crate) fn notify_configuration_hint() -> String {
@@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) ->
 #[derive(Clone)]
 pub struct NotifyConfigManager {
     config: Arc<RwLock<Config>>,
-    registry: Arc<TargetRegistry>,
+    lifecycle: NotifyLifecycleCoordinator,
     rule_engine: NotifyRuleEngine,
-    runtime_facade: NotifyRuntimeFacade,
 }
 
 impl NotifyConfigManager {
@@ -154,64 +147,21 @@ impl NotifyConfigManager {
         rule_engine: NotifyRuleEngine,
         runtime_facade: NotifyRuntimeFacade,
     ) -> Self {
+        let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
         Self {
             config,
-            registry,
+            lifecycle,
             rule_engine,
-            runtime_facade,
         }
     }
 
+    pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
+        self.lifecycle.clone()
+    }
+
     pub async fn init(&self) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initializing",
-            "notify runtime lifecycle"
-        );
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
-        }
-
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initialized",
-            "notify runtime lifecycle"
-        );
-        Ok(())
+        let config = self.config.read().await.clone();
+        self.lifecycle.set_mode(true, Some(config)).wait().await
     }
 
     pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
@@ -227,6 +177,7 @@ impl NotifyConfigManager {
 
         let ttype = target_type.to_lowercase();
         let tname = target_id.id.to_lowercase();
+        let log_target_id = target_id.clone();
 
         // Guard against orphaning bucket notification rules (backlog#979). Removing a
         // target while a bucket rule still references it would leave a dangling
@@ -240,7 +191,7 @@ impl NotifyConfigManager {
             )));
         }
 
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             let mut changed = false;
             if let Some(targets_of_type) = config.0.get_mut(&ttype) {
                 if targets_of_type.remove(&tname).is_some() {
@@ -249,7 +200,7 @@ impl NotifyConfigManager {
                             component = LOG_COMPONENT_NOTIFY,
                         subsystem = LOG_SUBSYSTEM_CONFIG,
                         action = "remove_target",
-                        target_id = %target_id,
+                        target_id = %log_target_id,
                         result = "removed",
                         "notify config update"
                     );
@@ -265,7 +216,7 @@ impl NotifyConfigManager {
                     component = LOG_COMPONENT_NOTIFY,
                     subsystem = LOG_SUBSYSTEM_CONFIG,
                     action = "remove_target",
-                    target_id = %target_id,
+                    target_id = %log_target_id,
                     result = "not_found",
                     "notify config update"
                 );
@@ -287,7 +238,7 @@ impl NotifyConfigManager {
         );
         let ttype = target_type.to_lowercase();
         let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
             true
         })
@@ -316,7 +267,7 @@ impl NotifyConfigManager {
             )));
         }
 
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             let mut changed = false;
             if let Some(targets) = config.0.get_mut(&ttype) {
                 if targets.remove(&tname).is_some() {
@@ -332,8 +283,8 @@ impl NotifyConfigManager {
                     component = LOG_COMPONENT_NOTIFY,
                     subsystem = LOG_SUBSYSTEM_CONFIG,
                     action = "remove_target_config",
-                    target_type = %target_type,
-                    target_name = %target_name,
+                    target_type = %ttype,
+                    target_name = %tname,
                     result = "not_found",
                     "notify config update"
                 );
@@ -348,81 +299,62 @@ impl NotifyConfigManager {
     }
 
     pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloading",
-            "notify runtime lifecycle"
-        );
+        self.lifecycle.set_mode(true, Some(new_config)).wait().await
+    }
 
-        self.update_config(new_config.clone()).await;
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        let Some(store) = resolve_notify_object_store_handle() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to load target configuration: server storage not initialized".to_string(),
+            ));
+        };
+        self.reload_persisted_config_from_store(store).await
+    }
 
-        // Stop the currently running replay workers *before* activating the new ones
-        // (backlog#970). Each replay worker drains a per-target persisted store; if the
-        // new workers start while the old ones are still running against the same
-        // stores, both drain the same queues and re-deliver events. `replace_targets`
-        // below also stops workers, but only after `activate_targets_with_replay` has
-        // already spawned the new ones — so without this explicit stop-before-start
-        // there is a window where old and new workers overlap. (The full "signal +
-        // join" shutdown lives in the targets crate and is tracked under the same
-        // issue; this reorders the notify-side lifecycle.)
-        self.runtime_facade.stop_replay_workers().await;
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
         }
 
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloaded",
-            "notify runtime lifecycle"
-        );
+        let read_store = store.clone();
+        let config_cache = self.config.clone();
+        let lifecycle = self.lifecycle.clone();
+        let transition = with_notify_server_config_read_lock(store, move || async move {
+            let config = crate::read_existing_notify_server_config_no_lock(read_store)
+                .await
+                .map_err(NotificationError::ReadConfig)?;
+            Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
+                None
+            } else {
+                Some(lifecycle.update_config(config))
+            })
+        })
+        .await
+        .map_err(NotificationError::StorageNotAvailable)??;
+
+        if let Some(transition) = transition {
+            transition.wait().await?;
+        }
         Ok(())
     }
 
-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
     where
-        F: FnMut(&mut Config) -> bool,
+        F: FnMut(&mut Config) -> bool + Send + 'static,
     {
-        let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err {
-            NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ),
-            NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
-            NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
-        })?
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
+            .await
+            .map_err(|err| match err {
+                NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
+                NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
+                    "Failed to save target configuration: server storage not initialized".to_string(),
+                ),
+                NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
+                NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
+            })?
         else {
             debug!(
                 event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -443,15 +375,15 @@ impl NotifyConfigManager {
             result = "updated",
             "notify config update"
         );
-        self.reload_config(new_config).await
+        transition.wait().await
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write};
-    use crate::NotificationError;
+    use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
     use crate::rules::RulesMap;
+    use crate::{NotificationError, NotificationRuntimeState};
     use crate::{
         integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
         runtime_facade::NotifyRuntimeFacade,
@@ -464,7 +396,10 @@ mod tests {
     use rustfs_s3_types::EventName;
     use rustfs_targets::ReplayWorkerManager;
     use rustfs_targets::arn::TargetID;
-    use std::sync::Arc;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    };
     use tokio::sync::{RwLock, Semaphore};
 
     fn build_manager() -> NotifyConfigManager {
@@ -474,9 +409,10 @@ mod tests {
         let rule_engine = NotifyRuleEngine::new();
         let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
         let target_list = notifier.target_list();
-        let runtime_facade = NotifyRuntimeFacade::new(
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
             target_list,
             Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
             Arc::new(Semaphore::new(4)),
             metrics,
         );
@@ -524,78 +460,39 @@ mod tests {
             .reload_config(Config::default())
             .await
             .expect("reload_config should succeed for empty targets");
+        assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
     }
 
-    // Regression test for backlog#968: the read-modify-write over the persisted
-    // notify config must be serialized. Many tasks concurrently add a distinct
-    // target through the same production RMW path (`serialized_read_modify_write`,
-    // which holds the global RMW lock across read→modify→write) against a shared
-    // in-memory backend. Every update must survive — no lost updates. Without the
-    // lock, concurrent tasks would read the same base config and clobber each
-    // other's writes, leaving only a subset of targets.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_config_updates_preserve_all_targets() {
-        // Shared in-memory stand-in for the persisted config backend.
-        let backend = Arc::new(RwLock::new(Config::default()));
-        const TASKS: usize = 32;
+    #[tokio::test]
+    async fn read_modify_write_publishes_only_after_save() {
+        let saved = Arc::new(AtomicBool::new(false));
+        let saved_by_writer = saved.clone();
+        let observed_by_publisher = saved.clone();
 
-        let mut handles = Vec::with_capacity(TASKS);
-        for idx in 0..TASKS {
-            let backend = backend.clone();
-            handles.push(tokio::spawn(async move {
-                let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase();
-                let tname = format!("target-{idx}");
-
-                let read_backend = backend.clone();
-                let save_backend = backend.clone();
-
-                let result = serialized_read_modify_write(
-                    |config: &mut Config| {
-                        config
-                            .0
-                            .entry(ttype.clone())
-                            .or_default()
-                            .insert(tname.clone(), KVS::default());
-                        true
-                    },
-                    move || async move {
-                        let snapshot = read_backend.read().await.clone();
-                        // Yield inside the critical section to widen the race window:
-                        // if the RMW were not serialized, other tasks would read this
-                        // same base snapshot and their writes would clobber ours.
-                        tokio::task::yield_now().await;
-                        Ok::<_, NotifyConfigStoreError>(snapshot)
-                    },
-                    move |config: Config| async move {
-                        *save_backend.write().await = config;
-                        Ok::<_, NotifyConfigStoreError>(())
-                    },
-                )
-                .await
-                .expect("serialized RMW should succeed");
-                assert!(result.is_some(), "modifier reported a change, expected Some(config)");
-            }));
-        }
-
-        for handle in handles {
-            handle.await.expect("update task should not panic");
-        }
-
-        let final_config = backend.read().await;
-        let webhook_targets = final_config
-            .0
-            .get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase())
-            .expect("webhook subsystem should exist after updates");
-
-        assert_eq!(
-            webhook_targets.len(),
-            TASKS,
-            "all concurrent target additions must be preserved (no lost updates)"
-        );
-        for idx in 0..TASKS {
-            let tname = format!("target-{idx}");
-            assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost");
-        }
+        read_modify_write(
+            |config| {
+                config
+                    .0
+                    .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+                    .or_default()
+                    .insert("primary".to_string(), KVS::default());
+                true
+            },
+            || async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
+            move |_config| async move {
+                saved_by_writer.store(true, Ordering::Release);
+                Ok::<_, NotifyConfigStoreError>(())
+            },
+            move |_config| {
+                assert!(
+                    observed_by_publisher.load(Ordering::Acquire),
+                    "publication must observe the completed save"
+                );
+            },
+        )
+        .await
+        .expect("read-modify-write should succeed")
+        .expect("changed config should publish");
     }
 
     #[test]

crates/notify/src/error.rs

@@ -15,6 +15,7 @@
 use rustfs_targets::{TargetError, arn::TargetID};
 use std::io;
 use thiserror::Error;
+use tokio::task::JoinError;
 
 /// Errors related to the notification system's lifecycle.
 #[derive(Debug, Error)]
@@ -70,3 +71,24 @@ pub enum NotificationError {
     #[error("Storage not available: {0}")]
     StorageNotAvailable(String),
 }
+
+pub(crate) fn transition_join_error(error: JoinError) -> NotificationError {
+    let reason = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    NotificationError::Initialization(format!("Notification lifecycle transition task {reason}"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::transition_join_error;
+
+    #[tokio::test]
+    async fn transition_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = transition_join_error(join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+}

crates/notify/src/global.rs

@@ -14,38 +14,86 @@
 
 use crate::{
     BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot,
-    NotificationSystem, NotificationTargetMetricSnapshot,
+    NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error,
 };
 use rustfs_config::server_config::Config;
 use rustfs_s3_types::EventName;
 use rustfs_targets::arn::TargetID;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak};
 use tracing::error;
 
 static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
+static LEGACY_INITIALIZATION: LazyLock<Mutex<Option<LegacyInitialization>>> = LazyLock::new(|| Mutex::new(None));
+
+enum LegacyInitialization {
+    Initializing(Weak<NotificationSystem>),
+    Retryable(Weak<NotificationSystem>),
+    Initialized,
+}
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_GLOBAL: &str = "global";
 const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
 
-/// Initialize the global notification system with the given configuration.
-/// This function should only be called once throughout the application life cycle.
-pub async fn initialize(config: Config) -> Result<(), NotificationError> {
-    // `new` is synchronous and responsible for creating instances
-    let system = NotificationSystem::new(config);
-    // `init` is asynchronous and responsible for performing I/O-intensive initialization
-    system.init().await?;
+fn notification_system_or_init(config: Config) -> Arc<NotificationSystem> {
+    NOTIFICATION_SYSTEM
+        .get_or_init(|| Arc::new(NotificationSystem::new(config)))
+        .clone()
+}
 
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(losing_system) => {
-            // Another initializer won the race. `init()` above already started this
-            // system's targets and replay workers, so simply dropping it would leak
-            // those background tasks. Shut the losing instance down cleanly before
-            // reporting the conflict (backlog#984).
-            losing_system.shutdown().await;
-            Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized))
+/// Initialize the global notification system with the given configuration.
+///
+/// This preserves the historical one-shot API contract. Server lifecycle code
+/// that needs idempotent reconciliation should use [`reconcile`] instead.
+pub async fn initialize(config: Config) -> Result<(), NotificationError> {
+    let system = {
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        match legacy.as_ref() {
+            Some(LegacyInitialization::Retryable(system)) => {
+                let Some(system) = system.upgrade() else {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                };
+                if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
+            Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => {
+                return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+            }
+            None => {
+                if NOTIFICATION_SYSTEM.get().is_some() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                let system = Arc::new(NotificationSystem::new(config.clone()));
+                if NOTIFICATION_SYSTEM.set(system.clone()).is_err() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
         }
-    }
+    };
+
+    let task_system = system.clone();
+    tokio::spawn(async move {
+        let result = task_system.set_targets_enabled(true, Some(config)).await;
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        if matches!(
+            legacy.as_ref(),
+            Some(LegacyInitialization::Initializing(current))
+                if current.upgrade().is_some_and(|current| Arc::ptr_eq(&current, &task_system))
+        ) {
+            *legacy = Some(if result.is_ok() {
+                LegacyInitialization::Initialized
+            } else {
+                LegacyInitialization::Retryable(Arc::downgrade(&task_system))
+            });
+        }
+        result
+    })
+    .await
+    .map_err(transition_join_error)?
 }
 
 /// Initialize the global notification system only for live in-process consumers.
@@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
 /// ListenBucketNotification clients can receive live events even when external
 /// notification targets are disabled.
 pub fn initialize_live_events() -> Result<(), NotificationError> {
-    let system = NotificationSystem::new(Config::new());
-
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)),
+    if NOTIFICATION_SYSTEM
+        .set(Arc::new(NotificationSystem::new(Config::new())))
+        .is_err()
+    {
+        return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
     }
+    Ok(())
+}
+
+/// Ensures the stable process-wide live-event container exists.
+pub fn ensure_live_events() -> Arc<NotificationSystem> {
+    notification_system_or_init(Config::new())
+}
+
+/// Ensures the stable singleton exists and reconciles its target runtime.
+pub async fn reconcile(config: Config) -> Result<(), NotificationError> {
+    let system = notification_system_or_init(config.clone());
+    system.set_targets_enabled(true, Some(config)).await
 }
 
 /// Returns a handle to the global NotificationSystem instance.

crates/notify/src/integration.rs

@@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::{EventNotifier, TargetList};
 use crate::services::NotifyServices;
 use crate::{
-    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    Event,
+    error::NotificationError,
+    lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
+    pipeline::LiveEventHistory,
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
     rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
@@ -160,11 +165,12 @@ impl NotificationMetrics {
 
 /// The notification system that integrates all components
 pub struct NotificationSystem {
-    /// The event notifier
+    /// Event dispatcher. Runtime target mutation remains lifecycle-owned.
     pub notifier: Arc<EventNotifier>,
-    /// The target registry
+    /// Target factory registry. Creating a target does not publish it.
     pub registry: Arc<TargetRegistry>,
-    /// The current configuration
+    /// The current cached configuration. Runtime publication must still go
+    /// through the lifecycle methods on this type.
     pub config: Arc<RwLock<Config>>,
     services: NotifyServices,
 }
@@ -220,10 +226,12 @@ impl NotificationSystem {
         self.services.runtime_view.get_active_targets().await
     }
 
-    /// Gets the complete Target list, including both active and inactive Targets.
-    ///
-    /// # Return
-    /// An `Arc<RwLock<TargetList>>` containing all Targets.
+    pub async fn config_snapshot(&self) -> Config {
+        self.config.read().await.clone()
+    }
+
+    /// Gets a read-only runtime container handle. Public mutation methods on
+    /// `TargetList` are intentionally unavailable outside this crate.
     pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
         self.services.runtime_view.get_all_targets()
     }
@@ -327,6 +335,43 @@ impl NotificationSystem {
         self.services.config_manager.reload_config(new_config).await
     }
 
+    /// Synchronously publishes a config generation without changing target mode.
+    pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().update_config(new_config)
+    }
+
+    /// Reconciles the cached and active configuration with the persisted
+    /// server config without changing the target-runtime mode.
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config().await
+    }
+
+    /// Reconciles from an explicitly selected storage context.
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config_from_store(store).await
+    }
+
+    /// Enables or suspends configured notification targets without replacing
+    /// the process-wide live-event container.
+    pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
+        self.publish_targets_enabled(enabled, config).wait().await
+    }
+
+    /// Synchronously accepts a target-runtime mode transition. The returned
+    /// receipt can be awaited after the caller releases its persistence lock.
+    pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().set_mode(enabled, config)
+    }
+
+    pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
+        self.services.config_manager.lifecycle().state()
+    }
+
+    /// Returns whether the latest accepted lifecycle generation is active.
+    pub fn runtime_lifecycle_is_converged(&self) -> bool {
+        self.services.config_manager.lifecycle().is_converged()
+    }
+
     /// Loads the bucket notification configuration
     pub async fn load_bucket_notification_config(
         &self,
@@ -365,9 +410,13 @@ impl NotificationSystem {
         self.services.runtime_view.runtime_status_snapshot().await
     }
 
-    // Add a method to shut down the system
     pub async fn shutdown(&self) {
-        self.services.runtime_facade.shutdown().await;
+        let _ = self.shutdown_checked().await;
+    }
+
+    /// Irreversibly terminates the notification target runtime for this process.
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.lifecycle().terminate().wait().await
     }
 }
 

crates/notify/src/lib.rs

@@ -25,6 +25,7 @@ mod event;
 pub mod factory;
 mod global;
 pub mod integration;
+mod lifecycle;
 mod notification_system_subscriber;
 pub mod notifier;
 mod pipeline;
@@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
 pub use global::{
-    initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
-    notification_target_metrics, notifier_global,
+    ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot,
+    notification_system, notification_target_metrics, notifier_global, reconcile,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState};
 pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline};
 pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
@@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade;
 pub use runtime_view::NotifyRuntimeView;
 pub use services::NotifyServices;
 pub use status_view::NotifyStatusView;
+pub use storage_api::NotifyStore;
 pub(crate) use storage_api::crate_boundary::{
-    read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+    read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+    resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+    with_notify_server_config_write_lock,
 };

crates/notify/src/notifier.rs

@@ -12,13 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
+use crate::error::NotificationError;
+use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
+use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
-use std::sync::Arc;
-use tokio::sync::{RwLock, Semaphore};
+use rustfs_targets::{SharedTarget, TargetRuntimeManager};
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
+use tokio::sync::{RwLock, Semaphore, watch};
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};
 
 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
 const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 
+async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
+    for handle in handles {
+        if let Err(e) = handle.await {
+            let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
+            error!(
+                event = EVENT_NOTIFY_DISPATCH_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_DISPATCH,
+                reason,
+                "Notify dispatch task failed"
+            );
+        }
+    }
+}
+
 pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
 
+static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
+
+pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
+    let key = Arc::as_ptr(target_list) as usize;
+    let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
+    gates.retain(|_, gate| gate.strong_count() != 0);
+    if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
+        return gate;
+    }
+    let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
+    gates.insert(key, Arc::downgrade(&gate));
+    gate
+}
+
+pub(crate) struct DirectDispatchTracker {
+    cancellation: CancellationToken,
+    inflight: watch::Sender<usize>,
+}
+
+impl DirectDispatchTracker {
+    fn new() -> Self {
+        let (inflight, _) = watch::channel(0);
+        Self {
+            cancellation: CancellationToken::new(),
+            inflight,
+        }
+    }
+
+    fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
+        self.inflight.send_modify(|count| *count += 1);
+        DirectDispatchLease {
+            tracker: Arc::clone(self),
+        }
+    }
+
+    pub(crate) async fn wait_idle(&self) {
+        let mut inflight = self.inflight.subscribe();
+        while *inflight.borrow_and_update() != 0 {
+            if inflight.changed().await.is_err() {
+                return;
+            }
+        }
+    }
+
+    fn cancel_pending(&self) {
+        self.cancellation.cancel();
+    }
+}
+
+struct DirectDispatchLease {
+    tracker: Arc<DirectDispatchTracker>,
+}
+
+impl DirectDispatchLease {
+    fn cancellation(&self) -> CancellationToken {
+        self.tracker.cancellation.clone()
+    }
+}
+
+impl Drop for DirectDispatchLease {
+    fn drop(&mut self) {
+        self.tracker.inflight.send_modify(|count| {
+            if let Some(next) = count.checked_sub(1) {
+                *count = next;
+            } else {
+                error!(
+                    event = EVENT_NOTIFY_DISPATCH_FAILED,
+                    component = LOG_COMPONENT_NOTIFY,
+                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                    reason = "direct_lease_underflow",
+                    "Notify direct dispatch lease accounting underflowed"
+                );
+            }
+        });
+    }
+}
+
 /// Resolves the effective send concurrency (semaphore permit count).
 ///
 /// A value of `0` would build a zero-permit semaphore, so `acquire` never
@@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize {
 
 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
+    dispatch_gate: Arc<RwLock<()>>,
+    enqueue_limiter: Arc<Semaphore>,
     metrics: Arc<NotificationMetrics>,
     rule_engine: NotifyRuleEngine,
     target_list: SharedNotifyTargetList,
@@ -82,10 +182,14 @@ impl EventNotifier {
     /// Returns a new instance of EventNotifier.
     pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
         let max_inflight = resolve_send_concurrency();
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
         EventNotifier {
+            dispatch_gate,
+            enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
             metrics,
             rule_engine,
-            target_list: Arc::new(RwLock::new(TargetList::new())),
+            target_list,
             send_limiter: Arc::new(Semaphore::new(max_inflight)),
         }
     }
@@ -99,6 +203,10 @@ impl EventNotifier {
         Arc::clone(&self.target_list)
     }
 
+    pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
+        self.dispatch_gate.clone()
+    }
+
     /// Returns a list of ARNs for the registered targets
     ///
     /// # Arguments
@@ -115,12 +223,9 @@ impl EventNotifier {
             .collect()
     }
 
-    /// Removes all targets
     pub async fn remove_all_bucket_targets(&self) {
         let mut target_list_guard = self.target_list.write().await;
-        // The logic for sending cancel signals via stream_cancel_senders would be removed.
-        // TargetList::clear_targets_only already handles calling target.close().
-        target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
+        target_list_guard.clear_targets_only().await;
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -155,7 +260,14 @@ impl EventNotifier {
             return;
         }
         let target_ids_len = target_ids.len();
-        let mut handles = vec![];
+        let mut deferred_handles = Vec::new();
+        let mut direct_handles = Vec::new();
+
+        // A lifecycle writer holds this gate only while handing queue-store
+        // ownership from one runtime generation to the next. Taking the read
+        // guard before cloning targets means the writer both blocks new sends
+        // and drains every save already using the old generation.
+        let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);
 
         // Use scope to limit the borrow scope of target_list
         let target_list_guard = self.target_list.read().await;
@@ -185,10 +297,17 @@ impl EventNotifier {
                     );
                     continue;
                 }
-                let limiter = self.send_limiter.clone();
+                let is_deferred = target_for_task.store().is_some();
+                let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
+                let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
+                let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
+                let limiter = if is_deferred {
+                    self.enqueue_limiter.clone()
+                } else {
+                    self.send_limiter.clone()
+                };
                 let metrics = self.metrics.clone();
                 let event_clone = event.clone();
-                let is_deferred = target_for_task.store().is_some();
                 let target_name_for_task = target_for_task.name(); // Get the name before generating the task
                 debug!(
                     event = EVENT_NOTIFY_DISPATCH_STARTED,
@@ -207,9 +326,32 @@ impl EventNotifier {
                     data: event_clone.as_ref().clone(),
                 });
                 let handle = tokio::spawn(async move {
+                    let _direct_dispatch_lease = direct_dispatch_lease;
+                    let _deferred_dispatch_guard = deferred_dispatch_guard;
                     metrics.increment_processing();
-                    let _permit = match limiter.acquire_owned().await {
-                        Ok(p) => p,
+                    let permit = if let Some(cancellation) = direct_cancellation {
+                        tokio::select! {
+                            biased;
+                            _ = cancellation.cancelled() => {
+                                metrics.decrement_processing();
+                                metrics.increment_skipped();
+                                debug!(
+                                    event = EVENT_NOTIFY_DISPATCH_SKIPPED,
+                                    component = LOG_COMPONENT_NOTIFY,
+                                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                                    target_id = %target_name_for_task,
+                                    reason = "runtime_generation_replaced",
+                                    "Skipped pending direct notify dispatch"
+                                );
+                                return;
+                            }
+                            permit = limiter.acquire_owned() => permit,
+                        }
+                    } else {
+                        limiter.acquire_owned().await
+                    };
+                    let _permit = match permit {
+                        Ok(permit) => permit,
                         Err(e) => {
                             error!(
                                 event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -218,7 +360,7 @@ impl EventNotifier {
                                 target_id = %target_name_for_task,
                                 error = %e,
                                 reason = "permit_acquire_failed",
-                                "Failed to acquire notify send permit"
+                                "Failed to acquire notify dispatch permit"
                             );
                             metrics.increment_failed();
                             return;
@@ -260,7 +402,11 @@ impl EventNotifier {
                         );
                     }
                 });
-                handles.push(handle);
+                if is_deferred {
+                    deferred_handles.push(handle);
+                } else {
+                    direct_handles.push(handle);
+                }
             } else {
                 warn!(
                     event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -276,19 +422,13 @@ impl EventNotifier {
         // target_list is automatically released here
         drop(target_list_guard);
 
-        // Wait for all tasks to be completed
-        for handle in handles {
-            if let Err(e) = handle.await {
-                error!(
-                    event = EVENT_NOTIFY_DISPATCH_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_DISPATCH,
-                    error = %e,
-                    reason = "join_failed",
-                    "Notify dispatch task failed"
-                );
-            }
-        }
+        // Every store-backed save owns a share of the generation guard, so
+        // caller cancellation cannot race lifecycle handoff with an enqueue.
+        // Direct targets own no queue store and may finish against the detached
+        // target while lifecycle progresses.
+        drop(dispatch_guard);
+        wait_for_dispatch_tasks(deferred_handles).await;
+        wait_for_dispatch_tasks(direct_handles).await;
         debug!(
             event = EVENT_NOTIFY_DISPATCH_COMPLETED,
             component = LOG_COMPONENT_NOTIFY,
@@ -320,7 +460,7 @@ impl EventNotifier {
             target_list_guard.add(target)?;
         }
 
-        info!(
+        tracing::info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
             subsystem = LOG_SUBSYSTEM_DISPATCH,
@@ -334,6 +474,7 @@ impl EventNotifier {
 
 /// A thread-safe list of targets
 pub struct TargetList {
+    direct_dispatches: Arc<DirectDispatchTracker>,
     /// Map of TargetID to Target
     runtime: TargetRuntimeManager<Event>,
 }
@@ -348,6 +489,7 @@ impl TargetList {
     /// Creates a new TargetList
     pub fn new() -> Self {
         TargetList {
+            direct_dispatches: Arc::new(DirectDispatchTracker::new()),
             runtime: TargetRuntimeManager::new(),
         }
     }
@@ -435,6 +577,20 @@ impl TargetList {
         self.runtime.status_snapshot(replay_workers)
     }
 
+    fn direct_dispatch_lease(&self) -> DirectDispatchLease {
+        self.direct_dispatches.acquire()
+    }
+
+    pub(crate) fn replace_runtime(
+        &mut self,
+        replacement: TargetRuntimeManager<Event>,
+    ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
+        let runtime = std::mem::replace(&mut self.runtime, replacement);
+        let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
+        direct_dispatches.cancel_pending();
+        (runtime, direct_dispatches)
+    }
+
     pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
         &mut self.runtime
     }
@@ -458,7 +614,7 @@ mod tests {
     use rustfs_s3_types::EventName;
     use rustfs_targets::StoreError;
     use rustfs_targets::{
-        TargetError,
+        ReplayWorkerManager, TargetError,
         store::{Key, QueueStore, Store},
         target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
     };
@@ -466,6 +622,7 @@ mod tests {
         Arc,
         atomic::{AtomicUsize, Ordering},
     };
+    use tokio::sync::Notify;
 
     #[tokio::test]
     async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
@@ -525,19 +682,44 @@ mod tests {
 
     #[derive(Clone)]
     struct TestTarget {
+        block_first_save: Option<(Arc<Notify>, Arc<Notify>)>,
+        close_calls: Arc<AtomicUsize>,
+        close_entered: Option<Arc<Notify>>,
         id: TargetID,
         enabled: bool,
         save_calls: Arc<AtomicUsize>,
+        selected_calls: Arc<AtomicUsize>,
+        store: Option<QueueStore<QueuedPayload>>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str, enabled: bool) -> Self {
             Self {
+                block_first_save: None,
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                close_entered: None,
                 id: TargetID::new(id.to_string(), name.to_string()),
                 enabled,
                 save_calls: Arc::new(AtomicUsize::new(0)),
+                selected_calls: Arc::new(AtomicUsize::new(0)),
+                store: None,
             }
         }
+
+        fn with_blocked_first_save(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.block_first_save = Some((entered, release));
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
+
+        fn with_close_observer(mut self, close_entered: Arc<Notify>) -> Self {
+            self.close_entered = Some(close_entered);
+            self
+        }
     }
 
     #[async_trait]
@@ -554,7 +736,13 @@ mod tests {
         }
 
         async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-            self.save_calls.fetch_add(1, Ordering::SeqCst);
+            let call = self.save_calls.fetch_add(1, Ordering::SeqCst);
+            if call == 0
+                && let Some((entered, release)) = &self.block_first_save
+            {
+                entered.notify_one();
+                release.notified().await;
+            }
             Ok(())
         }
 
@@ -563,11 +751,17 @@ mod tests {
         }
 
         async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(close_entered) = &self.close_entered {
+                close_entered.notify_one();
+            }
             Ok(())
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -580,10 +774,425 @@ mod tests {
         }
 
         fn is_enabled(&self) -> bool {
+            self.selected_calls.fetch_add(1, Ordering::SeqCst);
             self.enabled
         }
     }
 
+    #[tokio::test]
+    async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("gated-target", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let first_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
+            _ = std::future::ready(()) => {}
+        }
+
+        save_release.notify_one();
+        first_dispatch.await.expect("first dispatch task should finish");
+        let pause_guard = pause.await;
+
+        let replacement = TestTarget::new("gated-target", "webhook", true);
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.clear();
+            target_list
+                .add(Arc::new(replacement.clone()))
+                .expect("replacement target install should succeed");
+        }
+
+        let mut second_dispatch =
+            Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
+        tokio::select! {
+            biased;
+            _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(
+            replacement.selected_calls.load(Ordering::SeqCst),
+            0,
+            "a paused dispatch must not select a target from the replacement generation early"
+        );
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0);
+
+        drop(pause_guard);
+        second_dispatch.await;
+        assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let target =
+            TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone());
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
+        };
+        drop(pause_guard);
+        save_release.notify_one();
+        dispatch.await.expect("direct dispatch should finish after release");
+    }
+
+    #[tokio::test]
+    async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let first_entered = Arc::new(Notify::new());
+        let first_release = Arc::new(Notify::new());
+        let close_entered = Arc::new(Notify::new());
+        let target = TestTarget::new("direct-target", "webhook", true)
+            .with_blocked_first_save(first_entered.clone(), first_release.clone())
+            .with_close_observer(close_entered);
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target should install");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics.clone(),
+        );
+        let first = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        first_entered.notified().await;
+
+        // This task selects the old generation and acquires its lease before
+        // waiting for the saturated direct-send permit.
+        let second = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while target.selected_calls.load(Ordering::SeqCst) != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("both direct sends should select the old generation");
+
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+        let mut replace = Box::pin(facade.replace_targets(activation));
+        tokio::select! {
+            biased;
+            result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 0);
+
+        first_release.notify_one();
+        first.await.expect("first direct dispatch should finish");
+        second.await.expect("permit-waiting direct dispatch should be cancelled");
+        replace.await.expect("replacement should close after direct leases drain");
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(metrics.processing_count(), 0);
+        assert_eq!(metrics.processed_count(), 1);
+        assert_eq!(metrics.skipped_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn caller_abort_does_not_release_deferred_generation_lease() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target should install");
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        dispatch.abort();
+        let _ = dispatch.await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("caller abort released the deferred generation lease"),
+            _ = std::future::ready(()) => {}
+        }
+        save_release.notify_one();
+        let pause_guard = pause.await;
+        drop(pause_guard);
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_concurrency_is_bounded() {
+        const LIMIT: usize = 2;
+        const TARGETS: usize = 3;
+
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
+            ..EventNotifier::new(metrics, rule_engine.clone())
+        });
+        let entered = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let mut targets = Vec::new();
+        let mut rules_map = RulesMap::new();
+        for index in 0..TARGETS {
+            let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true)
+                .with_blocked_first_save(entered.clone(), release.clone())
+                .with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event"));
+            rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+            notifier
+                .target_list()
+                .write()
+                .await
+                .add(Arc::new(target.clone()))
+                .expect("target should install");
+            targets.push(target);
+        }
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        let total_calls = || {
+            targets
+                .iter()
+                .map(|target| target.save_calls.load(Ordering::SeqCst))
+                .sum::<usize>()
+        };
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != LIMIT {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the configured number of enqueues should enter");
+        for _ in 0..10 {
+            tokio::task::yield_now().await;
+        }
+        assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
+
+        release.notify_waiters();
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != TARGETS {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the waiting enqueue should enter after a permit is released");
+        release.notify_waiters();
+        dispatch.await.expect("all bounded enqueues should finish");
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let direct_entered = Arc::new(Notify::new());
+        let direct_release = Arc::new(Notify::new());
+        let direct =
+            TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone());
+        let deferred_entered = Arc::new(Notify::new());
+        let deferred_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let deferred = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(deferred_entered.clone(), deferred_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut direct_rules = RulesMap::new();
+        direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone());
+        rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
+        let mut deferred_rules = RulesMap::new();
+        deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone());
+        rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.add(Arc::new(direct)).expect("direct target should install");
+            target_list.add(Arc::new(deferred)).expect("deferred target should install");
+        }
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let direct_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        direct_entered.notified().await;
+        let deferred_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
+            .await
+            .expect("queue persistence must not wait behind a direct network send permit");
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
+            _ = std::future::ready(()) => {}
+        }
+        deferred_release.notify_one();
+        deferred_dispatch
+            .await
+            .expect("deferred dispatch should finish after release");
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
+        };
+        drop(pause_guard);
+
+        direct_release.notify_one();
+        direct_dispatch.await.expect("direct dispatch should finish after release");
+    }
+
     #[tokio::test]
     async fn test_send_event_skips_disabled_target() {
         let rule_engine = NotifyRuleEngine::new();

crates/notify/src/registry.rs

@@ -38,6 +38,11 @@ impl TargetRegistry {
         TargetRegistry { plugins }
     }
 
+    #[cfg(test)]
+    pub(crate) fn with_plugins(plugins: TargetPluginRegistry<Event>) -> Self {
+        Self { plugins }
+    }
+
     pub fn supports_target_type(&self, target_type: &str) -> bool {
         self.plugins.supports_target_type(target_type)
     }
@@ -67,6 +72,15 @@ impl TargetRegistry {
     ) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
         self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
     }
+
+    pub(crate) async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+    ) -> Result<(Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>), TargetError> {
+        self.plugins
+            .create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX)
+            .await
+    }
 }
 
 #[cfg(test)]

crates/notify/src/runtime_facade.rs

@@ -12,13 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
-use rustfs_targets::{
-    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+use crate::{
+    Event, NotificationError,
+    error::transition_join_error,
+    integration::NotificationMetrics,
+    notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
+};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
+    RuntimeActivation, Target, TargetRuntimeManager,
+};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
 };
-use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{RwLock, Semaphore};
+use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};
 
 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
 const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
+const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
 
+pub(crate) struct DetachedNotifyRuntime {
+    direct_dispatches: Arc<DirectDispatchTracker>,
+    runtime: TargetRuntimeManager<Event>,
+    replay_workers: ReplayWorkerManager,
+}
+
+// Multi-lock publication order: replay_workers -> target_list ->
+// publication_gate. The lifecycle may already hold dispatch_gate while
+// entering this facade; no path that needs these three locks may acquire
+// publication_gate before either runtime lock.
 #[derive(Clone)]
 pub struct NotifyRuntimeFacade {
+    dispatch_gate: Arc<RwLock<()>>,
+    legacy_terminated: Arc<AtomicBool>,
     target_list: SharedNotifyTargetList,
     replay_workers: Arc<RwLock<ReplayWorkerManager>>,
-    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+    runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
 }
 
 impl NotifyRuntimeFacade {
@@ -41,6 +64,18 @@ impl NotifyRuntimeFacade {
         concurrency_limiter: Arc<Semaphore>,
         metrics: Arc<NotificationMetrics>,
     ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
+        Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
+    }
+
+    pub(crate) fn new_with_dispatch_gate(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        dispatch_gate: Arc<RwLock<()>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
         let replay_metrics = metrics;
         let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
             Arc::new(move |event: ReplayEvent<Event>| {
@@ -97,36 +132,166 @@ impl NotifyRuntimeFacade {
         );
 
         Self {
+            dispatch_gate,
+            legacy_terminated: Arc::new(AtomicBool::new(false)),
             target_list,
             replay_workers,
             runtime_adapter: Arc::new(runtime_adapter),
         }
     }
 
+    pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
+        self.dispatch_gate.clone().write_owned().await
+    }
+
     pub async fn activate_targets_with_replay(
         &self,
         targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
     ) -> RuntimeActivation<Event> {
-        self.runtime_adapter.activate_with_replay(targets).await
+        // The compatibility pair must not start replacement replay before
+        // replace_targets has stopped and joined the current generation.
+        self.runtime_adapter.prepare_dormant_compat_activation(targets).await
     }
 
-    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
-        // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+    pub(crate) async fn prepare_targets(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<Event> {
+        self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
+    }
+
+    pub(crate) async fn open_prepared_stores(
+        &self,
+        prepared: PreparedActivation<Event>,
+    ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
+        let runtime_adapter = self.runtime_adapter.clone();
+        tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
+            .await
+            .map_err(transition_join_error)
+    }
+
+    pub(crate) async fn commit_prepared<Committed>(
+        &self,
+        opened: OpenedActivation<Event>,
+        on_committed: Committed,
+    ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
+    where
+        Committed: FnOnce(bool),
+    {
+        // The lifecycle coordinator validates the generation immediately before
+        // entering the non-cancellable handoff. Once old replay workers have
+        // been joined, this generation must publish before a later accepted
+        // intent can run; abandoning it here would leave the old runtime
+        // visible without replay workers.
         let mut replay_workers = self.replay_workers.write().await;
         let mut target_list = self.target_list.write().await;
-        self.runtime_adapter
-            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
-            .await
-            .map_err(NotificationError::Target)?;
+
+        let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
+        let fully_activated = rejected.failure_summary().is_none();
+        let (runtime, replay_workers, direct_dispatches) =
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation);
+        on_committed(fully_activated);
+        (
+            DetachedNotifyRuntime {
+                direct_dispatches,
+                runtime,
+                replay_workers,
+            },
+            rejected,
+        )
+    }
+
+    pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
+    where
+        Committed: FnOnce(),
+    {
+        // Lock order: replay_workers -> target_list. The lifecycle coordinator
+        // crosses its publication barrier before stopping the old workers, so
+        // this non-cancellable commit must publish even if a newer intent was
+        // accepted while the workers joined.
+        let mut replay_workers = self.replay_workers.write().await;
+        let mut target_list = self.target_list.write().await;
+
+        let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+        let detached = DetachedNotifyRuntime {
+            direct_dispatches,
+            runtime,
+            replay_workers: std::mem::take(&mut *replay_workers),
+        };
+        on_committed();
+        detached
+    }
+
+    pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        // A caller may supply an activation created outside the compatibility
+        // prepare method. Stop any already-running replacement workers before
+        // entering the ordered handoff; the supported activate→replace pair is
+        // dormant here and therefore never overlaps the old generation.
+        self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
+        let dispatch_guard = self.pause_dispatch().await;
+        if self.legacy_terminated.load(Ordering::Acquire) {
+            drop(dispatch_guard);
+            self.runtime_adapter
+                .close_compat_activation(activation)
+                .await
+                .map_err(NotificationError::Target)?;
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        self.stop_active_replay_workers().await;
+        let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
+        let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        let (old_runtime, old_replay_workers, old_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation)
+        };
+        drop(dispatch_guard);
+        let close_old = self.close_detached_targets(DetachedNotifyRuntime {
+            direct_dispatches: old_direct_dispatches,
+            runtime: old_runtime,
+            replay_workers: old_replay_workers,
+        });
+        let close_open_rejected = self.close_prepared(open_rejected);
+        let close_activation_rejected = self.close_prepared(activation_rejected);
+        let (close_old, close_open_rejected, close_activation_rejected) =
+            tokio::join!(close_old, close_open_rejected, close_activation_rejected);
+        close_old?;
+        close_open_rejected?;
+        close_activation_rejected?;
+        if !activation_failures.is_empty() {
+            return Err(NotificationError::Initialization(format!(
+                "one or more notification targets failed to activate: {}",
+                activation_failures.join("; ")
+            )));
+        }
         Ok(())
     }
 
     pub async fn stop_replay_workers(&self) {
-        let mut replay_workers = self.replay_workers.write().await;
-        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.stop_active_replay_workers().await;
+    }
+
+    pub(crate) async fn stop_active_replay_workers(&self) {
+        let mut detached = {
+            let mut replay_workers = self.replay_workers.write().await;
+            std::mem::take(&mut *replay_workers)
+        };
+        self.runtime_adapter.stop_replay_workers(&mut detached).await;
     }
 
     pub async fn shutdown(&self) {
+        let _ = self.shutdown_checked().await;
+    }
+
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.legacy_terminated.store(true, Ordering::Release);
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -135,7 +300,14 @@ impl NotifyRuntimeFacade {
             "notify runtime lifecycle"
         );
 
-        let active_targets = self.replay_workers.read().await.len();
+        let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+            (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
+        };
+        let active_targets = detached_replay_workers.len();
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -145,25 +317,22 @@ impl NotifyRuntimeFacade {
             "notify runtime lifecycle"
         );
 
-        {
-            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
-            let mut replay_workers = self.replay_workers.write().await;
-            let mut target_list = self.target_list.write().await;
-            if let Err(err) = self
-                .runtime_adapter
-                .shutdown(target_list.runtime_mut(), &mut replay_workers)
-                .await
-            {
-                tracing::error!(
-                    event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_RUNTIME,
-                    error = %err,
-                    "Failed to shutdown notify runtime cleanly"
-                );
-            }
+        let shutdown_result = self
+            .close_detached(DetachedNotifyRuntime {
+                direct_dispatches: detached_direct_dispatches,
+                runtime: detached_runtime,
+                replay_workers: detached_replay_workers,
+            })
+            .await;
+        if let Err(err) = &shutdown_result {
+            tracing::error!(
+                event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                error = %err,
+                "Failed to shutdown notify runtime cleanly"
+            );
         }
-        tokio::time::sleep(Duration::from_millis(500)).await;
 
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
@@ -172,6 +341,64 @@ impl NotifyRuntimeFacade {
             state = "stopped",
             "notify runtime lifecycle"
         );
+        shutdown_result
+    }
+
+    fn swap_activation(
+        target_list: &mut crate::notifier::TargetList,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<Event>,
+    ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
+        let mut replacement = TargetRuntimeManager::new();
+        for target in activation.targets {
+            replacement.add_arc(target);
+        }
+        let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
+        (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
+    }
+
+    pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
+        // Replay join is intentionally not wrapped in the target-close timeout:
+        // returning before a worker is confirmed stopped could let a
+        // replacement drain the same persistent queue concurrently.
+        self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
+    }
+
+    pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        // Direct sends are allowed to finish after the replacement runtime is
+        // published, but the old targets must remain open until every task that
+        // selected that generation has released its lease.
+        detached.direct_dispatches.wait_idle().await;
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
+            Ok(close_errors) if close_errors.is_empty() => Ok(()),
+            Ok(close_errors) => {
+                let targets = close_errors
+                    .into_iter()
+                    .map(|(target_id, _)| target_id)
+                    .collect::<Vec<_>>()
+                    .join("; ");
+                Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
+                    "Failed to close {targets}"
+                ))))
+            }
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing replaced notification targets".to_string(),
+            ))),
+        }
+    }
+
+    async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        self.stop_detached_replay(&mut detached).await;
+        self.close_detached_targets(detached).await
+    }
+
+    pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
+            Ok(result) => result.map_err(NotificationError::Target),
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing superseded notification targets".to_string(),
+            ))),
+        }
     }
 }
 
@@ -184,26 +411,50 @@ mod tests {
     };
     use async_trait::async_trait;
     use rustfs_targets::arn::TargetID;
-    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::store::{Key, QueueStore, Store};
     use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
     use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
-    use tokio::sync::{RwLock, Semaphore};
+    use tokio::sync::{Notify, RwLock, Semaphore};
 
     #[derive(Clone)]
     struct TestTarget {
+        close_entered: Option<Arc<Notify>>,
+        close_error: bool,
+        close_release: Option<Arc<Notify>>,
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        store: Option<QueueStore<QueuedPayload>>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
+                close_entered: None,
+                close_error: false,
+                close_release: None,
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                store: None,
             }
         }
+
+        fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.close_entered = Some(entered);
+            self.close_release = Some(release);
+            self
+        }
+
+        fn with_close_error(mut self) -> Self {
+            self.close_error = true;
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
     }
 
     #[async_trait]
@@ -229,11 +480,22 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(entered) = &self.close_entered {
+                entered.notify_one();
+            }
+            if let Some(release) = &self.close_release {
+                release.notified().await;
+            }
+            if self.close_error {
+                return Err(TargetError::Storage("forced close failure".to_string()));
+            }
             Ok(())
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -254,7 +516,13 @@ mod tests {
         let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
         let target_list = notifier.target_list();
         let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
-        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            replay_workers.clone(),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
         (facade, notifier, replay_workers)
     }
 
@@ -273,6 +541,26 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 0);
     }
 
+    #[tokio::test]
+    async fn compatibility_activation_stays_dormant_until_ordered_replace() {
+        let (facade, _, replay_workers) = build_facade();
+        let queue_root = tempfile::tempdir().expect("queue root");
+        let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
+        let target = TestTarget::new("primary", "webhook").with_store(store);
+
+        let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
+        assert_eq!(replay_workers.read().await.len(), 0);
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("ordered compatibility replace should succeed");
+        assert_eq!(replay_workers.read().await.len(), 1);
+        facade.shutdown_checked().await.expect("test runtime should shut down");
+    }
+
     #[tokio::test]
     async fn runtime_facade_replace_targets_commits_runtime_state() {
         let (facade, notifier, replay_workers) = build_facade();
@@ -292,4 +580,129 @@ mod tests {
         assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
         assert_eq!(replay_workers.read().await.len(), 0);
     }
+
+    #[tokio::test]
+    async fn runtime_queries_do_not_wait_for_target_close() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let close_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
+        close_release.notify_one();
+        shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect("shutdown should succeed");
+
+        let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_locks_are_released_while_replay_worker_joins() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let cancel_received = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
+        let join = tokio::spawn({
+            let cancel_received = cancel_received.clone();
+            let release = release.clone();
+            async move {
+                let _ = cancel_rx.recv().await;
+                cancel_received.notify_one();
+                release.notified().await;
+            }
+        });
+        replay_workers
+            .write()
+            .await
+            .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
+
+        let stop = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.stop_replay_workers().await }
+        });
+        cancel_received.notified().await;
+
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
+
+        release.notify_one();
+        stop.await.expect("stop task should finish");
+    }
+
+    #[tokio::test]
+    async fn shutdown_returns_close_error_after_detaching_runtime() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook").with_close_error();
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
+
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn shutdown_bounds_a_target_that_never_closes() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let never_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+        tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
+
+        let err = shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect_err("blocked close must time out");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
 }

crates/notify/src/services.rs

@@ -57,7 +57,13 @@ impl NotifyServices {
         live_event_history: Arc<RwLock<LiveEventHistory>>,
     ) -> Self {
         let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
-        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            stream_cancellers,
+            notifier.dispatch_gate(),
+            concurrency_limiter,
+            metrics.clone(),
+        );
         let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
         let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
         let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);

crates/notify/src/storage_api.rs

@@ -15,35 +15,70 @@
 use std::sync::Arc;
 
 use rustfs_ecstore::api::config::com::{
-    read_config_without_migrate as read_notify_config_without_migrate_from_backend,
-    save_server_config as save_notify_server_config_to_backend,
+    read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
+    read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
+    save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
+    with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
+    with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
 };
 use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
-pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
+pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;
 
 pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
     resolve_notify_object_store_handle_from_backend()
 }
 
-pub(crate) async fn read_notify_server_config_without_migrate(
+pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
     store: Arc<NotifyStore>,
 ) -> Result<rustfs_config::server_config::Config, String> {
-    read_notify_config_without_migrate_from_backend(store)
+    read_notify_config_without_migrate_from_backend_no_lock(store)
         .await
         .map_err(|err| err.to_string())
 }
 
-pub(crate) async fn save_notify_server_config(
+pub(crate) async fn read_existing_notify_server_config_no_lock(
+    store: Arc<NotifyStore>,
+) -> Result<rustfs_config::server_config::Config, String> {
+    read_existing_notify_config_from_backend_no_lock(store)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn save_notify_server_config_no_lock(
     store: Arc<NotifyStore>,
     config: &rustfs_config::server_config::Config,
 ) -> Result<(), String> {
-    save_notify_server_config_to_backend(store, config)
+    save_notify_server_config_to_backend_no_lock(store, config)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_write_lock_from_backend(store, operation)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_read_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_read_lock_from_backend(store, operation)
         .await
         .map_err(|err| err.to_string())
 }
 
 pub(crate) mod crate_boundary {
     pub(crate) use super::{
-        read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+        read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+        resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+        with_notify_server_config_write_lock,
     };
 }

crates/notify/tests/global_lifecycle.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::server_config::Config;
+use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile};
+use std::sync::Arc;
+
+fn assert_terminated(error: NotificationError) {
+    assert!(matches!(
+        error,
+        NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
+    ));
+}
+
+#[tokio::test]
+async fn global_singleton_survives_suspend_but_not_process_termination() {
+    let system = ensure_live_events();
+    let same_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &same_system));
+    assert!(Arc::ptr_eq(
+        &system,
+        &notification_system().expect("global notification system should exist")
+    ));
+
+    system
+        .set_targets_enabled(true, Some(Config::new()))
+        .await
+        .expect("empty target runtime should enable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .set_targets_enabled(false, None)
+        .await
+        .expect("disable should suspend targets without terminating the singleton");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
+
+    system
+        .set_targets_enabled(true, None)
+        .await
+        .expect("a suspended target runtime should be restartable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .shutdown_checked()
+        .await
+        .expect("process shutdown should terminate the target runtime");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+
+    let lazy_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &lazy_system));
+    assert_terminated(
+        reconcile(Config::new())
+            .await
+            .expect_err("lazy reconciliation must not restart a terminated process runtime"),
+    );
+    assert_terminated(
+        lazy_system
+            .reload_config(Config::new())
+            .await
+            .expect_err("config reload must not restart a terminated process runtime"),
+    );
+    assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+}

crates/notify/tests/legacy_initialize_retry.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
+use rustfs_config::server_config::{Config, KVS};
+use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT};
+use rustfs_notify::{initialize, notification_system};
+
+#[tokio::test]
+async fn failed_legacy_initialize_can_retry_the_stable_singleton() {
+    let mut invalid_target = KVS::new();
+    invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string());
+    invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
+    let mut invalid_config = Config::new();
+    invalid_config
+        .0
+        .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+        .or_default()
+        .insert("primary".to_string(), invalid_target);
+
+    initialize(invalid_config)
+        .await
+        .expect_err("invalid first target activation should fail");
+    initialize(Config::new())
+        .await
+        .expect("legacy initialize should retry after the configuration is fixed");
+
+    notification_system()
+        .expect("stable singleton should remain available")
+        .shutdown_checked()
+        .await
+        .expect("test runtime should terminate");
+}

crates/targets/Cargo.toml

@@ -18,6 +18,7 @@ rustfs-tls-runtime = { workspace = true }
 rustfs-s3-types = { workspace = true }
 rustfs-utils = { workspace = true, features = ["egress"] }
 async-trait = { workspace = true }
+futures-util = { workspace = true }
 async-nats = { workspace = true }
 deadpool-postgres = { workspace = true }
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
@@ -26,6 +27,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust
 libc = { workspace = true }
 pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
 regex = { workspace = true }
+rayon = { workspace = true }
 reqwest = { workspace = true }
 rumqttc = { workspace = true, features = ["websocket"] }
 redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
@@ -38,6 +40,7 @@ sha2 = { workspace = true }
 snap = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
+tokio-util = { workspace = true }
 tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] }
 tokio-postgres-rustls = { workspace = true }
 tracing = { workspace = true }

crates/targets/src/config/common.rs

@@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<St
         .max_by_key(|(field, _)| field.len())
 }
 
-pub(super) fn is_target_enabled(config: &KVS) -> bool {
-    config
-        .lookup(ENABLE_KEY)
-        .map(|v| {
-            EnableState::from_str(v.as_str())
-                .ok()
-                .map(|s| s.is_enabled())
-                .unwrap_or(false)
-        })
-        .unwrap_or(false)
+pub(super) fn is_target_enabled(config: &KVS) -> Result<bool, TargetError> {
+    let Some(value) = config.lookup(ENABLE_KEY) else {
+        return Ok(false);
+    };
+    EnableState::from_str(value.as_str())
+        .map(EnableState::is_enabled)
+        .map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'")))
 }
 
 pub(super) fn parse_target_bool(value: Option<&str>) -> Option<bool> {

crates/targets/src/config/instance.rs

@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::loader::collect_merged_target_configs_from_env;
+use super::loader::{
+    MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env,
+};
+use crate::TargetError;
 use crate::domain::TargetDomain;
 use rustfs_config::server_config::{Config, KVS};
 use std::collections::HashSet;
@@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances(
     normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
 }
 
+pub fn try_normalize_target_plugin_instances(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError> {
+    try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
+}
+
 pub fn normalize_target_plugin_instances_from_env<I>(
     config: &Config,
     descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
@@ -100,7 +110,7 @@ where
         .map(|field| (*field).to_string())
         .collect::<HashSet<_>>();
 
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         descriptor.subsystem,
         descriptor.route_prefix,
@@ -109,7 +119,42 @@ where
         env_vars,
     )
     .into_iter()
-    .map(|record| TargetPluginInstanceRecord {
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect()
+}
+
+pub fn try_normalize_target_plugin_instances_from_env<I>(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    env_vars: I,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let valid_fields = descriptor
+        .valid_fields
+        .iter()
+        .map(|field| (*field).to_string())
+        .collect::<HashSet<_>>();
+
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        descriptor.subsystem,
+        descriptor.route_prefix,
+        descriptor.target_type,
+        &valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect())
+}
+
+fn target_plugin_instance_record(
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    record: MergedTargetConfigRecord,
+) -> TargetPluginInstanceRecord {
+    TargetPluginInstanceRecord {
         domain: descriptor.domain,
         plugin_id: descriptor.plugin_id.to_string(),
         target_type: descriptor.target_type.to_string(),
@@ -123,8 +168,7 @@ where
             has_env_instance: record.has_env_instance,
         },
         effective_config: record.effective_config,
-    })
-    .collect()
+    }
 }
 
 pub fn normalize_legacy_target_instances(
@@ -149,8 +193,9 @@ where
 mod tests {
     use super::{
         TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
-        normalize_target_plugin_instances_from_env,
+        try_normalize_target_plugin_instances_from_env,
     };
+    use crate::TargetError;
     use crate::domain::TargetDomain;
     use crate::manifest::builtin_target_manifest;
     use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -338,9 +383,50 @@ mod tests {
         let descriptor = notify_webhook_descriptor();
         let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];
 
-        let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
+        let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone())
+            .expect("canonical normalization should succeed");
         let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);
 
         assert_eq!(canonical, compatibility);
     }
+
+    #[test]
+    fn normalize_instances_rejects_invalid_enable_value() {
+        let error = try_normalize_target_plugin_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must be propagated by the public normalizer");
+
+        match error {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() {
+        let instances = normalize_legacy_target_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(instances.len(), 2);
+        let good = instances
+            .iter()
+            .find(|instance| instance.instance_id == "good")
+            .expect("valid instance should remain present");
+        let bad = instances
+            .iter()
+            .find(|instance| instance.instance_id == "bad")
+            .expect("invalid legacy instance should remain visible");
+        assert!(good.enabled);
+        assert!(!bad.enabled);
+    }
 }

crates/targets/src/config/loader.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::common::{is_target_enabled, split_env_field_and_instance};
+use crate::TargetError;
 use rustfs_config::server_config::{Config, KVS};
 use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
 use std::collections::{HashMap, HashSet};
@@ -27,6 +28,15 @@ pub fn collect_target_configs(
     collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
 }
 
+pub fn try_collect_target_configs(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+) -> Result<Vec<(String, KVS)>, TargetError> {
+    try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
+}
+
 fn is_sensitive_target_field(field_name: &str) -> bool {
     let field_name = field_name.to_ascii_lowercase();
     field_name.contains("password")
@@ -123,7 +133,7 @@ pub fn collect_target_configs_from_env<I>(
 where
     I: IntoIterator<Item = (String, String)>,
 {
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         &format!("{route_prefix}{target_type}").to_lowercase(),
         route_prefix,
@@ -137,6 +147,30 @@ where
     .collect()
 }
 
+pub fn try_collect_target_configs_from_env<I>(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Result<Vec<(String, KVS)>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .filter(|record| record.enabled)
+    .map(|record| (record.instance_id, record.effective_config))
+    .collect())
+}
+
 pub(crate) fn collect_merged_target_configs_from_env<I>(
     config: &Config,
     section_name: &str,
@@ -144,7 +178,52 @@ pub(crate) fn collect_merged_target_configs_from_env<I>(
     target_type: &str,
     valid_fields: &HashSet<String>,
     env_vars: I,
+) -> Result<Vec<MergedTargetConfigRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| result.map_err(|(_, err)| err))
+        .collect()
+}
+
+pub(crate) fn collect_merged_target_configs_compat_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
 ) -> Vec<MergedTargetConfigRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| match result {
+            Ok(record) => record,
+            Err((record, err)) => {
+                warn!(
+                    target_type,
+                    instance_id = %record.instance_id,
+                    error = %err,
+                    "Treating target instance with invalid enable configuration as disabled"
+                );
+                record
+            }
+        })
+        .collect()
+}
+
+fn collect_merged_target_config_results_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Vec<Result<MergedTargetConfigRecord, (MergedTargetConfigRecord, TargetError)>>
 where
     I: IntoIterator<Item = (String, String)>,
 {
@@ -220,14 +299,28 @@ where
             let redacted_config = redacted_target_config(&merged_config);
             debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
         }
-        merged_configs.push(MergedTargetConfigRecord {
-            instance_id: id,
-            enabled: is_target_enabled(&merged_config),
-            effective_config: merged_config,
-            has_file_default,
-            has_file_instance,
-            has_env_default,
-            has_env_instance,
+        merged_configs.push(match is_target_enabled(&merged_config) {
+            Ok(enabled) => Ok(MergedTargetConfigRecord {
+                instance_id: id,
+                enabled,
+                effective_config: merged_config,
+                has_file_default,
+                has_file_instance,
+                has_env_default,
+                has_env_instance,
+            }),
+            Err(err) => Err((
+                MergedTargetConfigRecord {
+                    instance_id: id,
+                    enabled: false,
+                    effective_config: merged_config,
+                    has_file_default,
+                    has_file_instance,
+                    has_env_default,
+                    has_env_instance,
+                },
+                err,
+            )),
         });
     }
 
@@ -238,8 +331,9 @@ where
 mod tests {
     use super::{
         collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
-        redacted_target_config,
+        redacted_target_config, try_collect_target_configs_from_env,
     };
+    use crate::TargetError;
     use rustfs_config::notify::{
         ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
         ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX,
@@ -269,7 +363,7 @@ mod tests {
 
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -282,7 +376,8 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()),
                 ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
             ],
-        );
+        )
+        .expect("valid env target");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         assert_eq!(configs.len(), 2);
@@ -295,7 +390,7 @@ mod tests {
     #[test]
     fn collect_target_configs_discovers_enabled_instance_from_env() {
         let cfg = Config(HashMap::new());
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -307,7 +402,8 @@ mod tests {
                     "https://example.com/from-env".to_string(),
                 ),
             ],
-        );
+        )
+        .expect("valid target configs");
 
         assert_eq!(configs.len(), 1);
         assert_eq!(configs[0].0, "primary");
@@ -323,7 +419,7 @@ mod tests {
         subsystem.insert("_".to_string(), default_kvs);
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -332,7 +428,8 @@ mod tests {
                 "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
                 "https://example.com/secondary".to_string(),
             )],
-        );
+        )
+        .expect("valid target configs");
 
         assert!(configs.is_empty());
     }
@@ -361,7 +458,7 @@ mod tests {
         let cfg = Config(HashMap::new());
         let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect();
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "redis",
@@ -372,7 +469,8 @@ mod tests {
                 (format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()),
                 (format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()),
             ],
-        );
+        )
+        .expect("valid redis target config");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         let redis_config = configs.get("primary").expect("redis env target should be discovered");
@@ -383,6 +481,57 @@ mod tests {
         assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off"));
     }
 
+    #[test]
+    fn collect_target_configs_rejects_invalid_instance_enable_value() {
+        let err = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must not look like a disabled target");
+
+        match err {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() {
+        let configs = collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "good");
+    }
+
+    #[test]
+    fn collect_target_configs_preserves_whitespace_padded_legacy_value() {
+        let configs = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())],
+        )
+        .expect("the shared enable parser accepts surrounding whitespace");
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "primary");
+        assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on "));
+    }
+
     #[test]
     fn redact_target_field_value_redacts_sensitive_fields() {
         assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");

crates/targets/src/config/mod.rs

@@ -21,10 +21,11 @@ pub use instance::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use loader::{
     collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
-    collect_target_configs_from_env,
+    collect_target_configs_from_env, try_collect_target_configs, try_collect_target_configs_from_env,
 };
 pub use target_args::{
     build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,

crates/targets/src/lib.rs

@@ -43,6 +43,7 @@ pub use config::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use control_plane::{
     TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError,
@@ -65,8 +66,9 @@ pub use plugin::{
     TargetPluginRegistry, TargetRequestValidator, boxed_target,
 };
 pub use runtime::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
+    OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot,
+    RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager,
+    activate_targets_with_replay,
     adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
     init_target_and_optionally_start_replay,
     ops_diagnostics::{

crates/targets/src/plugin.rs

@@ -14,8 +14,9 @@
 
 use crate::{
     PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
-    config::collect_target_configs,
+    config::try_collect_target_configs,
     manifest::{TargetPluginManifest, builtin_target_manifest},
+    target::with_deferred_queue_store_open,
 };
 use hashbrown::HashMap;
 use rustfs_config::server_config::{Config, KVS};
@@ -317,39 +318,68 @@ where
         config: &Config,
         route_prefix: &str,
     ) -> Result<Vec<BoxedTarget<E>>, TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, false)
+            .await
+            .map(|(targets, _)| targets)
+    }
+
+    /// Creates targets while deferring queue-store open until runtime handoff.
+    /// Unlike the compatibility activation API, lifecycle preparation reports
+    /// any invalid or unconstructable configured instance so the originating
+    /// Admin request cannot report a false success.
+    pub async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, true)
+            .await
+    }
+
+    async fn create_targets_from_config_with_store_mode(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+        defer_store_open: bool,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
         let mut successful_targets = Vec::new();
-        let mut failed_targets = 0usize;
+        let mut failures = Vec::new();
 
         for (target_type, plugin) in &self.plugins {
             info!(target_type = %target_type, "Start working on target type");
-            for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
+            for (id, merged_config) in try_collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set())? {
                 info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
-                match self.create_target(target_type, id.clone(), &merged_config) {
+                let created = if defer_store_open {
+                    with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config))
+                } else {
+                    self.create_target(target_type, id.clone(), &merged_config)
+                };
+                match created {
                     Ok(target) => {
                         info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
                         successful_targets.push(target);
                     }
-                    Err(err) => {
-                        failed_targets += 1;
-                        error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
+                    Err(_) => {
+                        failures.push(format!("{target_type}/{id}: target construction failed"));
+                        error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target");
                     }
                 }
             }
         }
 
-        if failed_targets > 0 {
+        if !failures.is_empty() {
             warn!(
                 created = successful_targets.len(),
-                failed = failed_targets,
+                failed = failures.len(),
                 "Some configured targets failed to create and were skipped"
             );
         }
         info!(
             count = successful_targets.len(),
-            failed = failed_targets,
+            failed = failures.len(),
             "All target processing completed"
         );
-        Ok(successful_targets)
+        Ok((successful_targets, failures))
     }
 
     pub async fn create_activation_from_config<A>(

crates/targets/src/runtime/adapter.rs

@@ -13,21 +13,53 @@
 // limitations under the License.
 
 use super::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+    OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
+    RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
+    start_replay_worker,
 };
 use crate::plugin::PluginEvent;
-use crate::{Target, TargetError};
+use crate::{SharedTarget, Target, TargetError};
 use async_trait::async_trait;
+use rayon::prelude::*;
 use std::future::Future;
+use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
 
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
 
+const MAX_PARALLEL_STORE_OPENS: usize = 4;
+
+static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(MAX_PARALLEL_STORE_OPENS)
+        .thread_name(|index| format!("rustfs-target-store-open-{index}"))
+        .build()
+});
+
+enum StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    Accepted(SharedTarget<E>),
+    Rejected { panicked: bool, target: SharedTarget<E> },
+}
+
+fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
+        Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
+        Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
+        Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
+    }
+}
+
 /// Shared runtime contract for target plugins.
 #[async_trait]
 pub trait PluginRuntimeAdapter<E>: Send + Sync
@@ -96,6 +128,234 @@ where
             stop_log_prefix: stop_log_prefix.into(),
         }
     }
+
+    pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, None).await
+    }
+
+    pub async fn prepare_targets_cancellable(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, Some(cancellation)).await
+    }
+
+    async fn prepare_targets_inner(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: Option<&CancellationToken>,
+    ) -> PreparedActivation<E> {
+        let mut prepared = Vec::with_capacity(targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        let mut targets = targets.into_iter();
+        while let Some(target) = targets.next() {
+            match prepare_target(target, cancellation).await {
+                PrepareTargetResult::Ready(target) => prepared.push(target),
+                PrepareTargetResult::Degraded { error, target } => {
+                    drop(error);
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: initialization failed", target.id()),
+                    });
+                    prepared.push(target);
+                }
+                PrepareTargetResult::Failed { error, target } => {
+                    drop(error);
+                    let target_id = target.id().to_string();
+                    tracing::warn!(
+                        target_id,
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: initialization failed"),
+                    });
+                    rejected_targets.push(Arc::from(target));
+                }
+                PrepareTargetResult::Cancelled(target) => {
+                    prepared.push(Arc::from(target));
+                    prepared.extend(targets.map(Arc::from));
+                    break;
+                }
+            }
+        }
+        PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: prepared,
+        }
+    }
+
+    /// Opens queue stores only after the previous runtime generation has been
+    /// quiesced. Targets whose stores cannot be opened retain the established
+    /// fault-isolation behavior and are returned for lock-free shutdown.
+    pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
+        let mut accepted = Vec::with_capacity(prepared.targets.len());
+        let mut failures = prepared.failures;
+        let mut rejected = prepared.rejected_targets;
+        let outcomes = if prepared.targets.len() < 2 {
+            prepared.targets.into_iter().map(open_target_store).collect()
+        } else {
+            match STORE_OPEN_POOL.as_ref() {
+                // Vec's indexed parallel iterator preserves configuration
+                // order in collect, keeping failure summaries deterministic.
+                Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
+                Err(err) => {
+                    tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
+                    prepared.targets.into_iter().map(open_target_store).collect()
+                }
+            }
+        };
+        for outcome in outcomes {
+            match outcome {
+                StoreOpenOutcome::Accepted(target) => accepted.push(target),
+                StoreOpenOutcome::Rejected { panicked, target } => {
+                    if panicked {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_panicked",
+                            "Target queue store panicked while opening during runtime handoff"
+                        );
+                    } else {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_failed",
+                            "Failed to open target queue store during runtime handoff"
+                        );
+                    }
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: queue store open failed", target.id()),
+                    });
+                    rejected.push(target);
+                }
+            }
+        }
+        (
+            OpenedActivation { targets: accepted },
+            PreparedActivation {
+                failures,
+                rejected_targets: rejected,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
+        let mut replay_workers = ReplayWorkerManager::new();
+        let mut accepted = Vec::with_capacity(opened.targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        for target in opened.targets {
+            let target_id = target.id().to_string();
+            let replay = catch_unwind(AssertUnwindSafe(|| {
+                target.store().filter(|_| target.is_enabled()).map(|store| {
+                    start_replay_worker(
+                        store.boxed_clone(),
+                        Arc::clone(&target),
+                        Arc::clone(&self.replay_hook),
+                        self.replay_semaphore.clone(),
+                        self.batch_timeout,
+                        self.idle_sleep,
+                    )
+                })
+            }));
+            let replay = match replay {
+                Ok(replay) => replay,
+                Err(_) => {
+                    tracing::error!(
+                        target_id,
+                        reason = "replay_activation_panicked",
+                        "Target replay activation panicked during runtime handoff"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: replay activation failed"),
+                    });
+                    rejected_targets.push(target);
+                    continue;
+                }
+            };
+            (self.replay_start_observer)(&target_id, replay.is_some());
+            if let Some((cancel_tx, join)) = replay {
+                replay_workers.insert_with_handle(target_id, cancel_tx, join);
+            }
+            accepted.push(target);
+        }
+
+        (
+            RuntimeActivation {
+                replay_workers,
+                targets: accepted,
+            },
+            PreparedActivation {
+                failures,
+                rejected_targets,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    #[doc(hidden)]
+    pub async fn prepare_dormant_compat_activation(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    ) -> RuntimeActivation<E> {
+        let PreparedActivation {
+            failures,
+            rejected_targets,
+            targets,
+        } = self.prepare_targets(targets).await;
+        let rejected = PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: Vec::new(),
+        };
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
+        }
+        RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn start_dormant_compat_activation(
+        &self,
+        activation: RuntimeActivation<E>,
+    ) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
+        let prepared = PreparedActivation {
+            failures: Vec::new(),
+            rejected_targets: Vec::new(),
+            targets: activation.targets,
+        };
+        let (opened, open_rejected) = self.open_prepared_stores(prepared);
+        let (activation, activation_rejected) = self.try_activate_prepared(opened);
+        (activation, open_rejected, activation_rejected)
+    }
+
+    #[doc(hidden)]
+    pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+        self.shutdown(&mut runtime, &mut activation.replay_workers).await
+    }
+
+    pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
+            runtime.add_arc(target);
+        }
+        let mut replay_workers = ReplayWorkerManager::new();
+        self.shutdown(&mut runtime, &mut replay_workers).await
+    }
 }
 
 #[async_trait]
@@ -104,36 +364,16 @@ where
     E: PluginEvent,
 {
     async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
-        let replay_hook = Arc::clone(&self.replay_hook);
-        let replay_start_observer = Arc::clone(&self.replay_start_observer);
-        let replay_semaphore = self.replay_semaphore.clone();
-        let batch_timeout = self.batch_timeout;
-        let idle_sleep = self.idle_sleep;
-
-        activate_targets_with_replay(targets, move |target| {
-            let replay_hook = Arc::clone(&replay_hook);
-            let replay_start_observer = Arc::clone(&replay_start_observer);
-            let replay_semaphore = replay_semaphore.clone();
-
-            async move {
-                init_target_and_optionally_start_replay(
-                    target,
-                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
-                    move |store, target| {
-                        start_replay_worker(
-                            store,
-                            target,
-                            Arc::clone(&replay_hook),
-                            replay_semaphore.clone(),
-                            batch_timeout,
-                            idle_sleep,
-                        )
-                    },
-                )
-                .await
-            }
-        })
-        .await
+        let prepared = self.prepare_targets(targets).await;
+        let (opened, rejected) = self.open_prepared_stores(prepared);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
+        }
+        let (activation, rejected) = self.try_activate_prepared(opened);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
+        }
+        activation
     }
 
     async fn replace_runtime_targets(
@@ -189,7 +429,7 @@ where
         if !close_errors.is_empty() {
             let detail = close_errors
                 .into_iter()
-                .map(|(target_id, err)| format!("{target_id}: {err}"))
+                .map(|(target_id, _)| target_id)
                 .collect::<Vec<_>>()
                 .join("; ");
             return Err(TargetError::Storage(format!("Failed to close {detail}")));
@@ -200,24 +440,143 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
     use crate::PluginEvent;
     use crate::arn::TargetID;
     use crate::store::{Key, QueueStore, Store};
     use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use crate::{StoreError, Target, TargetError};
     use async_trait::async_trait;
-    use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Condvar, Mutex};
     use std::time::Duration;
     use tempfile::tempdir;
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
+
+    type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
+
+    type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
+
+    #[derive(Clone)]
+    struct TestOpenStore {
+        before_clone: BeforeOpen,
+        before_open: BeforeOpen,
+        store: QueueStore<QueuedPayload>,
+    }
+
+    impl Store<QueuedPayload> for TestOpenStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            (self.before_open)();
+            self.store.open()
+        }
+
+        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put(item)
+        }
+
+        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put_multiple(items)
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            self.store.put_raw(data)
+        }
+
+        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            self.store.get(key)
+        }
+
+        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            self.store.get_multiple(key)
+        }
+
+        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            self.store.get_raw(key)
+        }
+
+        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+            self.store.del(key)
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            self.store.delete()
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            self.store.list()
+        }
+
+        fn len(&self) -> usize {
+            self.store.len()
+        }
+
+        fn is_empty(&self) -> bool {
+            self.store.is_empty()
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            (self.before_clone)();
+            Box::new(self.clone())
+        }
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGate {
+        changed: Condvar,
+        state: Mutex<StoreOpenGateState>,
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGateState {
+        active: usize,
+        max_active: usize,
+        released: bool,
+    }
+
+    impl StoreOpenGate {
+        fn enter(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.active += 1;
+            state.max_active = state.max_active.max(state.active);
+            self.changed.notify_all();
+            while !state.released {
+                state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
+            }
+            state.active -= 1;
+        }
+
+        fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
+            let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            let (state, _) = self
+                .changed
+                .wait_timeout_while(state, timeout, |state| state.max_active < expected)
+                .unwrap_or_else(|err| err.into_inner());
+            state.max_active >= expected
+        }
+
+        fn release(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.released = true;
+            self.changed.notify_all();
+        }
+
+        fn max_active(&self) -> usize {
+            self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
+        }
+    }
 
     #[derive(Clone)]
     struct TestTarget {
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        init_calls: Arc<AtomicUsize>,
+        init_entered: Option<Arc<Notify>>,
         init_fails: bool,
-        store: Option<QueueStore<QueuedPayload>>,
+        store: Option<Arc<TestStore>>,
     }
 
     impl TestTarget {
@@ -225,6 +584,8 @@ mod tests {
             Self {
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                init_calls: Arc::new(AtomicUsize::new(0)),
+                init_entered: None,
                 init_fails: false,
                 store: None,
             }
@@ -235,11 +596,16 @@ mod tests {
             self
         }
 
+        fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
+            self.init_entered = Some(init_entered);
+            self
+        }
+
         fn with_store(mut self) -> Self {
             let dir = tempdir().expect("tempdir should be created for queue store tests");
             let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
             store.open().expect("queue store should open");
-            self.store = Some(store);
+            self.store = Some(Arc::new(store));
             self
         }
     }
@@ -271,9 +637,7 @@ mod tests {
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            self.store
-                .as_ref()
-                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+            self.store.as_deref()
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -281,6 +645,11 @@ mod tests {
         }
 
         async fn init(&self) -> Result<(), TargetError> {
+            self.init_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(init_entered) = &self.init_entered {
+                init_entered.notify_one();
+                return std::future::pending().await;
+            }
             if self.init_fails {
                 return Err(TargetError::Configuration("forced init failure".to_string()));
             }
@@ -334,6 +703,212 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 1);
     }
 
+    #[tokio::test]
+    async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert_eq!(prepared.targets.len(), 1);
+        assert!(
+            prepared
+                .failure_summary()
+                .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
+        );
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.failure_summary().is_some());
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation.replay_workers.stop_all("stop degraded target replay worker").await;
+    }
+
+    #[tokio::test]
+    async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
+        let adapter = builtin_adapter();
+        let init_entered = Arc::new(Notify::new());
+        let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
+        let first_close_calls = first.close_calls.clone();
+        let second = TestTarget::new("second", "webhook");
+        let second_close_calls = second.close_calls.clone();
+        let second_init_calls = second.init_calls.clone();
+        let cancellation = CancellationToken::new();
+        let prepare_adapter = adapter.clone();
+        let prepare_cancellation = cancellation.clone();
+        let prepare = tokio::spawn(async move {
+            prepare_adapter
+                .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
+                .await
+        });
+
+        init_entered.notified().await;
+        cancellation.cancel();
+        let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
+            .await
+            .expect("cancellation should interrupt target initialization")
+            .expect("preparation task should finish");
+        assert_eq!(prepared.targets.len(), 2);
+
+        adapter
+            .close_prepared(prepared)
+            .await
+            .expect("cancelled targets should close");
+        assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn prepared_activation_opens_store_before_starting_replay() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let queue_path = dir.path().join("queue");
+        let mut target = TestTarget::new("primary", "webhook");
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.targets.is_empty());
+        assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation
+            .replay_workers
+            .stop_all("stop prepared activation test worker")
+            .await;
+    }
+
+    #[tokio::test]
+    async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let invalid_base = dir.path().join("not-a-directory");
+        std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
+        let mut target = TestTarget::new("primary", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
+        const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
+
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let gate = Arc::new(StoreOpenGate::default());
+        let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
+        let mut expected_ids = Vec::with_capacity(TARGETS);
+        for index in 0..TARGETS {
+            let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
+            expected_ids.push(target.id.to_string());
+            let open_gate = gate.clone();
+            target.store = Some(Arc::new(TestOpenStore {
+                before_clone: Arc::new(|| {}),
+                before_open: Arc::new(move || open_gate.enter()),
+                store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
+            }));
+            targets.push(Box::new(target));
+        }
+
+        let prepared = adapter.prepare_targets(targets).await;
+        let open_adapter = adapter.clone();
+        let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
+        let wait_gate = gate.clone();
+        let reached_bound =
+            tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
+                .await
+                .expect("store-open observer should not panic");
+        gate.release();
+        let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
+        let opened_ids = opened
+            .targets
+            .iter()
+            .map(|target| target.id().to_string())
+            .collect::<Vec<_>>();
+
+        assert!(reached_bound, "store opens did not use the configured parallelism");
+        assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
+        assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
+        assert!(rejected.targets.is_empty());
+        assert!(rejected.failure_summary().is_none());
+    }
+
+    #[tokio::test]
+    async fn panicking_store_open_rejects_and_closes_only_that_target() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| {}),
+            before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store should report a generic activation failure");
+        let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
+
+        assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
+        assert!(
+            activation.replay_workers.is_empty(),
+            "a rejected target must not publish without a replay worker"
+        );
+        assert!(activation_rejected.failure_summary().is_none());
+        assert!(summary.contains("queue store open failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected after a store panic should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking-clone", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
+            before_open: Arc::new(|| {}),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
+        assert!(open_rejected.failure_summary().is_none());
+        let (activation, rejected) = adapter.try_activate_prepared(opened);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store clone should report a generic activation failure");
+
+        assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
+        assert!(activation.replay_workers.is_empty());
+        assert!(summary.contains("replay activation failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected during replay activation should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[tokio::test]
     async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
         let adapter = builtin_adapter();

crates/targets/src/runtime/mod.rs

@@ -27,11 +27,21 @@ use crate::store::{Key, Store, ensure_store_entry_raw_readable};
 use crate::target::QueuedPayload;
 use crate::target::TargetDeliverySnapshot;
 use crate::{StoreError, TargetError};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::sync::Arc;
 use std::{collections::HashMap, fmt::Debug};
 use std::{future::Future, pin::Pin, time::Duration};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}
 
 /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
 /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
@@ -68,11 +78,21 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
 pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 
-/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
-/// replay worker to observe its cancel signal and exit before it is forcibly
-/// aborted. Workers observe cancellation promptly (including during retry
-/// backoff), so this only guards against a wedged task.
-const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
+pub(crate) enum PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    Ready(SharedTarget<E>),
+    Degraded {
+        error: TargetError,
+        target: SharedTarget<E>,
+    },
+    Failed {
+        error: TargetError,
+        target: Box<dyn Target<E> + Send + Sync>,
+    },
+    Cancelled(Box<dyn Target<E> + Send + Sync>),
+}
 
 /// Tracks a running replay worker: its cancel channel and, when the worker was
 /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
@@ -113,7 +133,7 @@ impl ReplayWorkerManager {
     }
 
     /// Registers a cancel channel together with the worker's join handle so
-    /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
+    /// `stop_all` can await the worker's exit.
     pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
         self.cancellers.insert(
             target_id,
@@ -140,37 +160,36 @@ impl ReplayWorkerManager {
     }
 
     /// Stops every replay worker: it first signals cancellation to all of them,
-    /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
-    /// which the task is aborted). Signalling before joining lets all workers
-    /// wind down concurrently, and joining guarantees no worker keeps draining
-    /// the shared store after this returns — preventing duplicate delivery and
-    /// orphaned tasks across reloads and shutdown.
+    /// then strictly awaits each worker's exit. A worker already awaiting a
+    /// delivery acknowledgement is allowed to finish; aborting it at an
+    /// arbitrary deadline could leave an acknowledged queue entry undeleted and
+    /// make the replacement worker deliver it again. Signalling before joining
+    /// lets all workers wind down concurrently. Legacy joinless registrations
+    /// can only be signalled.
     pub async fn stop_all(&mut self, log_prefix: &str) {
-        let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let mut joins = std::collections::VecDeque::new();
 
         // Phase 1: signal cancellation to all workers.
-        for (target_id, handle) in &handles {
+        for (target_id, handle) in handles {
             tracing::info!(target_id = %target_id, "{log_prefix}");
-            let _ = handle.cancel_tx.send(()).await;
+            let _ = handle.cancel_tx.try_send(());
+            if let Some(join) = handle.join {
+                joins.push_back((target_id, join));
+            } else {
+                tracing::warn!(
+                    target_id = %target_id,
+                    "Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
+                );
+            }
         }
 
-        // Phase 2: await each worker's exit, forcibly aborting any that overrun.
-        for (target_id, handle) in handles.drain(..) {
-            let Some(mut join) = handle.join else {
-                continue;
-            };
-            match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
-                Ok(Ok(())) => {}
-                Ok(Err(err)) => {
-                    tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
-                }
-                Err(_) => {
-                    join.abort();
-                    tracing::warn!(
-                        target_id = %target_id,
-                        "Timed out awaiting replay worker exit; task aborted"
-                    );
-                }
+        // Phase 2: strict join. Delivery operations own their own protocol
+        // deadlines; lifecycle must not invent a shorter deadline that turns an
+        // acknowledgement race into duplicate delivery.
+        while let Some((target_id, join)) = joins.pop_front() {
+            if let Err(err) = join.await {
+                tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
             }
         }
     }
@@ -184,6 +203,55 @@ where
     pub targets: Vec<SharedTarget<E>>,
 }
 
+/// Targets whose persistent queue stores are open and are ready to start
+/// replay. This distinct stage prevents activation from skipping store open.
+pub struct OpenedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+struct TargetActivationFailure {
+    detail: String,
+}
+
+/// Targets that have completed initialization but have not started replay
+/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
+/// the previous workers before the replacement workers are spawned.
+pub struct PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    failures: Vec<TargetActivationFailure>,
+    rejected_targets: Vec<SharedTarget<E>>,
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+impl<E> PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub fn failure_summary(&self) -> Option<String> {
+        if self.failures.is_empty() {
+            return None;
+        }
+
+        Some(
+            self.failures
+                .iter()
+                .map(|failure| failure.detail.clone())
+                .collect::<Vec<_>>()
+                .join("; "),
+        )
+    }
+
+    pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
+        self.failures
+            .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct RuntimeStatusSnapshot {
     pub replay_worker_count: usize,
@@ -339,17 +407,19 @@ where
     /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
     /// runtime.
     pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
-        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        let targets = std::mem::take(&mut self.targets);
+        let mut closes = FuturesUnordered::new();
+        for (target_id, target) in targets {
+            closes.push(async move { (target_id, target.close().await) });
+        }
+
         let mut errors = Vec::new();
-        for target_id in target_ids {
-            if let Some(target) = self.targets.remove(&target_id)
-                && let Err(err) = target.close().await
-            {
+        while let Some((target_id, result)) = closes.next().await {
+            if let Err(err) = result {
                 tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
                 errors.push((target_id, err));
             }
         }
-        self.targets.clear();
         errors
     }
 
@@ -440,21 +510,16 @@ where
         SharedTarget<E>,
     ) -> (mpsc::Sender<()>, JoinHandle<()>),
 {
-    let target_id = target.id().to_string();
-    let has_store = target.store().is_some();
-
-    if let Err(err) = target.init().await {
-        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
-        if !has_store {
+    let shared = match prepare_target(target, None).await {
+        PrepareTargetResult::Ready(target) => target,
+        PrepareTargetResult::Degraded { target, .. } => target,
+        PrepareTargetResult::Failed { target, .. } => {
+            let _ = target.close().await;
             return None;
         }
-        tracing::warn!(
-            target_id = %target_id,
-            "Proceeding with store-backed target despite init failure"
-        );
-    }
-
-    let shared: SharedTarget<E> = Arc::from(target);
+        PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
+    };
+    let target_id = shared.id().to_string();
     if !shared.is_enabled() {
         on_replay_start(&target_id, false);
         return Some((shared, None));
@@ -467,6 +532,45 @@ where
     Some((shared, cancel))
 }
 
+pub(crate) async fn prepare_target<E>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    cancellation: Option<&CancellationToken>,
+) -> PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    let init_result = match cancellation {
+        Some(cancellation) => {
+            tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
+                result = target.init() => result,
+            }
+        }
+        None => target.init().await,
+    };
+
+    if let Err(err) = init_result {
+        tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
+        if !has_store {
+            return PrepareTargetResult::Failed { error: err, target };
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+        return PrepareTargetResult::Degraded {
+            error: err,
+            target: Arc::from(target),
+        };
+    }
+
+    PrepareTargetResult::Ready(Arc::from(target))
+}
+
 type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);
 
 pub async fn activate_targets_with_replay<E, F, Fut>(
@@ -584,7 +688,11 @@ async fn stream_replay_worker<E>(
                 }
                 Ok(Ok(_)) => {}
                 Err(join_err) => {
-                    tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = join_failure_reason(&join_err),
+                        "The failed-events maintenance task failed to join"
+                    );
                 }
             }
             last_prune = tokio::time::Instant::now();
@@ -835,7 +943,8 @@ mod tests {
     use crate::{Target, TargetError};
     use async_trait::async_trait;
     use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use tokio::sync::{Notify, Semaphore};
 
     #[tokio::test(start_paused = true)]
     async fn seed_interval_start_backdates_by_one_interval() {
@@ -900,14 +1009,20 @@ mod tests {
     #[derive(Clone)]
     struct TestTarget {
         id: TargetID,
+        block_on_close: Arc<AtomicBool>,
+        close_gate: Arc<Semaphore>,
         close_calls: Arc<AtomicUsize>,
+        close_started: Arc<Notify>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
                 id: TargetID::new(id.to_string(), name.to_string()),
+                block_on_close: Arc::new(AtomicBool::new(false)),
+                close_gate: Arc::new(Semaphore::new(0)),
                 close_calls: Arc::new(AtomicUsize::new(0)),
+                close_started: Arc::new(Notify::new()),
             }
         }
     }
@@ -935,6 +1050,10 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            self.close_started.notify_one();
+            if self.block_on_close.load(Ordering::SeqCst) {
+                let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
+            }
             Ok(())
         }
 
@@ -966,6 +1085,45 @@ mod tests {
         assert_eq!(close_calls.load(Ordering::SeqCst), 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let first = TestTarget::new("first", "webhook");
+        let second = TestTarget::new("second", "webhook");
+        let first_observer = first.clone();
+        let second_observer = second.clone();
+        manager.add_boxed(Box::new(first));
+        manager.add_boxed(Box::new(second));
+
+        let first_close_key = manager
+            .keys()
+            .into_iter()
+            .next()
+            .expect("two targets should have a first close key");
+        let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
+            (first_observer, second_observer)
+        } else {
+            (second_observer, first_observer)
+        };
+        blocked.block_on_close.store(true, Ordering::SeqCst);
+
+        let close_task = tokio::spawn(async move { manager.clear_and_close().await });
+
+        tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
+            .await
+            .expect("the first target close should start");
+        tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
+            .await
+            .expect("a blocked first close must not prevent the second close from starting");
+        assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
+
+        blocked.close_gate.add_permits(1);
+        let errors = close_task.await.expect("clear_and_close task should join");
+        assert!(errors.is_empty());
+        assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[test]
     fn runtime_manager_snapshots_targets() {
         let mut manager = TargetRuntimeManager::<String>::new();
@@ -1030,6 +1188,51 @@ mod tests {
         assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
+        use super::ReplayWorkerManager;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use tokio::sync::Notify;
+
+        let mut manager = ReplayWorkerManager::new();
+        let acknowledgement = Arc::new(Notify::new());
+        let worker_started = Arc::new(Notify::new());
+        let exited = Arc::new(AtomicBool::new(false));
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
+        let worker_acknowledgement = acknowledgement.clone();
+        let worker_started_signal = worker_started.clone();
+        let worker_exited = exited.clone();
+        let join = tokio::spawn(async move {
+            worker_started_signal.notify_one();
+            let _ = cancel_rx.recv().await;
+            // Model a protocol operation that has accepted the request but has
+            // not returned its acknowledgement yet. Lifecycle must not abort
+            // this future or the same durable entry can be sent twice.
+            worker_acknowledgement.notified().await;
+            worker_exited.store(true, Ordering::SeqCst);
+        });
+        manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
+        worker_started.notified().await;
+
+        let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
+            _ = std::future::ready(()) => {}
+        }
+        tokio::time::advance(std::time::Duration::from_secs(60)).await;
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
+            _ = std::future::ready(()) => {}
+        }
+
+        acknowledgement.notify_one();
+        stop.await;
+        assert!(exited.load(Ordering::SeqCst));
+        assert!(manager.is_empty());
+    }
+
     mod classifier {
         use super::super::{ReplayEvent, stream_replay_worker};
         use crate::arn::TargetID;

crates/targets/src/target/kafka.rs

@@ -24,7 +24,7 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
         open_target_queue_store, persist_queued_payload_to_store,
     },
 };
@@ -32,13 +32,84 @@ use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
 use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
-use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};
 
 pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
+const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct KafkaDeliveryAttempt<'a> {
+    armed: bool,
+    poisoned: &'a AtomicBool,
+}
+
+impl KafkaDeliveryAttempt<'_> {
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for KafkaDeliveryAttempt<'_> {
+    fn drop(&mut self) {
+        if self.armed {
+            self.poisoned.store(true, Ordering::Release);
+        }
+    }
+}
+
+fn kafka_delivery_timeout() -> TargetError {
+    TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
+}
+
+async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
+    delivery_lock: &Mutex<()>,
+    delivery_poisoned: &AtomicBool,
+    select_producer: Select,
+    deliver: Deliver,
+    invalidate: Invalidate,
+) -> Result<T, TargetError>
+where
+    P: Send,
+    T: Send,
+    Select: FnOnce() -> SelectFuture + Send,
+    SelectFuture: Future<Output = Result<P, TargetError>> + Send,
+    Deliver: FnOnce(P) -> DeliveryFuture + Send,
+    DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
+    Invalidate: Fn() -> InvalidateFuture + Send,
+    InvalidateFuture: Future<Output = ()> + Send,
+{
+    let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
+    let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    let mut attempt = KafkaDeliveryAttempt {
+        armed: true,
+        poisoned: delivery_poisoned,
+    };
+
+    if delivery_poisoned.load(Ordering::Acquire) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+
+    let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    if result.as_ref().is_err_and(is_connectivity_error) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+    attempt.disarm();
+    result
+}
 
 /// Arguments for configuring a Kafka target
 #[derive(Clone)]
@@ -233,6 +304,8 @@ where
     args: KafkaArgs,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_lock: Arc<Mutex<()>>,
+    delivery_poisoned: Arc<AtomicBool>,
     tls_state: Arc<Mutex<TargetTlsState>>,
     /// Adapter that bridges this target to the TLS reload coordinator.
     /// When `Some`, the target uses coordinator-managed material; when `None`,
@@ -291,6 +364,8 @@ where
             args,
             store: queue_store,
             producer: Arc::new(Mutex::new(None)),
+            delivery_lock: Arc::new(Mutex::new(())),
+            delivery_poisoned: Arc::new(AtomicBool::new(false)),
             tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
             tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -307,7 +382,7 @@ where
         };
 
         let mut config = AsyncProducerConfig::new()
-            .with_ack_timeout(Duration::from_secs(30))
+            .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
             .with_required_acks(acks);
 
         if let Some(security) = self.args.security_config(true)? {
@@ -388,20 +463,26 @@ where
             "Sending Kafka payload"
         );
 
-        let producer = self.get_or_build_producer().await?;
-
-        // Use "<bucket>/<object>" as the message key so all events for the same
-        // object hash to the same partition and preserve per-object ordering
-        // across multiple partitions (backlog#983).
-        let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
-        if let Err(err) = producer
-            .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
-            .await
-        {
-            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
-            return Err(mapped);
-        }
+        // rustfs-kafka-async does not validate response correlation IDs. Keep
+        // producer selection, send, and timeout invalidation serialized so a
+        // waiter cannot reuse a connection with an unread timed-out response.
+        with_serialized_kafka_delivery(
+            &self.delivery_lock,
+            &self.delivery_poisoned,
+            || self.get_or_build_producer(),
+            |producer| async move {
+                // Use "<bucket>/<object>" as the message key so all events for the same
+                // object hash to the same partition and preserve per-object ordering
+                // across multiple partitions (backlog#983).
+                let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
+                producer
+                    .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
+                    .await
+                    .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
+            },
+            || self.invalidate_cached_producer(),
+        )
+        .await?;
 
         debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
         self.delivery_counters.record_success();
@@ -415,6 +496,8 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             producer: Arc::clone(&self.producer),
+            delivery_lock: Arc::clone(&self.delivery_lock),
+            delivery_poisoned: Arc::clone(&self.delivery_poisoned),
             tls_state: Arc::clone(&self.tls_state),
             tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -559,6 +642,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::atomic::AtomicUsize;
+    use tokio::sync::Notify;
 
     fn base_args() -> KafkaArgs {
         KafkaArgs {
@@ -580,6 +665,158 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    move |selected| async move {
+                        assert_eq!(selected, 1);
+                        first_entered.notify_one();
+                        std::future::pending::<Result<usize, TargetError>>().await
+                    },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        first_entered.notified().await;
+        tokio::time::advance(Duration::from_secs(1)).await;
+        let second = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    |selected| async move { Ok(selected) },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        assert!(matches!(
+            first.await.expect("first delivery task should not panic"),
+            Err(TargetError::Timeout(_))
+        ));
+        assert_eq!(
+            second
+                .await
+                .expect("second delivery task should not panic")
+                .expect("second delivery should succeed"),
+            2,
+            "the waiter must select a fresh producer generation after timeout invalidation"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = AtomicBool::new(false);
+        let selected = Arc::new(AtomicBool::new(false));
+        let _held = delivery_lock.lock().await;
+
+        let error = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let selected = Arc::clone(&selected);
+                move || async move {
+                    selected.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            },
+            |()| async { Ok(()) },
+            || async {},
+        )
+        .await
+        .expect_err("lock admission must share the absolute delivery deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
+        assert!(!delivery_poisoned.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    || async { Ok(1usize) },
+                    move |_| async move {
+                        first_entered.notify_one();
+                        std::future::pending::<Result<(), TargetError>>().await
+                    },
+                    || async {},
+                )
+                .await
+            })
+        };
+        first_entered.notified().await;
+        first.abort();
+        assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
+        assert!(delivery_poisoned.load(Ordering::Acquire));
+
+        let selected = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let generation = Arc::clone(&generation);
+                move || async move { Ok(generation.load(Ordering::SeqCst)) }
+            },
+            |selected| async move { Ok(selected) },
+            {
+                let generation = Arc::clone(&generation);
+                move || {
+                    let generation = Arc::clone(&generation);
+                    async move { generation.store(2, Ordering::SeqCst) }
+                }
+            },
+        )
+        .await
+        .expect("the next delivery should recover from cancellation poisoning");
+
+        assert_eq!(selected, 2);
+        assert!(!delivery_poisoned.load(Ordering::Acquire));
+    }
+
     #[test]
     fn test_validate_empty_brokers() {
         let args = KafkaArgs {

crates/targets/src/target/mod.rs

@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
+use std::cell::Cell;
 use std::fmt::Formatter;
 use std::future::Future;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::thread_local;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};
 
 pub mod amqp;
@@ -111,7 +113,12 @@ where
     /// Checks if the target is active and reachable
     async fn is_active(&self) -> Result<bool, TargetError>;
 
-    /// Saves an event (either sends it immediately or stores it for later)
+    /// Saves an event (either sends it immediately or stores it for later).
+    ///
+    /// A target whose [`Self::store`] returns `Some` must only persist the event
+    /// here; network delivery belongs to its replay worker. Runtime lifecycle
+    /// handoff drains these durable enqueues while allowing a direct network
+    /// send to finish against a detached target.
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
 
     /// Sends an event from the store using the queued raw body and metadata.
@@ -605,6 +612,24 @@ pub(crate) fn open_target_queue_store(
     Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
 }
 
+thread_local! {
+    static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
+}
+
+pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
+    struct Reset(bool);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
+        }
+    }
+
+    let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
+    let _reset = Reset(previous);
+    operation()
+}
+
 /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
 /// failed-store capability holds it directly rather than through the type-erased Store handle.
 pub(crate) fn open_target_queue_store_typed(
@@ -625,9 +650,11 @@ pub(crate) fn open_target_queue_store_typed(
         TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
     };
     let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
-    store
-        .open()
-        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
+        store
+            .open()
+            .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    }
 
     Ok(Some(store))
 }
@@ -649,6 +676,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
     matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
 }
 
+/// Applies an absolute deadline to one protocol delivery attempt.
+///
+/// Target clients expose different timeout controls, and several of them only
+/// place a timeout value in the wire request without bounding the local socket
+/// future. Keeping the outer deadline here gives every caller the same typed,
+/// retryable timeout without changing the target-specific error mapping.
+pub(crate) async fn with_delivery_deadline<T, F>(
+    deadline: Duration,
+    operation: &'static str,
+    delivery: F,
+) -> Result<T, TargetError>
+where
+    F: Future<Output = Result<T, TargetError>>,
+{
+    match tokio::time::timeout(deadline, delivery).await {
+        Ok(result) => result,
+        Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
+    }
+}
+
 pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
 where
     F: FnOnce() -> Fut,
@@ -1137,6 +1184,29 @@ mod tests {
         let _ = fs::remove_file(base);
     }
 
+    #[test]
+    fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let store = with_deferred_queue_store_open(|| {
+            open_target_queue_store(
+                base.to_str().unwrap(),
+                100,
+                TargetType::NotifyEvent,
+                ChannelTargetType::Kafka.as_str(),
+                &target_id,
+                "deferred open",
+            )
+        })
+        .expect("deferred construction must not open the queue directory")
+        .expect("non-empty queue directory should create a dormant store");
+
+        assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
+        let _ = fs::remove_file(base);
+    }
+
     #[test]
     fn persist_queued_payload_to_store_writes_encoded_payload() {
         let store = MockQueuedStore::new(false);
@@ -1182,6 +1252,19 @@ mod tests {
         assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
+        let error = with_delivery_deadline(
+            Duration::from_secs(30),
+            "test delivery",
+            std::future::pending::<Result<(), TargetError>>(),
+        )
+        .await
+        .expect_err("a stalled delivery must hit its hard deadline");
+
+        assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
+    }
+
     #[tokio::test]
     async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
         let marker = Arc::new(AtomicBool::new(false));

crates/targets/src/target/mqtt.rs

@@ -765,11 +765,6 @@ where
 
     #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
     async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
         debug!(
             event = EVENT_MQTT_DELIVERY_STATE,
             component = LOG_COMPONENT_TARGETS,
@@ -790,10 +785,22 @@ where
         // silently dropped the event while its durable copy was already deleted
         // (backlog#971). Error classification now matches on the typed error
         // instead of substring matching on the display string.
-        let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
-            Ok(notice) => notice,
-            Err(e) => {
-                let err = classify_mqtt_client_error(&e);
+        let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
+            let client_guard = self.client.lock().await;
+            let client = client_guard
+                .as_ref()
+                .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+            let notice = client
+                .publish_tracked(&self.args.topic, self.args.qos, false, body)
+                .await
+                .map_err(|error| classify_mqtt_client_error(&error))?;
+            drop(client_guard);
+            Ok(notice)
+        })
+        .await
+        {
+            Ok(Ok(notice)) => notice,
+            Ok(Err(err)) => {
                 warn!(
                     event = EVENT_MQTT_DELIVERY_STATE,
                     component = LOG_COMPONENT_TARGETS,
@@ -801,18 +808,29 @@ where
                     target_id = %self.id,
                     state = "publish_failed",
                     reason = "enqueue_error",
-                    error = %e,
+                    error = %err,
                     "mqtt delivery state"
                 );
                 mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                 return Err(err);
             }
+            Err(_) => {
+                warn!(
+                    event = EVENT_MQTT_DELIVERY_STATE,
+                    component = LOG_COMPONENT_TARGETS,
+                    subsystem = LOG_SUBSYSTEM_MQTT,
+                    target_id = %self.id,
+                    state = "publish_failed",
+                    reason = "enqueue_timeout",
+                    "mqtt delivery state"
+                );
+                // Admission can time out because the local bounded request
+                // channel is full while the MQTT session remains connected.
+                // Only protocol/client failures are evidence of disconnect.
+                return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
+            }
         };
 
-        // Release the client lock before awaiting the broker acknowledgement so a
-        // slow/hung broker never blocks other senders from queueing publishes.
-        drop(client_guard);
-
         match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
             Ok(Ok(())) => {
                 debug!(
@@ -1708,9 +1726,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
-        classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
-        validate_mqtt_broker_url,
+        AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
+        MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
+        next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
     };
     use crate::error::TargetError;
     use crate::target::{REDACTED_SECRET, TargetType};
@@ -1720,6 +1738,23 @@ mod tests {
     use tokio::sync::mpsc;
     use url::Url;
 
+    fn base_mqtt_args() -> MQTTArgs {
+        MQTTArgs {
+            enable: true,
+            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
+            topic: "rustfs/events".to_string(),
+            qos: QoS::AtLeastOnce,
+            username: String::new(),
+            password: String::new(),
+            tls: MQTTTlsConfig::default(),
+            max_reconnect_interval: Duration::from_secs(1),
+            keep_alive: Duration::from_secs(30),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
     #[test]
     fn mqtt_client_error_classified_as_not_connected() {
         // A publish that cannot be handed to the event loop means the client is
@@ -1752,6 +1787,38 @@ mod tests {
         assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn enqueue_timeout_keeps_a_live_session_connected() {
+        let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
+        let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
+            .capacity(1)
+            .build();
+        client
+            .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
+            .await
+            .expect("first publish should fill the local channel");
+        *target.client.lock().await = Some(client);
+        target.connected.store(true, Ordering::SeqCst);
+        let meta = QueuedPayloadMeta::new(
+            rustfs_s3_types::EventName::ObjectCreatedPut,
+            "bucket".to_string(),
+            "object".to_string(),
+            "application/json",
+            2,
+        );
+
+        let error = target
+            .send_body(b"{}".to_vec(), &meta)
+            .await
+            .expect_err("a full local request channel should hit the enqueue deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(
+            target.connected.load(Ordering::SeqCst),
+            "local admission pressure is not evidence that the MQTT session disconnected"
+        );
+    }
+
     #[test]
     fn next_reconnect_backoff_doubles_until_capped() {
         let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
@@ -1864,21 +1931,13 @@ mod tests {
     #[test]
     fn debug_redacts_mqtt_secret_fields() {
         let args = MQTTArgs {
-            enable: true,
-            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
-            topic: "rustfs/events".to_string(),
-            qos: QoS::AtLeastOnce,
             username: "mqtt-user".to_string(),
             password: "mqtt-password".to_string(),
             tls: MQTTTlsConfig {
                 client_key_path: "/etc/rustfs/mqtt.key".to_string(),
                 ..MQTTTlsConfig::default()
             },
-            max_reconnect_interval: Duration::from_secs(1),
-            keep_alive: Duration::from_secs(30),
-            queue_dir: String::new(),
-            queue_limit: 0,
-            target_type: TargetType::NotifyEvent,
+            ..base_mqtt_args()
         };
 
         let rendered = format!("{args:?}");

crates/targets/src/target/mysql.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -47,6 +47,8 @@ use uuid::Uuid;
 /// `TargetError::Timeout`, a connectivity error, so the payload stays queued
 /// for replay.
 const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
+/// Absolute ceiling for one INSERT, including pool checkout and server execution.
+const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Name of the optional idempotency-key column / primary key. Present on tables
 /// created by this target; absent on legacy two-column tables.
@@ -784,29 +786,33 @@ where
             "Inserting MySQL event"
         );
 
-        let pool = self.get_or_init_pool().await?;
-        // At this point the pool has already been initialized (get_or_init_pool
-        // succeeded above), so get_conn() failures are always transient: the
-        // connection was lost or the pool is temporarily exhausted.
-        let mut conn = checkout_conn(&pool).await?;
-
         let event_time = extract_event_time(body)?;
         let event_data =
             std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;
 
         let quoted_table = quote_table_name(&self.args.table)?;
+        with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
+            let pool = self.get_or_init_pool().await?;
+            // At this point the pool has already been initialized (get_or_init_pool
+            // succeeded above), so get_conn() failures are always transient: the
+            // connection was lost or the pool is temporarily exhausted.
+            let mut conn = checkout_conn(&pool).await?;
 
-        if self.idempotency_supported.load(Ordering::Relaxed) {
-            let sql = mysql_insert_sql_with_event_id(&quoted_table);
-            conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        } else {
-            let sql = mysql_insert_sql_legacy(&quoted_table);
-            conn.exec_drop(sql, (event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        }
+            if self.idempotency_supported.load(Ordering::Relaxed) {
+                let sql = mysql_insert_sql_with_event_id(&quoted_table);
+                conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            } else {
+                let sql = mysql_insert_sql_legacy(&quoted_table);
+                conn.exec_drop(sql, (event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            }
+
+            Ok(())
+        })
+        .await?;
 
         self.delivery_counters.record_success();
         debug!(target_id = %self.id, "MySQL event inserted");

crates/targets/src/target/nats/mod.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
-        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
+        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
 pub(crate) use jetstream::resolve_dedup_id;
 pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
 
+const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
 #[derive(Clone)]
 pub struct NATSArgs {
     pub enable: bool,
@@ -397,9 +399,21 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        let client = self.get_or_connect().await?;
-        if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
-            let err = classify_nats_publish_error(&e);
+        let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
+            let client = self.get_or_connect().await?;
+            client
+                .publish(self.args.subject.clone(), body.into())
+                .await
+                .map_err(|err| classify_nats_publish_error(&err))?;
+
+            // publish only enqueues the message on the client's outbound channel. Flush to confirm the
+            // message reached the server before delivery is treated as successful (backlog#971).
+            client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = result {
             if is_connectivity_error(&err) {
                 self.invalidate_cached_client_connection().await;
                 self.connected.store(false, Ordering::SeqCst);
@@ -407,15 +421,6 @@ where
             return Err(err);
         }
 
-        // publish only enqueues the message on the client's outbound channel. Flush to confirm the
-        // message reached the server before delivery is treated as successful (backlog#971).
-        if let Err(e) = client.flush().await {
-            let err = classify_nats_flush_error(&e);
-            self.invalidate_cached_client_connection().await;
-            self.connected.store(false, Ordering::SeqCst);
-            return Err(err);
-        }
-
         self.delivery_counters.record_success();
         Ok(())
     }

crates/targets/src/target/postgres.rs

@@ -38,7 +38,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
-        redacted_secret,
+        redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
 /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
 /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
 const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
+/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
+const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Returns `true` for any `s3:ObjectRemoved:*` event.
 ///
@@ -730,30 +732,29 @@ where
 
         let key = resolve_payload_key(&payload, meta);
 
-        let result = match self.args.format {
-            // For the single-row `namespace` format, an object removal must
-            // delete the row rather than UPSERT it, otherwise stale state
-            // lingers in the table after the object is gone.
-            PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
-                client.execute(&self.namespace_delete_sql, &[&key]).await
+        with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
+            match self.args.format {
+                // For the single-row `namespace` format, an object removal must
+                // delete the row rather than UPSERT it, otherwise stale state
+                // lingers in the table after the object is gone.
+                PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
+                    client.execute(&self.namespace_delete_sql, &[&key]).await
+                }
+                PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
+                PostgresFormat::Access => {
+                    let event_name_str = meta.event_name.to_string();
+                    let queued_at_ms = meta.queued_at_unix_ms as i64;
+                    client
+                        .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
+                        .await
+                }
             }
-            PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
-            PostgresFormat::Access => {
-                let event_name_str = meta.event_name.to_string();
-                let queued_at_ms = meta.queued_at_unix_ms as i64;
-                client
-                    .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
-                    .await
-            }
-        };
+            .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
+        })
+        .await?;
 
-        match result {
-            Ok(_) => {
-                self.delivery_counters.record_success();
-                Ok(())
-            }
-            Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
-        }
+        self.delivery_counters.record_success();
+        Ok(())
     }
 
     /// Probes the table from `init()`. Failure is non-fatal when a queue is

crates/targets/src/target/pulsar.rs

@@ -24,8 +24,9 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -39,11 +40,15 @@ use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;
 
+const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
+
 #[derive(Clone)]
 pub struct PulsarArgs {
     pub enable: bool,
@@ -276,6 +281,23 @@ where
         self.tls_state.lock().reset();
     }
 
+    async fn clear_failed_delivery_state(&self) {
+        match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
+            Ok(mut producer) => {
+                producer.take();
+            }
+            Err(_) => {
+                warn!(
+                    target_id = %self.id,
+                    reason = "producer_cleanup_lock_timeout",
+                    "Timed out clearing the Pulsar producer after a failed delivery"
+                );
+            }
+        }
+        self.clear_cached_client();
+        self.connected.store(false, Ordering::SeqCst);
+    }
+
     async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
         // When a TLS reload adapter is attached, it drives client rebuilds
         // in the background. The inline per-send fingerprint check is skipped.
@@ -334,20 +356,30 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        self.init_producer().await?;
-        let mut guard = self.producer.lock().await;
-        let producer = guard
-            .as_mut()
-            .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
-        let receipt = producer
-            .send_non_blocking(body)
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
-        receipt
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
-        self.delivery_counters.record_success();
-        Ok(())
+        let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
+            self.init_producer().await?;
+            let mut guard = self.producer.lock().await;
+            let producer = guard
+                .as_mut()
+                .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
+            let receipt = producer
+                .send_non_blocking(body)
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
+            receipt
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
+            self.delivery_counters.record_success();
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = &result
+            && is_connectivity_error(err)
+        {
+            self.clear_failed_delivery_state().await;
+        }
+        result
     }
 }
 
@@ -525,6 +557,22 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
+        let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
+        target.connected.store(true, Ordering::SeqCst);
+        let producer_guard = target.producer.lock().await;
+        let cleanup = {
+            let target = Arc::clone(&target);
+            tokio::spawn(async move { target.clear_failed_delivery_state().await })
+        };
+
+        cleanup.await.expect("cleanup task should not panic");
+
+        assert!(!target.connected.load(Ordering::SeqCst));
+        drop(producer_guard);
+    }
+
     #[test]
     fn debug_redacts_pulsar_secret_fields() {
         let args = PulsarArgs {

crates/targets/src/target/redis.rs

@@ -26,6 +26,7 @@ use crate::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
         mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
 use tracing::{debug, info, instrument, warn};
 use url::Url;
 
+const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+
+fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
+    let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
+    let per_attempt = args
+        .connection_timeout
+        .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
+        .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
+        .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
+    per_attempt.saturating_mul(attempts)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RedisTlsPolicy {
     SystemCa,
@@ -214,6 +228,16 @@ impl RedisArgs {
             ));
         }
 
+        if self.connection_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration(
+                "Redis connection_timeout must be greater than zero".to_string(),
+            ));
+        }
+
+        if self.response_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
+        }
+
         if self.pipeline_buffer_size == Some(0) {
             return Err(TargetError::Configuration(
                 "Redis pipeline_buffer_size must be greater than zero".to_string(),
@@ -464,71 +488,97 @@ where
             "Sending Redis payload"
         );
 
-        let mut attempt = 0usize;
-        let mut last_error = None;
-        while attempt < self.args.max_retry_attempts {
-            attempt += 1;
+        let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
+            let mut attempt = 0usize;
+            let mut last_error = None;
+            while attempt < self.args.max_retry_attempts {
+                attempt += 1;
 
-            let mut publisher = self.get_or_create_publisher().await?;
-            match publisher
-                .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
+                let mut publisher = match with_delivery_deadline(
+                    connection_timeout,
+                    "Redis connection",
+                    self.get_or_create_publisher(),
+                )
                 .await
-            {
-                Ok(receiver_count) => {
-                    // PUBLISH returns the number of subscribers that received the
-                    // message. Redis pub/sub is best-effort: with zero subscribers
-                    // the event is delivered to no one, yet the durable copy is
-                    // deleted. Warn so operators relying on reliable delivery are
-                    // not silently losing events (backlog#982).
-                    if receiver_count == 0 {
+                {
+                    Ok(publisher) => publisher,
+                    Err(err) => {
+                        invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                        return Err(err);
+                    }
+                };
+                let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
+                match with_delivery_deadline(response_timeout, "Redis publish response", async {
+                    publisher
+                        .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                        .await
+                        .map_err(map_redis_error)
+                })
+                .await
+                {
+                    Ok(receiver_count) => {
+                        // PUBLISH returns the number of subscribers that received the
+                        // message. Redis pub/sub is best-effort: with zero subscribers
+                        // the event is delivered to no one, yet the durable copy is
+                        // deleted. Warn so operators relying on reliable delivery are
+                        // not silently losing events (backlog#982).
+                        if receiver_count == 0 {
+                            warn!(
+                                target_id = %self.id,
+                                channel = %self.args.channel,
+                                "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            );
+                        }
+                        debug!(
+                            target_id = %self.id,
+                            channel = %self.args.channel,
+                            attempt,
+                            receiver_count,
+                            "Event published to Redis channel"
+                        );
+                        self.delivery_counters.record_success();
+                        return Ok(());
+                    }
+                    Err(mapped) => {
+                        invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
+
                         warn!(
                             target_id = %self.id,
                             channel = %self.args.channel,
-                            "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            attempt,
+                            max_attempts = self.args.max_retry_attempts,
+                            error = %mapped,
+                            "Redis publish attempt failed"
                         );
-                    }
-                    debug!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        receiver_count,
-                        "Event published to Redis channel"
-                    );
-                    self.delivery_counters.record_success();
-                    return Ok(());
-                }
-                Err(err) => {
-                    let mapped = map_redis_error(err);
-                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
 
-                    warn!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        max_attempts = self.args.max_retry_attempts,
-                        error = %mapped,
-                        "Redis publish attempt failed"
-                    );
+                        if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                            last_error = Some(mapped);
+                            break;
+                        }
 
-                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                         last_error = Some(mapped);
-                        break;
+                        tokio::time::sleep(compute_retry_delay(
+                            attempt,
+                            self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
+                            self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
+                        ))
+                        .await;
                     }
-
-                    last_error = Some(mapped);
-                    tokio::time::sleep(compute_retry_delay(
-                        attempt,
-                        self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
-                        self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
-                    ))
-                    .await;
                 }
             }
+
+            Err(last_error.unwrap_or(TargetError::Unknown(
+                "Redis publish failed without a captured error".to_string(),
+            )))
+        })
+        .await;
+
+        if let Err(err) = &result {
+            invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
+            self.connected.store(false, Ordering::SeqCst);
         }
-
-        self.connected.store(false, Ordering::SeqCst);
-
-        Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
+        result
     }
 }
 
@@ -551,7 +601,7 @@ where
         // thus a fresh TCP+TLS handshake — on every health check (backlog#982).
         // ensure_publisher_ready already invalidates the cached manager on a
         // connectivity error so the next attempt rebuilds it.
-        match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
+        match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
             Ok(Ok(())) => {
                 self.connected.store(true, Ordering::SeqCst);
                 Ok(true)
@@ -917,6 +967,26 @@ mod tests {
         assert!(args.validate().is_err());
     }
 
+    #[test]
+    fn validate_rejects_zero_connection_timeout() {
+        let args = RedisArgs {
+            connection_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_response_timeout() {
+        let args = RedisArgs {
+            response_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
     #[test]
     fn validate_accepts_custom_ca_tls_policy() {
         let args = RedisArgs {
@@ -1237,6 +1307,23 @@ mod tests {
         assert_eq!(target.delivery_snapshot().total_messages, 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
+        let mut args = base_args();
+        args.max_retry_attempts = 1;
+        args.response_timeout = Some(Duration::from_secs(90));
+        let manager_config = build_redis_connection_manager_config(&args);
+
+        assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
+        assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
+        with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
+            tokio::time::sleep(Duration::from_secs(70)).await;
+            Ok::<_, TargetError>(())
+        })
+        .await
+        .expect("the configured delivery budget must not impose a fixed sixty-second cap");
+    }
+
     #[tokio::test]
     async fn send_body_sets_connected_false_after_retry_exhaustion() {
         let listener = match TcpListener::bind("127.0.0.1:0").await {

crates/targets/src/target/webhook.rs

@@ -952,7 +952,7 @@ mod tests {
             .expect("https webhook probe should trust configured ca");
 
         assert_eq!(resp.status(), reqwest::StatusCode::OK);
-        assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
+        assert!(resp.bytes().await.expect("read response body").is_empty());
         handle.join().expect("tls server thread");
     }
 }