fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

Cargo.lock

@@ -9537,8 +9537,10 @@ dependencies = [
  "serde",
  "serde_json",
  "starshard",
+ "tempfile",
  "thiserror 2.0.19",
  "tokio",
+ "tokio-util",
  "tracing",
  "tracing-subscriber",
  "url",
@@ -9955,6 +9957,7 @@ dependencies = [
  "chrono",
  "criterion",
  "deadpool-postgres",
+ "futures-util",
  "hashbrown 0.17.1",
  "hyper",
  "hyper-rustls",
@@ -9964,6 +9967,7 @@ dependencies = [
  "mysql_async",
  "parking_lot",
  "pulsar",
+ "rayon",
  "rcgen",
  "redis",
  "regex",
@@ -9988,6 +9992,7 @@ dependencies = [
  "tokio",
  "tokio-postgres",
  "tokio-postgres-rustls",
+ "tokio-util",
  "tracing",
  "tracing-subscriber",
  "url",

crates/e2e_test/src/notification_webhook_test.rs

@@ -156,7 +156,43 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
     Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
 }
 
-fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool>, thread::JoinHandle<()>), BoxError> {
+struct HttpsEventCollector {
+    endpoint: String,
+    running: Arc<AtomicBool>,
+    handle: Option<thread::JoinHandle<()>>,
+    events: mpsc::UnboundedReceiver<Value>,
+}
+
+impl HttpsEventCollector {
+    fn endpoint(&self) -> &str {
+        &self.endpoint
+    }
+
+    fn events_mut(&mut self) -> &mut mpsc::UnboundedReceiver<Value> {
+        &mut self.events
+    }
+
+    fn shutdown(&mut self) -> TestResult {
+        self.running.store(false, Ordering::Relaxed);
+        if let Ok(parsed) = self.endpoint.parse::<reqwest::Url>()
+            && let Some(port) = parsed.port()
+        {
+            let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+        }
+        if let Some(handle) = self.handle.take() {
+            handle.join().map_err(|_| "https event collector thread panicked")?;
+        }
+        Ok(())
+    }
+}
+
+impl Drop for HttpsEventCollector {
+    fn drop(&mut self) {
+        let _ = self.shutdown();
+    }
+}
+
+fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, BoxError> {
     use rustls::{
         ServerConfig,
         pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
@@ -188,48 +224,103 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc<AtomicBool
 
     let running = Arc::new(AtomicBool::new(true));
     let server_running = Arc::clone(&running);
+    let (tx, events) = mpsc::unbounded_channel();
     let handle = thread::spawn(move || {
+        let mut connections = Vec::new();
         while server_running.load(Ordering::Relaxed) {
             match listener.accept() {
                 Ok((stream, _)) => {
                     let config = Arc::clone(&server_config);
-                    handle_https_probe(stream, config);
+                    let tx = tx.clone();
+                    connections.push(thread::spawn(move || {
+                        let _ = handle_https_request(stream, config, tx);
+                    }));
                 }
                 Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)),
                 Err(_) => break,
             }
         }
+        for connection in connections {
+            let _ = connection.join();
+        }
     });
 
-    Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle))
+    Ok(HttpsEventCollector {
+        endpoint: format!("https://{endpoint_host}:{}/events", addr.port()),
+        running,
+        handle: Some(handle),
+        events,
+    })
 }
 
-fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc<rustls::ServerConfig>) {
-    use std::io::{Read, Write};
-
-    let _ = stream.set_read_timeout(Some(Duration::from_secs(5)));
-    let _ = stream.set_write_timeout(Some(Duration::from_secs(5)));
-    let Ok(connection) = rustls::ServerConnection::new(server_config) else {
-        return;
+fn read_sync_http_message<R: std::io::Read>(stream: &mut R) -> Result<(String, Vec<u8>), BoxError> {
+    let mut buffer = Vec::new();
+    let mut chunk = [0_u8; 4096];
+    let header_end = loop {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request headers were complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+        if let Some(pos) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
+            break pos;
+        }
     };
-    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
-    let mut buf = [0u8; 1024];
-    if tls_stream.read(&mut buf).is_err() {
-        return;
+
+    let header_text = std::str::from_utf8(&buffer[..header_end])?;
+    let mut lines = header_text.split("\r\n");
+    let method = lines
+        .next()
+        .and_then(|line| line.split_whitespace().next())
+        .ok_or("missing request method")?
+        .to_string();
+    let mut content_length = 0usize;
+    for line in lines {
+        if let Some((name, value)) = line.split_once(':')
+            && name.trim().eq_ignore_ascii_case("content-length")
+        {
+            content_length = value.trim().parse()?;
+        }
     }
-    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
-    let _ = tls_stream.write_all(response.as_bytes());
-    let _ = tls_stream.flush();
+
+    let body_offset = header_end + 4;
+    while buffer.len().saturating_sub(body_offset) < content_length {
+        let read = stream.read(&mut chunk)?;
+        if read == 0 {
+            return Err("connection closed before request body was complete".into());
+        }
+        buffer.extend_from_slice(&chunk[..read]);
+    }
+    Ok((method, buffer[body_offset..body_offset + content_length].to_vec()))
 }
 
-fn stop_https_event_collector(endpoint: &str, running: Arc<AtomicBool>, handle: thread::JoinHandle<()>) -> TestResult {
-    running.store(false, Ordering::Relaxed);
-    if let Ok(parsed) = endpoint.parse::<reqwest::Url>()
-        && let Some(port) = parsed.port()
-    {
-        let _ = std::net::TcpStream::connect(("127.0.0.1", port));
+fn handle_https_request(
+    stream: std::net::TcpStream,
+    server_config: Arc<rustls::ServerConfig>,
+    tx: mpsc::UnboundedSender<Value>,
+) -> Result<(), BoxError> {
+    use std::io::Write;
+
+    stream.set_nonblocking(false)?;
+    stream.set_read_timeout(Some(Duration::from_secs(5)))?;
+    stream.set_write_timeout(Some(Duration::from_secs(5)))?;
+    let connection = rustls::ServerConnection::new(server_config)?;
+    let mut tls_stream = rustls::StreamOwned::new(connection, stream);
+    let (method, body) = read_sync_http_message(&mut tls_stream)?;
+    let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n";
+    tls_stream.write_all(response.as_bytes())?;
+    tls_stream.flush()?;
+    tls_stream.conn.send_close_notify();
+    while tls_stream.conn.wants_write() {
+        tls_stream.conn.write_tls(&mut tls_stream.sock)?;
+    }
+    let _ = tls_stream.sock.shutdown(std::net::Shutdown::Write);
+    if method == "POST"
+        && !body.is_empty()
+        && let Ok(event) = serde_json::from_slice(&body)
+    {
+        let _ = tx.send(event);
     }
-    handle.join().map_err(|_| "https event collector thread panicked")?;
     Ok(())
 }
 
@@ -415,17 +506,20 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s
     Err(format!("target {target_name} was not registered in admin ARNs").into())
 }
 
-async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
+async fn wait_for_target_online(env: &RustFSTestEnvironment, target_name: &str) -> TestResult {
     let url = format!("{}/rustfs/admin/v3/target/list", env.url);
     for _ in 0..40 {
         let response = signed_admin_request(env, http::Method::GET, &url, None).await?;
         if response.status() == StatusCode::OK {
             let body: Value = serde_json::from_slice(&response.bytes().await?)?;
+            if body["notify_enabled"].as_bool() != Some(true) {
+                return Err(format!("admin target list did not report notify_enabled=true: {body}").into());
+            }
             let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| {
                 endpoints.iter().any(|endpoint| {
                     endpoint["account_id"].as_str() == Some(target_name)
                         && endpoint["service"].as_str() == Some("webhook")
-                        && endpoint["status"].as_str().is_some()
+                        && endpoint["status"].as_str() == Some("online")
                 })
             });
             if listed {
@@ -434,7 +528,7 @@ async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str)
         }
         tokio::time::sleep(Duration::from_millis(250)).await;
     }
-    Err(format!("target {target_name} was not listed in admin targets").into())
+    Err(format!("target {target_name} did not become online in admin targets").into())
 }
 
 /// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events,
@@ -484,11 +578,11 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
 // ---------------------------------------------------------------------------
 
 /// Regression for rustfs#5052: with the notify module enabled through
-/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted
-/// and remain visible in the admin target list.
+/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become
+/// online and receive a real S3 event POST.
 #[tokio::test]
 #[serial]
-async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult {
+async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult {
     init_logging();
 
     let mut env = RustFSTestEnvironment::new().await?;
@@ -496,22 +590,40 @@ async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult
         .await?;
 
     let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
-    let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?;
+    let mut collector = spawn_https_event_collector(&ca_path)?;
     let target = "peri1https";
+    let bucket = "peri1-https-events";
+    let key = "uploads/https.dat";
+    let client = env.create_s3_client();
+    client.create_bucket().bucket(bucket).send().await?;
 
     configure_webhook_target_with_key_values(
         &env,
         target,
         vec![
-            ("endpoint", endpoint.clone()),
+            ("endpoint", collector.endpoint().to_string()),
             ("client_ca", ca_path.to_string_lossy().into_owned()),
         ],
     )
     .await?;
-    wait_for_target_listed(&env, target).await?;
+    wait_for_target_online(&env, target).await?;
+    wait_for_target_registered(&env, target).await?;
+    put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+
+    client
+        .put_object()
+        .bucket(bucket)
+        .key(key)
+        .body(ByteStream::from_static(b"https webhook event body"))
+        .send()
+        .await?;
+    let event = wait_for_event(collector.events_mut(), key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
+    assert_eq!(event["EventName"].as_str(), Some("s3:ObjectCreated:Put"));
+    assert_eq!(event["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket));
+    assert_eq!(event_key(&event).as_deref(), Some(key));
 
     env.stop_server();
-    stop_https_event_collector(&endpoint, running, handle)?;
+    collector.shutdown()?;
     Ok(())
 }
 

crates/ecstore/src/api/mod.rs

@@ -241,8 +241,10 @@ pub mod config {
         pub use crate::config::com::{
             COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
             ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config,
-            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, save_config, save_config_with_opts,
-            save_server_config, try_migrate_server_config,
+            read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock,
+            read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
+            save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock,
+            with_server_config_read_lock, with_server_config_write_lock,
         };
     }
 
@@ -383,9 +385,9 @@ pub mod rio {
 
 pub mod rpc {
     pub use crate::cluster::rpc::{
-        LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
-        SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
-        gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
+        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
+        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor,
+        gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
         node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
         sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof,
         verify_tonic_rpc_signature,

crates/ecstore/src/cluster/rpc/mod.rs

@@ -37,8 +37,8 @@ pub use http_auth::{
 pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
 pub use internode_data_transport::build_internode_data_transport_from_env;
 pub use peer_rest_client::{
-    PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
-    ScannerPeerActivity,
+    PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
+    SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
 };
 pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
 pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys};

crates/ecstore/src/config/com.rs

@@ -17,12 +17,15 @@ use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
 use crate::runtime::sources as runtime_sources;
+use crate::set_disk::get_lock_acquire_timeout;
 use crate::storage_api_contracts::{
     admin::StorageAdminApi,
     heal::HealOperations,
+    namespace::NamespaceLocking,
     object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete},
     range::HTTPRangeSpec,
 };
+use crate::store::ECStore;
 use http::HeaderMap;
 use rustfs_common::heal_channel::{HealOpts, HealScanMode};
 use rustfs_config::audit::{
@@ -45,10 +48,94 @@ use serde_json::{Map, Value};
 use std::collections::{HashMap, HashSet};
 use std::sync::LazyLock;
 use std::sync::{Arc, RwLock};
+use tokio::sync::RwLock as AsyncRwLock;
 use tracing::{debug, error, info, instrument, warn};
 
 pub const CONFIG_PREFIX: &str = "config";
-const CONFIG_FILE: &str = "config.json";
+const SERVER_CONFIG_OBJECT: &str = "config/config.json";
+
+// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock
+// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order.
+static SERVER_CONFIG_LOCK: LazyLock<AsyncRwLock<()>> = LazyLock::new(|| AsyncRwLock::new(()));
+
+fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error {
+    let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    Error::other(format!("{operation} task {outcome}"))
+}
+
+/// Runs one complete server-config transaction while holding both the local
+/// process guard and the distributed namespace write lock for `config.json`.
+/// The operation must use the corresponding no-lock read/save functions.
+pub async fn with_server_config_write_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace write lock.
+        let _local_guard = SERVER_CONFIG_LOCK.write().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config write", error))?
+}
+
+/// Reads and synchronously publishes one server-config snapshot while holding
+/// a shared namespace lock. Concurrent peer reloads may proceed together, but
+/// no writer can interleave between the snapshot read and generation accept.
+pub async fn with_server_config_read_lock<F, Fut, T>(store: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        // Lock order: SERVER_CONFIG_LOCK -> namespace read lock.
+        let _local_guard = SERVER_CONFIG_LOCK.read().await;
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("server config read", error))?
+}
+
+/// Runs a cancellation-safe transaction under the distributed write lock for
+/// one metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_write_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object write", error))?
+}
+
+/// Runs one read-and-publish transaction under the distributed read lock for
+/// a metadata config object. The operation must use no-lock object I/O.
+pub async fn with_config_object_read_lock<F, Fut, T>(store: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    tokio::spawn(async move {
+        let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
+        let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?;
+        Ok(operation().await)
+    })
+    .await
+    .map_err(|error| config_task_join_error("config object read", error))?
+}
 
 /// Environment variable gating the startup fallback to the default server
 /// config when the persisted `config.json` object is corrupt beyond repair
@@ -393,6 +480,31 @@ where
     .await
 }
 
+pub async fn save_config_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_config_with_opts(
+        api,
+        file,
+        data,
+        &ObjectOptions {
+            max_parity: true,
+            no_lock: true,
+            ..Default::default()
+        },
+    )
+    .await
+}
+
 #[instrument(skip(api))]
 pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
 where
@@ -462,8 +574,17 @@ where
     Ok(cfg)
 }
 
+async fn new_and_save_server_config_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: EcstoreObjectIO + StorageAdminApi,
+{
+    let cfg = new_server_config();
+    save_server_config_no_lock(api, &cfg).await?;
+    Ok(cfg)
+}
+
 fn get_config_file() -> String {
-    format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
+    SERVER_CONFIG_OBJECT.to_string()
 }
 
 fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS {
@@ -1267,13 +1388,17 @@ where
 }
 
 /// Handle the situation where the configuration file does not exist, create and save a new configuration
-async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
+async fn handle_missing_config<S>(api: Arc<S>, context: &str, namespace_lock_held: bool) -> Result<Config>
 where
     S: EcstoreObjectIO + StorageAdminApi,
 {
     warn!("Configuration not found ({}): Start initializing new configuration", context);
     let cfg = if runtime_sources::first_cluster_node_is_local().await {
-        new_and_save_server_config(api.clone()).await?
+        if namespace_lock_held {
+            new_and_save_server_config_no_lock(api.clone()).await?
+        } else {
+            new_and_save_server_config(api.clone()).await?
+        }
     } else {
         new_server_config()
     };
@@ -1288,6 +1413,46 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
 }
 
 pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, false).await
+}
+
+/// Reads the server config while an upper layer holds the namespace write lock
+/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching
+/// no-lock save path and therefore cannot recursively acquire the same lock.
+pub async fn read_config_without_migrate_no_lock<S>(api: Arc<S>) -> Result<Config>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        > + StorageAdminApi,
+{
+    read_config_without_migrate_inner(api, true).await
+}
+
+/// Reads an already-initialized server config while a caller owns a namespace
+/// read lock. This never initializes or migrates a missing object.
+pub async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<Config> {
+    let data = read_config_no_lock(api, SERVER_CONFIG_OBJECT).await?;
+    Ok(decode_persisted_server_config(&data)?.merge())
+}
+
+async fn read_config_without_migrate_inner<S>(api: Arc<S>, namespace_lock_held: bool) -> Result<Config>
 where
     S: ObjectIO<
             Error = Error,
@@ -1303,13 +1468,13 @@ where
 
     // Try to read the configuration file
     match read_config_no_lock(api.clone(), &config_file).await {
-        Ok(data) => read_server_config(api, &data).await,
-        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
+        Ok(data) => read_server_config(api, &data, namespace_lock_held).await,
+        Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await,
         Err(err) => handle_config_read_error(err, &config_file),
     }
 }
 
-async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
+async fn read_server_config<S>(api: Arc<S>, data: &[u8], namespace_lock_held: bool) -> Result<Config>
 where
     S: EcstoreObjectIO + StorageAdminApi,
 {
@@ -1324,7 +1489,9 @@ where
                 let cfg = decode_persisted_server_config(&cfg_data)?;
                 return Ok(cfg.merge());
             }
-            Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
+            Err(Error::ConfigNotFound) => {
+                return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await;
+            }
             Err(err) => return handle_config_read_error(err, &config_file),
         }
     }
@@ -1534,6 +1701,38 @@ fn fallback_server_config_after_corruption(err: Error, config_file: &str, recove
 }
 
 pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, false).await
+}
+
+/// Saves the server config while an upper layer holds the namespace write
+/// lock for [`SERVER_CONFIG_OBJECT`].
+pub async fn save_server_config_no_lock<S>(api: Arc<S>, cfg: &Config) -> Result<()>
+where
+    S: ObjectIO<
+            Error = Error,
+            RangeSpec = HTTPRangeSpec,
+            HeaderMap = HeaderMap,
+            ObjectOptions = ObjectOptions,
+            ObjectInfo = ObjectInfo,
+            GetObjectReader = GetObjectReader,
+            PutObjectReader = PutObjReader,
+        >,
+{
+    save_server_config_inner(api, cfg, true).await
+}
+
+async fn save_server_config_inner<S>(api: Arc<S>, cfg: &Config, no_lock: bool) -> Result<()>
 where
     S: ObjectIO<
             Error = Error,
@@ -1546,7 +1745,11 @@ where
         >,
 {
     let config_file = get_config_file();
-    let existing = match read_config(api.clone(), &config_file).await {
+    let existing = match if no_lock {
+        read_config_no_lock(api.clone(), &config_file).await
+    } else {
+        read_config(api.clone(), &config_file).await
+    } {
         Ok(v) => Some(v),
         Err(Error::ConfigNotFound) => None,
         Err(err) => {
@@ -1570,7 +1773,21 @@ where
         return Ok(());
     }
 
-    save_config(api, &config_file, data).await
+    if no_lock {
+        save_config_with_opts(
+            api,
+            &config_file,
+            data,
+            &ObjectOptions {
+                max_parity: true,
+                no_lock: true,
+                ..Default::default()
+            },
+        )
+        .await
+    } else {
+        save_config(api, &config_file, data).await
+    }
 }
 
 pub async fn lookup_configs<S>(cfg: &mut Config, api: Arc<S>) -> Result<()>
@@ -1630,9 +1847,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        apply_dynamic_config_for_sub_sys_with, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
-        is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata,
-        storage_class_kvs_mut,
+        apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob,
+        encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty,
+        read_config_with_metadata, storage_class_kvs_mut,
     };
     use crate::config::{audit, notify, oidc};
     use crate::disk::endpoint::Endpoint;
@@ -1652,6 +1869,17 @@ mod tests {
     use rustfs_config::{
         DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
     };
+
+    #[tokio::test]
+    async fn config_task_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = config_task_join_error("server config write", join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
     use rustfs_lock::client::LockClient;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};

crates/ecstore/src/services/notification_sys.rs

@@ -253,7 +253,7 @@ impl NotificationSys {
         join_all(futures).await
     }
 
-    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+    async fn signal_dynamic_config(&self, sub_sys: &str, dry_run: bool) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {
             let sub_sys = sub_sys.to_string();
@@ -263,7 +263,7 @@ impl NotificationSys {
                         .signal_service(
                             crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC,
                             &sub_sys,
-                            false,
+                            dry_run,
                             SystemTime::UNIX_EPOCH,
                         )
                         .await
@@ -288,6 +288,14 @@ impl NotificationSys {
         join_all(futures).await
     }
 
+    pub async fn preflight_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, true).await
+    }
+
+    pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec<NotificationPeerErr> {
+        self.signal_dynamic_config(sub_sys, false).await
+    }
+
     pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
         let mut futures = Vec::with_capacity(self.peer_clients.len());
         for client in self.peer_clients.iter() {

crates/ecstore/src/store/object.rs

@@ -872,6 +872,8 @@ impl ECStore {
         let idx = if opts.data_movement && opts.version_id.is_some() {
             self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false)
                 .await?
+        } else if opts.no_lock {
+            self.get_pool_idx_no_lock(bucket, &object, data.size()).await?
         } else {
             self.get_pool_idx(bucket, &object, data.size()).await?
         };

crates/notify/Cargo.toml

@@ -44,6 +44,7 @@ serde = { workspace = true, features = ["derive"] }
 starshard = { workspace = true, features = ["rayon", "async", "serde"] }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] }
+tokio-util = { workspace = true }
 tracing = { workspace = true }
 url = { workspace = true }
 metrics = { workspace = true }
@@ -62,6 +63,7 @@ axum = { workspace = true }
 rustfs-utils = { workspace = true, features = ["path"] }
 serde_json = { workspace = true, features = ["raw_value"] }
 criterion = { workspace = true, features = ["html_reports"] }
+tempfile = { workspace = true }
 
 [lints]
 workspace = true

crates/notify/src/config_manager.rs

@@ -13,89 +13,83 @@
 // limitations under the License.
 
 use crate::{
-    Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
+    NotificationError,
+    lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
+    registry::TargetRegistry,
+    resolve_notify_object_store_handle,
+    rule_engine::NotifyRuleEngine,
     runtime_facade::NotifyRuntimeFacade,
+    with_notify_server_config_read_lock, with_notify_server_config_write_lock,
 };
 use rustfs_config::notify::{
     NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
     NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::server_config::{Config, KVS};
-use rustfs_targets::{Target, arn::TargetID};
-use std::sync::{Arc, LazyLock};
-use tokio::sync::{Mutex, RwLock};
+use rustfs_targets::arn::TargetID;
+use std::sync::Arc;
+use tokio::sync::RwLock;
 use tracing::{debug, info};
 
-/// Serializes the read-modify-write sequence over the persisted notify server
-/// config. The persisted config is a single process-global resource (there is
-/// only one backing object store), so without this guard two concurrent updates
-/// can both read the same base config, apply disjoint changes, and race their
-/// full-config writes — the later write silently overwrites the earlier one,
-/// losing updates. Holding this mutex across the whole read→modify→write makes
-/// concurrent updates apply serially so every change is preserved.
-///
-/// The lock is only ever acquired inside `update_server_config`; it never nests
-/// with the per-manager `config` RwLock (the in-memory reload runs after this
-/// guard is released), so it introduces no lock-ordering risk.
-static NOTIFY_CONFIG_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
-
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_CONFIG: &str = "config";
-const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";
 
 #[derive(Debug)]
 enum NotifyConfigStoreError {
+    Lock(String),
     StorageNotAvailable,
     Read(String),
     Save(String),
 }
 
-async fn update_server_config<F>(modifier: F) -> Result<Option<Config>, NotifyConfigStoreError>
+async fn update_server_config<F>(
+    modifier: F,
+    lifecycle: NotifyLifecycleCoordinator,
+) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
     let Some(store) = resolve_notify_object_store_handle() else {
         return Err(NotifyConfigStoreError::StorageNotAvailable);
     };
 
+    let store_for_read = store.clone();
     let store_for_save = store.clone();
-    serialized_read_modify_write(
-        modifier,
-        move || async move {
-            crate::read_notify_server_config_without_migrate(store)
-                .await
-                .map_err(NotifyConfigStoreError::Read)
-        },
-        move |config| async move {
-            crate::save_notify_server_config(store_for_save, &config)
-                .await
-                .map_err(NotifyConfigStoreError::Save)
-        },
-    )
+    with_notify_server_config_write_lock(store, move || {
+        read_modify_write(
+            modifier,
+            move || async move {
+                crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
+                    .await
+                    .map_err(NotifyConfigStoreError::Read)
+            },
+            move |config| async move {
+                crate::save_notify_server_config_no_lock(store_for_save, &config)
+                    .await
+                    .map_err(NotifyConfigStoreError::Save)
+            },
+            move |config| lifecycle.update_config(config),
+        )
+    })
     .await
+    .map_err(NotifyConfigStoreError::Lock)?
 }
 
-/// Runs a `read → modify → write` over the persisted notify config while holding
-/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber
-/// each other's changes (backlog#968). `read`/`save` are injected so the exact
-/// production serialization path can be exercised in tests without a live store.
-async fn serialized_read_modify_write<F, R, RFut, S, SFut>(
+async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
     mut modifier: F,
     read: R,
     save: S,
-) -> Result<Option<Config>, NotifyConfigStoreError>
+    publish: P,
+) -> Result<Option<T>, NotifyConfigStoreError>
 where
     F: FnMut(&mut Config) -> bool,
     R: FnOnce() -> RFut,
     RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
     S: FnOnce(Config) -> SFut,
     SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
+    P: FnOnce(Config) -> T,
 {
-    // Hold the RMW lock across the entire read→modify→write so concurrent
-    // updates serialize and cannot clobber each other's changes (backlog#968).
-    let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await;
-
     let mut new_config = read().await?;
 
     if !modifier(&mut new_config) {
@@ -104,7 +98,7 @@ where
 
     save(new_config.clone()).await?;
 
-    Ok(Some(new_config))
+    Ok(Some(publish(new_config)))
 }
 
 pub(crate) fn notify_configuration_hint() -> String {
@@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) ->
 #[derive(Clone)]
 pub struct NotifyConfigManager {
     config: Arc<RwLock<Config>>,
-    registry: Arc<TargetRegistry>,
+    lifecycle: NotifyLifecycleCoordinator,
     rule_engine: NotifyRuleEngine,
-    runtime_facade: NotifyRuntimeFacade,
 }
 
 impl NotifyConfigManager {
@@ -154,64 +147,21 @@ impl NotifyConfigManager {
         rule_engine: NotifyRuleEngine,
         runtime_facade: NotifyRuntimeFacade,
     ) -> Self {
+        let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
         Self {
             config,
-            registry,
+            lifecycle,
             rule_engine,
-            runtime_facade,
         }
     }
 
+    pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
+        self.lifecycle.clone()
+    }
+
     pub async fn init(&self) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initializing",
-            "notify runtime lifecycle"
-        );
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
-        }
-
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initialized",
-            "notify runtime lifecycle"
-        );
-        Ok(())
+        let config = self.config.read().await.clone();
+        self.lifecycle.set_mode(true, Some(config)).wait().await
     }
 
     pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
@@ -227,6 +177,7 @@ impl NotifyConfigManager {
 
         let ttype = target_type.to_lowercase();
         let tname = target_id.id.to_lowercase();
+        let log_target_id = target_id.clone();
 
         // Guard against orphaning bucket notification rules (backlog#979). Removing a
         // target while a bucket rule still references it would leave a dangling
@@ -240,7 +191,7 @@ impl NotifyConfigManager {
             )));
         }
 
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             let mut changed = false;
             if let Some(targets_of_type) = config.0.get_mut(&ttype) {
                 if targets_of_type.remove(&tname).is_some() {
@@ -249,7 +200,7 @@ impl NotifyConfigManager {
                             component = LOG_COMPONENT_NOTIFY,
                         subsystem = LOG_SUBSYSTEM_CONFIG,
                         action = "remove_target",
-                        target_id = %target_id,
+                        target_id = %log_target_id,
                         result = "removed",
                         "notify config update"
                     );
@@ -265,7 +216,7 @@ impl NotifyConfigManager {
                     component = LOG_COMPONENT_NOTIFY,
                     subsystem = LOG_SUBSYSTEM_CONFIG,
                     action = "remove_target",
-                    target_id = %target_id,
+                    target_id = %log_target_id,
                     result = "not_found",
                     "notify config update"
                 );
@@ -287,7 +238,7 @@ impl NotifyConfigManager {
         );
         let ttype = target_type.to_lowercase();
         let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
             true
         })
@@ -316,7 +267,7 @@ impl NotifyConfigManager {
             )));
         }
 
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
             let mut changed = false;
             if let Some(targets) = config.0.get_mut(&ttype) {
                 if targets.remove(&tname).is_some() {
@@ -332,8 +283,8 @@ impl NotifyConfigManager {
                     component = LOG_COMPONENT_NOTIFY,
                     subsystem = LOG_SUBSYSTEM_CONFIG,
                     action = "remove_target_config",
-                    target_type = %target_type,
-                    target_name = %target_name,
+                    target_type = %ttype,
+                    target_name = %tname,
                     result = "not_found",
                     "notify config update"
                 );
@@ -348,81 +299,62 @@ impl NotifyConfigManager {
     }
 
     pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloading",
-            "notify runtime lifecycle"
-        );
+        self.lifecycle.set_mode(true, Some(new_config)).wait().await
+    }
 
-        self.update_config(new_config.clone()).await;
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        let Some(store) = resolve_notify_object_store_handle() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to load target configuration: server storage not initialized".to_string(),
+            ));
+        };
+        self.reload_persisted_config_from_store(store).await
+    }
 
-        // Stop the currently running replay workers *before* activating the new ones
-        // (backlog#970). Each replay worker drains a per-target persisted store; if the
-        // new workers start while the old ones are still running against the same
-        // stores, both drain the same queues and re-deliver events. `replace_targets`
-        // below also stops workers, but only after `activate_targets_with_replay` has
-        // already spawned the new ones — so without this explicit stop-before-start
-        // there is a window where old and new workers overlap. (The full "signal +
-        // join" shutdown lives in the targets crate and is tracked under the same
-        // issue; this reorders the notify-side lifecycle.)
-        self.runtime_facade.stop_replay_workers().await;
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
         }
 
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloaded",
-            "notify runtime lifecycle"
-        );
+        let read_store = store.clone();
+        let config_cache = self.config.clone();
+        let lifecycle = self.lifecycle.clone();
+        let transition = with_notify_server_config_read_lock(store, move || async move {
+            let config = crate::read_existing_notify_server_config_no_lock(read_store)
+                .await
+                .map_err(NotificationError::ReadConfig)?;
+            Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
+                None
+            } else {
+                Some(lifecycle.update_config(config))
+            })
+        })
+        .await
+        .map_err(NotificationError::StorageNotAvailable)??;
+
+        if let Some(transition) = transition {
+            transition.wait().await?;
+        }
         Ok(())
     }
 
-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
     where
-        F: FnMut(&mut Config) -> bool,
+        F: FnMut(&mut Config) -> bool + Send + 'static,
     {
-        let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err {
-            NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ),
-            NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
-            NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
-        })?
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
+            .await
+            .map_err(|err| match err {
+                NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
+                NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
+                    "Failed to save target configuration: server storage not initialized".to_string(),
+                ),
+                NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
+                NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
+            })?
         else {
             debug!(
                 event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -443,15 +375,15 @@ impl NotifyConfigManager {
             result = "updated",
             "notify config update"
         );
-        self.reload_config(new_config).await
+        transition.wait().await
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write};
-    use crate::NotificationError;
+    use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
     use crate::rules::RulesMap;
+    use crate::{NotificationError, NotificationRuntimeState};
     use crate::{
         integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
         runtime_facade::NotifyRuntimeFacade,
@@ -464,7 +396,10 @@ mod tests {
     use rustfs_s3_types::EventName;
     use rustfs_targets::ReplayWorkerManager;
     use rustfs_targets::arn::TargetID;
-    use std::sync::Arc;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    };
     use tokio::sync::{RwLock, Semaphore};
 
     fn build_manager() -> NotifyConfigManager {
@@ -474,9 +409,10 @@ mod tests {
         let rule_engine = NotifyRuleEngine::new();
         let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
         let target_list = notifier.target_list();
-        let runtime_facade = NotifyRuntimeFacade::new(
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
             target_list,
             Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
             Arc::new(Semaphore::new(4)),
             metrics,
         );
@@ -524,78 +460,39 @@ mod tests {
             .reload_config(Config::default())
             .await
             .expect("reload_config should succeed for empty targets");
+        assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
     }
 
-    // Regression test for backlog#968: the read-modify-write over the persisted
-    // notify config must be serialized. Many tasks concurrently add a distinct
-    // target through the same production RMW path (`serialized_read_modify_write`,
-    // which holds the global RMW lock across read→modify→write) against a shared
-    // in-memory backend. Every update must survive — no lost updates. Without the
-    // lock, concurrent tasks would read the same base config and clobber each
-    // other's writes, leaving only a subset of targets.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_config_updates_preserve_all_targets() {
-        // Shared in-memory stand-in for the persisted config backend.
-        let backend = Arc::new(RwLock::new(Config::default()));
-        const TASKS: usize = 32;
+    #[tokio::test]
+    async fn read_modify_write_publishes_only_after_save() {
+        let saved = Arc::new(AtomicBool::new(false));
+        let saved_by_writer = saved.clone();
+        let observed_by_publisher = saved.clone();
 
-        let mut handles = Vec::with_capacity(TASKS);
-        for idx in 0..TASKS {
-            let backend = backend.clone();
-            handles.push(tokio::spawn(async move {
-                let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase();
-                let tname = format!("target-{idx}");
-
-                let read_backend = backend.clone();
-                let save_backend = backend.clone();
-
-                let result = serialized_read_modify_write(
-                    |config: &mut Config| {
-                        config
-                            .0
-                            .entry(ttype.clone())
-                            .or_default()
-                            .insert(tname.clone(), KVS::default());
-                        true
-                    },
-                    move || async move {
-                        let snapshot = read_backend.read().await.clone();
-                        // Yield inside the critical section to widen the race window:
-                        // if the RMW were not serialized, other tasks would read this
-                        // same base snapshot and their writes would clobber ours.
-                        tokio::task::yield_now().await;
-                        Ok::<_, NotifyConfigStoreError>(snapshot)
-                    },
-                    move |config: Config| async move {
-                        *save_backend.write().await = config;
-                        Ok::<_, NotifyConfigStoreError>(())
-                    },
-                )
-                .await
-                .expect("serialized RMW should succeed");
-                assert!(result.is_some(), "modifier reported a change, expected Some(config)");
-            }));
-        }
-
-        for handle in handles {
-            handle.await.expect("update task should not panic");
-        }
-
-        let final_config = backend.read().await;
-        let webhook_targets = final_config
-            .0
-            .get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase())
-            .expect("webhook subsystem should exist after updates");
-
-        assert_eq!(
-            webhook_targets.len(),
-            TASKS,
-            "all concurrent target additions must be preserved (no lost updates)"
-        );
-        for idx in 0..TASKS {
-            let tname = format!("target-{idx}");
-            assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost");
-        }
+        read_modify_write(
+            |config| {
+                config
+                    .0
+                    .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+                    .or_default()
+                    .insert("primary".to_string(), KVS::default());
+                true
+            },
+            || async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
+            move |_config| async move {
+                saved_by_writer.store(true, Ordering::Release);
+                Ok::<_, NotifyConfigStoreError>(())
+            },
+            move |_config| {
+                assert!(
+                    observed_by_publisher.load(Ordering::Acquire),
+                    "publication must observe the completed save"
+                );
+            },
+        )
+        .await
+        .expect("read-modify-write should succeed")
+        .expect("changed config should publish");
     }
 
     #[test]

crates/notify/src/error.rs

@@ -15,6 +15,7 @@
 use rustfs_targets::{TargetError, arn::TargetID};
 use std::io;
 use thiserror::Error;
+use tokio::task::JoinError;
 
 /// Errors related to the notification system's lifecycle.
 #[derive(Debug, Error)]
@@ -70,3 +71,24 @@ pub enum NotificationError {
     #[error("Storage not available: {0}")]
     StorageNotAvailable(String),
 }
+
+pub(crate) fn transition_join_error(error: JoinError) -> NotificationError {
+    let reason = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    NotificationError::Initialization(format!("Notification lifecycle transition task {reason}"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::transition_join_error;
+
+    #[tokio::test]
+    async fn transition_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = transition_join_error(join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+}

crates/notify/src/global.rs

@@ -14,38 +14,86 @@
 
 use crate::{
     BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot,
-    NotificationSystem, NotificationTargetMetricSnapshot,
+    NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error,
 };
 use rustfs_config::server_config::Config;
 use rustfs_s3_types::EventName;
 use rustfs_targets::arn::TargetID;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak};
 use tracing::error;
 
 static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
+static LEGACY_INITIALIZATION: LazyLock<Mutex<Option<LegacyInitialization>>> = LazyLock::new(|| Mutex::new(None));
+
+enum LegacyInitialization {
+    Initializing(Weak<NotificationSystem>),
+    Retryable(Weak<NotificationSystem>),
+    Initialized,
+}
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_GLOBAL: &str = "global";
 const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";
 
-/// Initialize the global notification system with the given configuration.
-/// This function should only be called once throughout the application life cycle.
-pub async fn initialize(config: Config) -> Result<(), NotificationError> {
-    // `new` is synchronous and responsible for creating instances
-    let system = NotificationSystem::new(config);
-    // `init` is asynchronous and responsible for performing I/O-intensive initialization
-    system.init().await?;
+fn notification_system_or_init(config: Config) -> Arc<NotificationSystem> {
+    NOTIFICATION_SYSTEM
+        .get_or_init(|| Arc::new(NotificationSystem::new(config)))
+        .clone()
+}
 
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(losing_system) => {
-            // Another initializer won the race. `init()` above already started this
-            // system's targets and replay workers, so simply dropping it would leak
-            // those background tasks. Shut the losing instance down cleanly before
-            // reporting the conflict (backlog#984).
-            losing_system.shutdown().await;
-            Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized))
+/// Initialize the global notification system with the given configuration.
+///
+/// This preserves the historical one-shot API contract. Server lifecycle code
+/// that needs idempotent reconciliation should use [`reconcile`] instead.
+pub async fn initialize(config: Config) -> Result<(), NotificationError> {
+    let system = {
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        match legacy.as_ref() {
+            Some(LegacyInitialization::Retryable(system)) => {
+                let Some(system) = system.upgrade() else {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                };
+                if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
+            Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => {
+                return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+            }
+            None => {
+                if NOTIFICATION_SYSTEM.get().is_some() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                let system = Arc::new(NotificationSystem::new(config.clone()));
+                if NOTIFICATION_SYSTEM.set(system.clone()).is_err() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
         }
-    }
+    };
+
+    let task_system = system.clone();
+    tokio::spawn(async move {
+        let result = task_system.set_targets_enabled(true, Some(config)).await;
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        if matches!(
+            legacy.as_ref(),
+            Some(LegacyInitialization::Initializing(current))
+                if current.upgrade().is_some_and(|current| Arc::ptr_eq(&current, &task_system))
+        ) {
+            *legacy = Some(if result.is_ok() {
+                LegacyInitialization::Initialized
+            } else {
+                LegacyInitialization::Retryable(Arc::downgrade(&task_system))
+            });
+        }
+        result
+    })
+    .await
+    .map_err(transition_join_error)?
 }
 
 /// Initialize the global notification system only for live in-process consumers.
@@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
 /// ListenBucketNotification clients can receive live events even when external
 /// notification targets are disabled.
 pub fn initialize_live_events() -> Result<(), NotificationError> {
-    let system = NotificationSystem::new(Config::new());
-
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)),
+    if NOTIFICATION_SYSTEM
+        .set(Arc::new(NotificationSystem::new(Config::new())))
+        .is_err()
+    {
+        return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
     }
+    Ok(())
+}
+
+/// Ensures the stable process-wide live-event container exists.
+pub fn ensure_live_events() -> Arc<NotificationSystem> {
+    notification_system_or_init(Config::new())
+}
+
+/// Ensures the stable singleton exists and reconciles its target runtime.
+pub async fn reconcile(config: Config) -> Result<(), NotificationError> {
+    let system = notification_system_or_init(config.clone());
+    system.set_targets_enabled(true, Some(config)).await
 }
 
 /// Returns a handle to the global NotificationSystem instance.

crates/notify/src/integration.rs

@@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::{EventNotifier, TargetList};
 use crate::services::NotifyServices;
 use crate::{
-    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    Event,
+    error::NotificationError,
+    lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
+    pipeline::LiveEventHistory,
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
     rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
@@ -160,11 +165,12 @@ impl NotificationMetrics {
 
 /// The notification system that integrates all components
 pub struct NotificationSystem {
-    /// The event notifier
+    /// Event dispatcher. Runtime target mutation remains lifecycle-owned.
     pub notifier: Arc<EventNotifier>,
-    /// The target registry
+    /// Target factory registry. Creating a target does not publish it.
     pub registry: Arc<TargetRegistry>,
-    /// The current configuration
+    /// The current cached configuration. Runtime publication must still go
+    /// through the lifecycle methods on this type.
     pub config: Arc<RwLock<Config>>,
     services: NotifyServices,
 }
@@ -220,10 +226,12 @@ impl NotificationSystem {
         self.services.runtime_view.get_active_targets().await
     }
 
-    /// Gets the complete Target list, including both active and inactive Targets.
-    ///
-    /// # Return
-    /// An `Arc<RwLock<TargetList>>` containing all Targets.
+    pub async fn config_snapshot(&self) -> Config {
+        self.config.read().await.clone()
+    }
+
+    /// Gets a read-only runtime container handle. Public mutation methods on
+    /// `TargetList` are intentionally unavailable outside this crate.
     pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
         self.services.runtime_view.get_all_targets()
     }
@@ -327,6 +335,43 @@ impl NotificationSystem {
         self.services.config_manager.reload_config(new_config).await
     }
 
+    /// Synchronously publishes a config generation without changing target mode.
+    pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().update_config(new_config)
+    }
+
+    /// Reconciles the cached and active configuration with the persisted
+    /// server config without changing the target-runtime mode.
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config().await
+    }
+
+    /// Reconciles from an explicitly selected storage context.
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config_from_store(store).await
+    }
+
+    /// Enables or suspends configured notification targets without replacing
+    /// the process-wide live-event container.
+    pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
+        self.publish_targets_enabled(enabled, config).wait().await
+    }
+
+    /// Synchronously accepts a target-runtime mode transition. The returned
+    /// receipt can be awaited after the caller releases its persistence lock.
+    pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().set_mode(enabled, config)
+    }
+
+    pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
+        self.services.config_manager.lifecycle().state()
+    }
+
+    /// Returns whether the latest accepted lifecycle generation is active.
+    pub fn runtime_lifecycle_is_converged(&self) -> bool {
+        self.services.config_manager.lifecycle().is_converged()
+    }
+
     /// Loads the bucket notification configuration
     pub async fn load_bucket_notification_config(
         &self,
@@ -365,9 +410,13 @@ impl NotificationSystem {
         self.services.runtime_view.runtime_status_snapshot().await
     }
 
-    // Add a method to shut down the system
     pub async fn shutdown(&self) {
-        self.services.runtime_facade.shutdown().await;
+        let _ = self.shutdown_checked().await;
+    }
+
+    /// Irreversibly terminates the notification target runtime for this process.
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.lifecycle().terminate().wait().await
     }
 }
 

crates/notify/src/lib.rs

@@ -25,6 +25,7 @@ mod event;
 pub mod factory;
 mod global;
 pub mod integration;
+mod lifecycle;
 mod notification_system_subscriber;
 pub mod notifier;
 mod pipeline;
@@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
 pub use global::{
-    initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
-    notification_target_metrics, notifier_global,
+    ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot,
+    notification_system, notification_target_metrics, notifier_global, reconcile,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState};
 pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline};
 pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
@@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade;
 pub use runtime_view::NotifyRuntimeView;
 pub use services::NotifyServices;
 pub use status_view::NotifyStatusView;
+pub use storage_api::NotifyStore;
 pub(crate) use storage_api::crate_boundary::{
-    read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+    read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+    resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+    with_notify_server_config_write_lock,
 };

crates/notify/src/notifier.rs

@@ -12,13 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
+use crate::error::NotificationError;
+use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
+use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
-use std::sync::Arc;
-use tokio::sync::{RwLock, Semaphore};
+use rustfs_targets::{SharedTarget, TargetRuntimeManager};
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
+use tokio::sync::{RwLock, Semaphore, watch};
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};
 
 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
 const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 
+async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
+    for handle in handles {
+        if let Err(e) = handle.await {
+            let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
+            error!(
+                event = EVENT_NOTIFY_DISPATCH_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_DISPATCH,
+                reason,
+                "Notify dispatch task failed"
+            );
+        }
+    }
+}
+
 pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;
 
+static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
+
+pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
+    let key = Arc::as_ptr(target_list) as usize;
+    let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
+    gates.retain(|_, gate| gate.strong_count() != 0);
+    if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
+        return gate;
+    }
+    let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
+    gates.insert(key, Arc::downgrade(&gate));
+    gate
+}
+
+pub(crate) struct DirectDispatchTracker {
+    cancellation: CancellationToken,
+    inflight: watch::Sender<usize>,
+}
+
+impl DirectDispatchTracker {
+    fn new() -> Self {
+        let (inflight, _) = watch::channel(0);
+        Self {
+            cancellation: CancellationToken::new(),
+            inflight,
+        }
+    }
+
+    fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
+        self.inflight.send_modify(|count| *count += 1);
+        DirectDispatchLease {
+            tracker: Arc::clone(self),
+        }
+    }
+
+    pub(crate) async fn wait_idle(&self) {
+        let mut inflight = self.inflight.subscribe();
+        while *inflight.borrow_and_update() != 0 {
+            if inflight.changed().await.is_err() {
+                return;
+            }
+        }
+    }
+
+    fn cancel_pending(&self) {
+        self.cancellation.cancel();
+    }
+}
+
+struct DirectDispatchLease {
+    tracker: Arc<DirectDispatchTracker>,
+}
+
+impl DirectDispatchLease {
+    fn cancellation(&self) -> CancellationToken {
+        self.tracker.cancellation.clone()
+    }
+}
+
+impl Drop for DirectDispatchLease {
+    fn drop(&mut self) {
+        self.tracker.inflight.send_modify(|count| {
+            if let Some(next) = count.checked_sub(1) {
+                *count = next;
+            } else {
+                error!(
+                    event = EVENT_NOTIFY_DISPATCH_FAILED,
+                    component = LOG_COMPONENT_NOTIFY,
+                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                    reason = "direct_lease_underflow",
+                    "Notify direct dispatch lease accounting underflowed"
+                );
+            }
+        });
+    }
+}
+
 /// Resolves the effective send concurrency (semaphore permit count).
 ///
 /// A value of `0` would build a zero-permit semaphore, so `acquire` never
@@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize {
 
 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
+    dispatch_gate: Arc<RwLock<()>>,
+    enqueue_limiter: Arc<Semaphore>,
     metrics: Arc<NotificationMetrics>,
     rule_engine: NotifyRuleEngine,
     target_list: SharedNotifyTargetList,
@@ -82,10 +182,14 @@ impl EventNotifier {
     /// Returns a new instance of EventNotifier.
     pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
         let max_inflight = resolve_send_concurrency();
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
         EventNotifier {
+            dispatch_gate,
+            enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
             metrics,
             rule_engine,
-            target_list: Arc::new(RwLock::new(TargetList::new())),
+            target_list,
             send_limiter: Arc::new(Semaphore::new(max_inflight)),
         }
     }
@@ -99,6 +203,10 @@ impl EventNotifier {
         Arc::clone(&self.target_list)
     }
 
+    pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
+        self.dispatch_gate.clone()
+    }
+
     /// Returns a list of ARNs for the registered targets
     ///
     /// # Arguments
@@ -115,12 +223,9 @@ impl EventNotifier {
             .collect()
     }
 
-    /// Removes all targets
     pub async fn remove_all_bucket_targets(&self) {
         let mut target_list_guard = self.target_list.write().await;
-        // The logic for sending cancel signals via stream_cancel_senders would be removed.
-        // TargetList::clear_targets_only already handles calling target.close().
-        target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
+        target_list_guard.clear_targets_only().await;
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -155,7 +260,14 @@ impl EventNotifier {
             return;
         }
         let target_ids_len = target_ids.len();
-        let mut handles = vec![];
+        let mut deferred_handles = Vec::new();
+        let mut direct_handles = Vec::new();
+
+        // A lifecycle writer holds this gate only while handing queue-store
+        // ownership from one runtime generation to the next. Taking the read
+        // guard before cloning targets means the writer both blocks new sends
+        // and drains every save already using the old generation.
+        let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);
 
         // Use scope to limit the borrow scope of target_list
         let target_list_guard = self.target_list.read().await;
@@ -185,10 +297,17 @@ impl EventNotifier {
                     );
                     continue;
                 }
-                let limiter = self.send_limiter.clone();
+                let is_deferred = target_for_task.store().is_some();
+                let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
+                let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
+                let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
+                let limiter = if is_deferred {
+                    self.enqueue_limiter.clone()
+                } else {
+                    self.send_limiter.clone()
+                };
                 let metrics = self.metrics.clone();
                 let event_clone = event.clone();
-                let is_deferred = target_for_task.store().is_some();
                 let target_name_for_task = target_for_task.name(); // Get the name before generating the task
                 debug!(
                     event = EVENT_NOTIFY_DISPATCH_STARTED,
@@ -207,9 +326,32 @@ impl EventNotifier {
                     data: event_clone.as_ref().clone(),
                 });
                 let handle = tokio::spawn(async move {
+                    let _direct_dispatch_lease = direct_dispatch_lease;
+                    let _deferred_dispatch_guard = deferred_dispatch_guard;
                     metrics.increment_processing();
-                    let _permit = match limiter.acquire_owned().await {
-                        Ok(p) => p,
+                    let permit = if let Some(cancellation) = direct_cancellation {
+                        tokio::select! {
+                            biased;
+                            _ = cancellation.cancelled() => {
+                                metrics.decrement_processing();
+                                metrics.increment_skipped();
+                                debug!(
+                                    event = EVENT_NOTIFY_DISPATCH_SKIPPED,
+                                    component = LOG_COMPONENT_NOTIFY,
+                                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                                    target_id = %target_name_for_task,
+                                    reason = "runtime_generation_replaced",
+                                    "Skipped pending direct notify dispatch"
+                                );
+                                return;
+                            }
+                            permit = limiter.acquire_owned() => permit,
+                        }
+                    } else {
+                        limiter.acquire_owned().await
+                    };
+                    let _permit = match permit {
+                        Ok(permit) => permit,
                         Err(e) => {
                             error!(
                                 event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -218,7 +360,7 @@ impl EventNotifier {
                                 target_id = %target_name_for_task,
                                 error = %e,
                                 reason = "permit_acquire_failed",
-                                "Failed to acquire notify send permit"
+                                "Failed to acquire notify dispatch permit"
                             );
                             metrics.increment_failed();
                             return;
@@ -260,7 +402,11 @@ impl EventNotifier {
                         );
                     }
                 });
-                handles.push(handle);
+                if is_deferred {
+                    deferred_handles.push(handle);
+                } else {
+                    direct_handles.push(handle);
+                }
             } else {
                 warn!(
                     event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -276,19 +422,13 @@ impl EventNotifier {
         // target_list is automatically released here
         drop(target_list_guard);
 
-        // Wait for all tasks to be completed
-        for handle in handles {
-            if let Err(e) = handle.await {
-                error!(
-                    event = EVENT_NOTIFY_DISPATCH_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_DISPATCH,
-                    error = %e,
-                    reason = "join_failed",
-                    "Notify dispatch task failed"
-                );
-            }
-        }
+        // Every store-backed save owns a share of the generation guard, so
+        // caller cancellation cannot race lifecycle handoff with an enqueue.
+        // Direct targets own no queue store and may finish against the detached
+        // target while lifecycle progresses.
+        drop(dispatch_guard);
+        wait_for_dispatch_tasks(deferred_handles).await;
+        wait_for_dispatch_tasks(direct_handles).await;
         debug!(
             event = EVENT_NOTIFY_DISPATCH_COMPLETED,
             component = LOG_COMPONENT_NOTIFY,
@@ -320,7 +460,7 @@ impl EventNotifier {
             target_list_guard.add(target)?;
         }
 
-        info!(
+        tracing::info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
             subsystem = LOG_SUBSYSTEM_DISPATCH,
@@ -334,6 +474,7 @@ impl EventNotifier {
 
 /// A thread-safe list of targets
 pub struct TargetList {
+    direct_dispatches: Arc<DirectDispatchTracker>,
     /// Map of TargetID to Target
     runtime: TargetRuntimeManager<Event>,
 }
@@ -348,6 +489,7 @@ impl TargetList {
     /// Creates a new TargetList
     pub fn new() -> Self {
         TargetList {
+            direct_dispatches: Arc::new(DirectDispatchTracker::new()),
             runtime: TargetRuntimeManager::new(),
         }
     }
@@ -435,6 +577,20 @@ impl TargetList {
         self.runtime.status_snapshot(replay_workers)
     }
 
+    fn direct_dispatch_lease(&self) -> DirectDispatchLease {
+        self.direct_dispatches.acquire()
+    }
+
+    pub(crate) fn replace_runtime(
+        &mut self,
+        replacement: TargetRuntimeManager<Event>,
+    ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
+        let runtime = std::mem::replace(&mut self.runtime, replacement);
+        let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
+        direct_dispatches.cancel_pending();
+        (runtime, direct_dispatches)
+    }
+
     pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
         &mut self.runtime
     }
@@ -458,7 +614,7 @@ mod tests {
     use rustfs_s3_types::EventName;
     use rustfs_targets::StoreError;
     use rustfs_targets::{
-        TargetError,
+        ReplayWorkerManager, TargetError,
         store::{Key, QueueStore, Store},
         target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
     };
@@ -466,6 +622,7 @@ mod tests {
         Arc,
         atomic::{AtomicUsize, Ordering},
     };
+    use tokio::sync::Notify;
 
     #[tokio::test]
     async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
@@ -525,19 +682,44 @@ mod tests {
 
     #[derive(Clone)]
     struct TestTarget {
+        block_first_save: Option<(Arc<Notify>, Arc<Notify>)>,
+        close_calls: Arc<AtomicUsize>,
+        close_entered: Option<Arc<Notify>>,
         id: TargetID,
         enabled: bool,
         save_calls: Arc<AtomicUsize>,
+        selected_calls: Arc<AtomicUsize>,
+        store: Option<QueueStore<QueuedPayload>>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str, enabled: bool) -> Self {
             Self {
+                block_first_save: None,
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                close_entered: None,
                 id: TargetID::new(id.to_string(), name.to_string()),
                 enabled,
                 save_calls: Arc::new(AtomicUsize::new(0)),
+                selected_calls: Arc::new(AtomicUsize::new(0)),
+                store: None,
             }
         }
+
+        fn with_blocked_first_save(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.block_first_save = Some((entered, release));
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
+
+        fn with_close_observer(mut self, close_entered: Arc<Notify>) -> Self {
+            self.close_entered = Some(close_entered);
+            self
+        }
     }
 
     #[async_trait]
@@ -554,7 +736,13 @@ mod tests {
         }
 
         async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-            self.save_calls.fetch_add(1, Ordering::SeqCst);
+            let call = self.save_calls.fetch_add(1, Ordering::SeqCst);
+            if call == 0
+                && let Some((entered, release)) = &self.block_first_save
+            {
+                entered.notify_one();
+                release.notified().await;
+            }
             Ok(())
         }
 
@@ -563,11 +751,17 @@ mod tests {
         }
 
         async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(close_entered) = &self.close_entered {
+                close_entered.notify_one();
+            }
             Ok(())
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -580,10 +774,425 @@ mod tests {
         }
 
         fn is_enabled(&self) -> bool {
+            self.selected_calls.fetch_add(1, Ordering::SeqCst);
             self.enabled
         }
     }
 
+    #[tokio::test]
+    async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("gated-target", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let first_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
+            _ = std::future::ready(()) => {}
+        }
+
+        save_release.notify_one();
+        first_dispatch.await.expect("first dispatch task should finish");
+        let pause_guard = pause.await;
+
+        let replacement = TestTarget::new("gated-target", "webhook", true);
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.clear();
+            target_list
+                .add(Arc::new(replacement.clone()))
+                .expect("replacement target install should succeed");
+        }
+
+        let mut second_dispatch =
+            Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
+        tokio::select! {
+            biased;
+            _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(
+            replacement.selected_calls.load(Ordering::SeqCst),
+            0,
+            "a paused dispatch must not select a target from the replacement generation early"
+        );
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0);
+
+        drop(pause_guard);
+        second_dispatch.await;
+        assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let target =
+            TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone());
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
+        };
+        drop(pause_guard);
+        save_release.notify_one();
+        dispatch.await.expect("direct dispatch should finish after release");
+    }
+
+    #[tokio::test]
+    async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let first_entered = Arc::new(Notify::new());
+        let first_release = Arc::new(Notify::new());
+        let close_entered = Arc::new(Notify::new());
+        let target = TestTarget::new("direct-target", "webhook", true)
+            .with_blocked_first_save(first_entered.clone(), first_release.clone())
+            .with_close_observer(close_entered);
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target should install");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics.clone(),
+        );
+        let first = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        first_entered.notified().await;
+
+        // This task selects the old generation and acquires its lease before
+        // waiting for the saturated direct-send permit.
+        let second = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while target.selected_calls.load(Ordering::SeqCst) != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("both direct sends should select the old generation");
+
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+        let mut replace = Box::pin(facade.replace_targets(activation));
+        tokio::select! {
+            biased;
+            result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 0);
+
+        first_release.notify_one();
+        first.await.expect("first direct dispatch should finish");
+        second.await.expect("permit-waiting direct dispatch should be cancelled");
+        replace.await.expect("replacement should close after direct leases drain");
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(metrics.processing_count(), 0);
+        assert_eq!(metrics.processed_count(), 1);
+        assert_eq!(metrics.skipped_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn caller_abort_does_not_release_deferred_generation_lease() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target should install");
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        dispatch.abort();
+        let _ = dispatch.await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("caller abort released the deferred generation lease"),
+            _ = std::future::ready(()) => {}
+        }
+        save_release.notify_one();
+        let pause_guard = pause.await;
+        drop(pause_guard);
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_concurrency_is_bounded() {
+        const LIMIT: usize = 2;
+        const TARGETS: usize = 3;
+
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
+            ..EventNotifier::new(metrics, rule_engine.clone())
+        });
+        let entered = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let mut targets = Vec::new();
+        let mut rules_map = RulesMap::new();
+        for index in 0..TARGETS {
+            let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true)
+                .with_blocked_first_save(entered.clone(), release.clone())
+                .with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event"));
+            rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+            notifier
+                .target_list()
+                .write()
+                .await
+                .add(Arc::new(target.clone()))
+                .expect("target should install");
+            targets.push(target);
+        }
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        let total_calls = || {
+            targets
+                .iter()
+                .map(|target| target.save_calls.load(Ordering::SeqCst))
+                .sum::<usize>()
+        };
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != LIMIT {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the configured number of enqueues should enter");
+        for _ in 0..10 {
+            tokio::task::yield_now().await;
+        }
+        assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
+
+        release.notify_waiters();
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != TARGETS {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the waiting enqueue should enter after a permit is released");
+        release.notify_waiters();
+        dispatch.await.expect("all bounded enqueues should finish");
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let direct_entered = Arc::new(Notify::new());
+        let direct_release = Arc::new(Notify::new());
+        let direct =
+            TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone());
+        let deferred_entered = Arc::new(Notify::new());
+        let deferred_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let deferred = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(deferred_entered.clone(), deferred_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut direct_rules = RulesMap::new();
+        direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone());
+        rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
+        let mut deferred_rules = RulesMap::new();
+        deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone());
+        rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.add(Arc::new(direct)).expect("direct target should install");
+            target_list.add(Arc::new(deferred)).expect("deferred target should install");
+        }
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let direct_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        direct_entered.notified().await;
+        let deferred_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
+            .await
+            .expect("queue persistence must not wait behind a direct network send permit");
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
+            _ = std::future::ready(()) => {}
+        }
+        deferred_release.notify_one();
+        deferred_dispatch
+            .await
+            .expect("deferred dispatch should finish after release");
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
+        };
+        drop(pause_guard);
+
+        direct_release.notify_one();
+        direct_dispatch.await.expect("direct dispatch should finish after release");
+    }
+
     #[tokio::test]
     async fn test_send_event_skips_disabled_target() {
         let rule_engine = NotifyRuleEngine::new();

crates/notify/src/registry.rs

@@ -38,6 +38,11 @@ impl TargetRegistry {
         TargetRegistry { plugins }
     }
 
+    #[cfg(test)]
+    pub(crate) fn with_plugins(plugins: TargetPluginRegistry<Event>) -> Self {
+        Self { plugins }
+    }
+
     pub fn supports_target_type(&self, target_type: &str) -> bool {
         self.plugins.supports_target_type(target_type)
     }
@@ -67,6 +72,15 @@ impl TargetRegistry {
     ) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
         self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
     }
+
+    pub(crate) async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+    ) -> Result<(Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>), TargetError> {
+        self.plugins
+            .create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX)
+            .await
+    }
 }
 
 #[cfg(test)]

crates/notify/src/runtime_facade.rs

@@ -12,13 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
-use rustfs_targets::{
-    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+use crate::{
+    Event, NotificationError,
+    error::transition_join_error,
+    integration::NotificationMetrics,
+    notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
+};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
+    RuntimeActivation, Target, TargetRuntimeManager,
+};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
 };
-use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{RwLock, Semaphore};
+use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};
 
 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
 const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
+const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
 
+pub(crate) struct DetachedNotifyRuntime {
+    direct_dispatches: Arc<DirectDispatchTracker>,
+    runtime: TargetRuntimeManager<Event>,
+    replay_workers: ReplayWorkerManager,
+}
+
+// Multi-lock publication order: replay_workers -> target_list ->
+// publication_gate. The lifecycle may already hold dispatch_gate while
+// entering this facade; no path that needs these three locks may acquire
+// publication_gate before either runtime lock.
 #[derive(Clone)]
 pub struct NotifyRuntimeFacade {
+    dispatch_gate: Arc<RwLock<()>>,
+    legacy_terminated: Arc<AtomicBool>,
     target_list: SharedNotifyTargetList,
     replay_workers: Arc<RwLock<ReplayWorkerManager>>,
-    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+    runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
 }
 
 impl NotifyRuntimeFacade {
@@ -41,6 +64,18 @@ impl NotifyRuntimeFacade {
         concurrency_limiter: Arc<Semaphore>,
         metrics: Arc<NotificationMetrics>,
     ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
+        Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
+    }
+
+    pub(crate) fn new_with_dispatch_gate(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        dispatch_gate: Arc<RwLock<()>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
         let replay_metrics = metrics;
         let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
             Arc::new(move |event: ReplayEvent<Event>| {
@@ -97,36 +132,166 @@ impl NotifyRuntimeFacade {
         );
 
         Self {
+            dispatch_gate,
+            legacy_terminated: Arc::new(AtomicBool::new(false)),
             target_list,
             replay_workers,
             runtime_adapter: Arc::new(runtime_adapter),
         }
     }
 
+    pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
+        self.dispatch_gate.clone().write_owned().await
+    }
+
     pub async fn activate_targets_with_replay(
         &self,
         targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
     ) -> RuntimeActivation<Event> {
-        self.runtime_adapter.activate_with_replay(targets).await
+        // The compatibility pair must not start replacement replay before
+        // replace_targets has stopped and joined the current generation.
+        self.runtime_adapter.prepare_dormant_compat_activation(targets).await
     }
 
-    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
-        // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+    pub(crate) async fn prepare_targets(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<Event> {
+        self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
+    }
+
+    pub(crate) async fn open_prepared_stores(
+        &self,
+        prepared: PreparedActivation<Event>,
+    ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
+        let runtime_adapter = self.runtime_adapter.clone();
+        tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
+            .await
+            .map_err(transition_join_error)
+    }
+
+    pub(crate) async fn commit_prepared<Committed>(
+        &self,
+        opened: OpenedActivation<Event>,
+        on_committed: Committed,
+    ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
+    where
+        Committed: FnOnce(bool),
+    {
+        // The lifecycle coordinator validates the generation immediately before
+        // entering the non-cancellable handoff. Once old replay workers have
+        // been joined, this generation must publish before a later accepted
+        // intent can run; abandoning it here would leave the old runtime
+        // visible without replay workers.
         let mut replay_workers = self.replay_workers.write().await;
         let mut target_list = self.target_list.write().await;
-        self.runtime_adapter
-            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
-            .await
-            .map_err(NotificationError::Target)?;
+
+        let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
+        let fully_activated = rejected.failure_summary().is_none();
+        let (runtime, replay_workers, direct_dispatches) =
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation);
+        on_committed(fully_activated);
+        (
+            DetachedNotifyRuntime {
+                direct_dispatches,
+                runtime,
+                replay_workers,
+            },
+            rejected,
+        )
+    }
+
+    pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
+    where
+        Committed: FnOnce(),
+    {
+        // Lock order: replay_workers -> target_list. The lifecycle coordinator
+        // crosses its publication barrier before stopping the old workers, so
+        // this non-cancellable commit must publish even if a newer intent was
+        // accepted while the workers joined.
+        let mut replay_workers = self.replay_workers.write().await;
+        let mut target_list = self.target_list.write().await;
+
+        let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+        let detached = DetachedNotifyRuntime {
+            direct_dispatches,
+            runtime,
+            replay_workers: std::mem::take(&mut *replay_workers),
+        };
+        on_committed();
+        detached
+    }
+
+    pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        // A caller may supply an activation created outside the compatibility
+        // prepare method. Stop any already-running replacement workers before
+        // entering the ordered handoff; the supported activate→replace pair is
+        // dormant here and therefore never overlaps the old generation.
+        self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
+        let dispatch_guard = self.pause_dispatch().await;
+        if self.legacy_terminated.load(Ordering::Acquire) {
+            drop(dispatch_guard);
+            self.runtime_adapter
+                .close_compat_activation(activation)
+                .await
+                .map_err(NotificationError::Target)?;
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        self.stop_active_replay_workers().await;
+        let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
+        let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        let (old_runtime, old_replay_workers, old_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation)
+        };
+        drop(dispatch_guard);
+        let close_old = self.close_detached_targets(DetachedNotifyRuntime {
+            direct_dispatches: old_direct_dispatches,
+            runtime: old_runtime,
+            replay_workers: old_replay_workers,
+        });
+        let close_open_rejected = self.close_prepared(open_rejected);
+        let close_activation_rejected = self.close_prepared(activation_rejected);
+        let (close_old, close_open_rejected, close_activation_rejected) =
+            tokio::join!(close_old, close_open_rejected, close_activation_rejected);
+        close_old?;
+        close_open_rejected?;
+        close_activation_rejected?;
+        if !activation_failures.is_empty() {
+            return Err(NotificationError::Initialization(format!(
+                "one or more notification targets failed to activate: {}",
+                activation_failures.join("; ")
+            )));
+        }
         Ok(())
     }
 
     pub async fn stop_replay_workers(&self) {
-        let mut replay_workers = self.replay_workers.write().await;
-        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.stop_active_replay_workers().await;
+    }
+
+    pub(crate) async fn stop_active_replay_workers(&self) {
+        let mut detached = {
+            let mut replay_workers = self.replay_workers.write().await;
+            std::mem::take(&mut *replay_workers)
+        };
+        self.runtime_adapter.stop_replay_workers(&mut detached).await;
     }
 
     pub async fn shutdown(&self) {
+        let _ = self.shutdown_checked().await;
+    }
+
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.legacy_terminated.store(true, Ordering::Release);
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -135,7 +300,14 @@ impl NotifyRuntimeFacade {
             "notify runtime lifecycle"
         );
 
-        let active_targets = self.replay_workers.read().await.len();
+        let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+            (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
+        };
+        let active_targets = detached_replay_workers.len();
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
             component = LOG_COMPONENT_NOTIFY,
@@ -145,25 +317,22 @@ impl NotifyRuntimeFacade {
             "notify runtime lifecycle"
         );
 
-        {
-            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
-            let mut replay_workers = self.replay_workers.write().await;
-            let mut target_list = self.target_list.write().await;
-            if let Err(err) = self
-                .runtime_adapter
-                .shutdown(target_list.runtime_mut(), &mut replay_workers)
-                .await
-            {
-                tracing::error!(
-                    event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_RUNTIME,
-                    error = %err,
-                    "Failed to shutdown notify runtime cleanly"
-                );
-            }
+        let shutdown_result = self
+            .close_detached(DetachedNotifyRuntime {
+                direct_dispatches: detached_direct_dispatches,
+                runtime: detached_runtime,
+                replay_workers: detached_replay_workers,
+            })
+            .await;
+        if let Err(err) = &shutdown_result {
+            tracing::error!(
+                event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                error = %err,
+                "Failed to shutdown notify runtime cleanly"
+            );
         }
-        tokio::time::sleep(Duration::from_millis(500)).await;
 
         info!(
             event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
@@ -172,6 +341,64 @@ impl NotifyRuntimeFacade {
             state = "stopped",
             "notify runtime lifecycle"
         );
+        shutdown_result
+    }
+
+    fn swap_activation(
+        target_list: &mut crate::notifier::TargetList,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<Event>,
+    ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
+        let mut replacement = TargetRuntimeManager::new();
+        for target in activation.targets {
+            replacement.add_arc(target);
+        }
+        let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
+        (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
+    }
+
+    pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
+        // Replay join is intentionally not wrapped in the target-close timeout:
+        // returning before a worker is confirmed stopped could let a
+        // replacement drain the same persistent queue concurrently.
+        self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
+    }
+
+    pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        // Direct sends are allowed to finish after the replacement runtime is
+        // published, but the old targets must remain open until every task that
+        // selected that generation has released its lease.
+        detached.direct_dispatches.wait_idle().await;
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
+            Ok(close_errors) if close_errors.is_empty() => Ok(()),
+            Ok(close_errors) => {
+                let targets = close_errors
+                    .into_iter()
+                    .map(|(target_id, _)| target_id)
+                    .collect::<Vec<_>>()
+                    .join("; ");
+                Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
+                    "Failed to close {targets}"
+                ))))
+            }
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing replaced notification targets".to_string(),
+            ))),
+        }
+    }
+
+    async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        self.stop_detached_replay(&mut detached).await;
+        self.close_detached_targets(detached).await
+    }
+
+    pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
+            Ok(result) => result.map_err(NotificationError::Target),
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing superseded notification targets".to_string(),
+            ))),
+        }
     }
 }
 
@@ -184,26 +411,50 @@ mod tests {
     };
     use async_trait::async_trait;
     use rustfs_targets::arn::TargetID;
-    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::store::{Key, QueueStore, Store};
     use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
     use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
-    use tokio::sync::{RwLock, Semaphore};
+    use tokio::sync::{Notify, RwLock, Semaphore};
 
     #[derive(Clone)]
     struct TestTarget {
+        close_entered: Option<Arc<Notify>>,
+        close_error: bool,
+        close_release: Option<Arc<Notify>>,
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        store: Option<QueueStore<QueuedPayload>>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
+                close_entered: None,
+                close_error: false,
+                close_release: None,
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                store: None,
             }
         }
+
+        fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.close_entered = Some(entered);
+            self.close_release = Some(release);
+            self
+        }
+
+        fn with_close_error(mut self) -> Self {
+            self.close_error = true;
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
     }
 
     #[async_trait]
@@ -229,11 +480,22 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(entered) = &self.close_entered {
+                entered.notify_one();
+            }
+            if let Some(release) = &self.close_release {
+                release.notified().await;
+            }
+            if self.close_error {
+                return Err(TargetError::Storage("forced close failure".to_string()));
+            }
             Ok(())
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -254,7 +516,13 @@ mod tests {
         let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
         let target_list = notifier.target_list();
         let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
-        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            replay_workers.clone(),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
         (facade, notifier, replay_workers)
     }
 
@@ -273,6 +541,26 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 0);
     }
 
+    #[tokio::test]
+    async fn compatibility_activation_stays_dormant_until_ordered_replace() {
+        let (facade, _, replay_workers) = build_facade();
+        let queue_root = tempfile::tempdir().expect("queue root");
+        let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
+        let target = TestTarget::new("primary", "webhook").with_store(store);
+
+        let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
+        assert_eq!(replay_workers.read().await.len(), 0);
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("ordered compatibility replace should succeed");
+        assert_eq!(replay_workers.read().await.len(), 1);
+        facade.shutdown_checked().await.expect("test runtime should shut down");
+    }
+
     #[tokio::test]
     async fn runtime_facade_replace_targets_commits_runtime_state() {
         let (facade, notifier, replay_workers) = build_facade();
@@ -292,4 +580,129 @@ mod tests {
         assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
         assert_eq!(replay_workers.read().await.len(), 0);
     }
+
+    #[tokio::test]
+    async fn runtime_queries_do_not_wait_for_target_close() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let close_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
+        close_release.notify_one();
+        shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect("shutdown should succeed");
+
+        let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_locks_are_released_while_replay_worker_joins() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let cancel_received = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
+        let join = tokio::spawn({
+            let cancel_received = cancel_received.clone();
+            let release = release.clone();
+            async move {
+                let _ = cancel_rx.recv().await;
+                cancel_received.notify_one();
+                release.notified().await;
+            }
+        });
+        replay_workers
+            .write()
+            .await
+            .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
+
+        let stop = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.stop_replay_workers().await }
+        });
+        cancel_received.notified().await;
+
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
+
+        release.notify_one();
+        stop.await.expect("stop task should finish");
+    }
+
+    #[tokio::test]
+    async fn shutdown_returns_close_error_after_detaching_runtime() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook").with_close_error();
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
+
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn shutdown_bounds_a_target_that_never_closes() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let never_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+        tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
+
+        let err = shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect_err("blocked close must time out");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
 }

crates/notify/src/services.rs

@@ -57,7 +57,13 @@ impl NotifyServices {
         live_event_history: Arc<RwLock<LiveEventHistory>>,
     ) -> Self {
         let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
-        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            stream_cancellers,
+            notifier.dispatch_gate(),
+            concurrency_limiter,
+            metrics.clone(),
+        );
         let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
         let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
         let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);

crates/notify/src/storage_api.rs

@@ -15,35 +15,70 @@
 use std::sync::Arc;
 
 use rustfs_ecstore::api::config::com::{
-    read_config_without_migrate as read_notify_config_without_migrate_from_backend,
-    save_server_config as save_notify_server_config_to_backend,
+    read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
+    read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
+    save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
+    with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
+    with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
 };
 use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
-pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
+pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;
 
 pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
     resolve_notify_object_store_handle_from_backend()
 }
 
-pub(crate) async fn read_notify_server_config_without_migrate(
+pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
     store: Arc<NotifyStore>,
 ) -> Result<rustfs_config::server_config::Config, String> {
-    read_notify_config_without_migrate_from_backend(store)
+    read_notify_config_without_migrate_from_backend_no_lock(store)
         .await
         .map_err(|err| err.to_string())
 }
 
-pub(crate) async fn save_notify_server_config(
+pub(crate) async fn read_existing_notify_server_config_no_lock(
+    store: Arc<NotifyStore>,
+) -> Result<rustfs_config::server_config::Config, String> {
+    read_existing_notify_config_from_backend_no_lock(store)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn save_notify_server_config_no_lock(
     store: Arc<NotifyStore>,
     config: &rustfs_config::server_config::Config,
 ) -> Result<(), String> {
-    save_notify_server_config_to_backend(store, config)
+    save_notify_server_config_to_backend_no_lock(store, config)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_write_lock_from_backend(store, operation)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_read_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_read_lock_from_backend(store, operation)
         .await
         .map_err(|err| err.to_string())
 }
 
 pub(crate) mod crate_boundary {
     pub(crate) use super::{
-        read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+        read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+        resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+        with_notify_server_config_write_lock,
     };
 }

crates/notify/tests/global_lifecycle.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::server_config::Config;
+use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile};
+use std::sync::Arc;
+
+fn assert_terminated(error: NotificationError) {
+    assert!(matches!(
+        error,
+        NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
+    ));
+}
+
+#[tokio::test]
+async fn global_singleton_survives_suspend_but_not_process_termination() {
+    let system = ensure_live_events();
+    let same_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &same_system));
+    assert!(Arc::ptr_eq(
+        &system,
+        &notification_system().expect("global notification system should exist")
+    ));
+
+    system
+        .set_targets_enabled(true, Some(Config::new()))
+        .await
+        .expect("empty target runtime should enable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .set_targets_enabled(false, None)
+        .await
+        .expect("disable should suspend targets without terminating the singleton");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
+
+    system
+        .set_targets_enabled(true, None)
+        .await
+        .expect("a suspended target runtime should be restartable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .shutdown_checked()
+        .await
+        .expect("process shutdown should terminate the target runtime");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+
+    let lazy_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &lazy_system));
+    assert_terminated(
+        reconcile(Config::new())
+            .await
+            .expect_err("lazy reconciliation must not restart a terminated process runtime"),
+    );
+    assert_terminated(
+        lazy_system
+            .reload_config(Config::new())
+            .await
+            .expect_err("config reload must not restart a terminated process runtime"),
+    );
+    assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+}

crates/notify/tests/legacy_initialize_retry.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
+use rustfs_config::server_config::{Config, KVS};
+use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT};
+use rustfs_notify::{initialize, notification_system};
+
+#[tokio::test]
+async fn failed_legacy_initialize_can_retry_the_stable_singleton() {
+    let mut invalid_target = KVS::new();
+    invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string());
+    invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
+    let mut invalid_config = Config::new();
+    invalid_config
+        .0
+        .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+        .or_default()
+        .insert("primary".to_string(), invalid_target);
+
+    initialize(invalid_config)
+        .await
+        .expect_err("invalid first target activation should fail");
+    initialize(Config::new())
+        .await
+        .expect("legacy initialize should retry after the configuration is fixed");
+
+    notification_system()
+        .expect("stable singleton should remain available")
+        .shutdown_checked()
+        .await
+        .expect("test runtime should terminate");
+}

crates/targets/Cargo.toml

@@ -18,6 +18,7 @@ rustfs-tls-runtime = { workspace = true }
 rustfs-s3-types = { workspace = true }
 rustfs-utils = { workspace = true, features = ["egress"] }
 async-trait = { workspace = true }
+futures-util = { workspace = true }
 async-nats = { workspace = true }
 deadpool-postgres = { workspace = true }
 hyper = { workspace = true, features = ["http2", "http1", "server"] }
@@ -26,6 +27,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust
 libc = { workspace = true }
 pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] }
 regex = { workspace = true }
+rayon = { workspace = true }
 reqwest = { workspace = true }
 rumqttc = { workspace = true, features = ["websocket"] }
 redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
@@ -38,6 +40,7 @@ sha2 = { workspace = true }
 snap = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
+tokio-util = { workspace = true }
 tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] }
 tokio-postgres-rustls = { workspace = true }
 tracing = { workspace = true }

crates/targets/src/config/common.rs

@@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet<St
         .max_by_key(|(field, _)| field.len())
 }
 
-pub(super) fn is_target_enabled(config: &KVS) -> bool {
-    config
-        .lookup(ENABLE_KEY)
-        .map(|v| {
-            EnableState::from_str(v.as_str())
-                .ok()
-                .map(|s| s.is_enabled())
-                .unwrap_or(false)
-        })
-        .unwrap_or(false)
+pub(super) fn is_target_enabled(config: &KVS) -> Result<bool, TargetError> {
+    let Some(value) = config.lookup(ENABLE_KEY) else {
+        return Ok(false);
+    };
+    EnableState::from_str(value.as_str())
+        .map(EnableState::is_enabled)
+        .map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'")))
 }
 
 pub(super) fn parse_target_bool(value: Option<&str>) -> Option<bool> {

crates/targets/src/config/instance.rs

@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use super::loader::collect_merged_target_configs_from_env;
+use super::loader::{
+    MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env,
+};
+use crate::TargetError;
 use crate::domain::TargetDomain;
 use rustfs_config::server_config::{Config, KVS};
 use std::collections::HashSet;
@@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances(
     normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
 }
 
+pub fn try_normalize_target_plugin_instances(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError> {
+    try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars())
+}
+
 pub fn normalize_target_plugin_instances_from_env<I>(
     config: &Config,
     descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
@@ -100,7 +110,7 @@ where
         .map(|field| (*field).to_string())
         .collect::<HashSet<_>>();
 
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         descriptor.subsystem,
         descriptor.route_prefix,
@@ -109,7 +119,42 @@ where
         env_vars,
     )
     .into_iter()
-    .map(|record| TargetPluginInstanceRecord {
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect()
+}
+
+pub fn try_normalize_target_plugin_instances_from_env<I>(
+    config: &Config,
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    env_vars: I,
+) -> Result<Vec<TargetPluginInstanceRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    let valid_fields = descriptor
+        .valid_fields
+        .iter()
+        .map(|field| (*field).to_string())
+        .collect::<HashSet<_>>();
+
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        descriptor.subsystem,
+        descriptor.route_prefix,
+        descriptor.target_type,
+        &valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .map(|record| target_plugin_instance_record(descriptor, record))
+    .collect())
+}
+
+fn target_plugin_instance_record(
+    descriptor: &TargetPluginInstanceCompatDescriptor<'_>,
+    record: MergedTargetConfigRecord,
+) -> TargetPluginInstanceRecord {
+    TargetPluginInstanceRecord {
         domain: descriptor.domain,
         plugin_id: descriptor.plugin_id.to_string(),
         target_type: descriptor.target_type.to_string(),
@@ -123,8 +168,7 @@ where
             has_env_instance: record.has_env_instance,
         },
         effective_config: record.effective_config,
-    })
-    .collect()
+    }
 }
 
 pub fn normalize_legacy_target_instances(
@@ -149,8 +193,9 @@ where
 mod tests {
     use super::{
         TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env,
-        normalize_target_plugin_instances_from_env,
+        try_normalize_target_plugin_instances_from_env,
     };
+    use crate::TargetError;
     use crate::domain::TargetDomain;
     use crate::manifest::builtin_target_manifest;
     use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
@@ -338,9 +383,50 @@ mod tests {
         let descriptor = notify_webhook_descriptor();
         let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())];
 
-        let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone());
+        let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone())
+            .expect("canonical normalization should succeed");
         let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env);
 
         assert_eq!(canonical, compatibility);
     }
+
+    #[test]
+    fn normalize_instances_rejects_invalid_enable_value() {
+        let error = try_normalize_target_plugin_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must be propagated by the public normalizer");
+
+        match error {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() {
+        let instances = normalize_legacy_target_instances_from_env(
+            &Config(HashMap::new()),
+            &notify_webhook_descriptor(),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(instances.len(), 2);
+        let good = instances
+            .iter()
+            .find(|instance| instance.instance_id == "good")
+            .expect("valid instance should remain present");
+        let bad = instances
+            .iter()
+            .find(|instance| instance.instance_id == "bad")
+            .expect("invalid legacy instance should remain visible");
+        assert!(good.enabled);
+        assert!(!bad.enabled);
+    }
 }

crates/targets/src/config/loader.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::common::{is_target_enabled, split_env_field_and_instance};
+use crate::TargetError;
 use rustfs_config::server_config::{Config, KVS};
 use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX};
 use std::collections::{HashMap, HashSet};
@@ -27,6 +28,15 @@ pub fn collect_target_configs(
     collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
 }
 
+pub fn try_collect_target_configs(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+) -> Result<Vec<(String, KVS)>, TargetError> {
+    try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars())
+}
+
 fn is_sensitive_target_field(field_name: &str) -> bool {
     let field_name = field_name.to_ascii_lowercase();
     field_name.contains("password")
@@ -123,7 +133,7 @@ pub fn collect_target_configs_from_env<I>(
 where
     I: IntoIterator<Item = (String, String)>,
 {
-    collect_merged_target_configs_from_env(
+    collect_merged_target_configs_compat_from_env(
         config,
         &format!("{route_prefix}{target_type}").to_lowercase(),
         route_prefix,
@@ -137,6 +147,30 @@ where
     .collect()
 }
 
+pub fn try_collect_target_configs_from_env<I>(
+    config: &Config,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Result<Vec<(String, KVS)>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    Ok(collect_merged_target_configs_from_env(
+        config,
+        &format!("{route_prefix}{target_type}").to_lowercase(),
+        route_prefix,
+        target_type,
+        valid_fields,
+        env_vars,
+    )?
+    .into_iter()
+    .filter(|record| record.enabled)
+    .map(|record| (record.instance_id, record.effective_config))
+    .collect())
+}
+
 pub(crate) fn collect_merged_target_configs_from_env<I>(
     config: &Config,
     section_name: &str,
@@ -144,7 +178,52 @@ pub(crate) fn collect_merged_target_configs_from_env<I>(
     target_type: &str,
     valid_fields: &HashSet<String>,
     env_vars: I,
+) -> Result<Vec<MergedTargetConfigRecord>, TargetError>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| result.map_err(|(_, err)| err))
+        .collect()
+}
+
+pub(crate) fn collect_merged_target_configs_compat_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
 ) -> Vec<MergedTargetConfigRecord>
+where
+    I: IntoIterator<Item = (String, String)>,
+{
+    collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars)
+        .into_iter()
+        .map(|result| match result {
+            Ok(record) => record,
+            Err((record, err)) => {
+                warn!(
+                    target_type,
+                    instance_id = %record.instance_id,
+                    error = %err,
+                    "Treating target instance with invalid enable configuration as disabled"
+                );
+                record
+            }
+        })
+        .collect()
+}
+
+fn collect_merged_target_config_results_from_env<I>(
+    config: &Config,
+    section_name: &str,
+    route_prefix: &str,
+    target_type: &str,
+    valid_fields: &HashSet<String>,
+    env_vars: I,
+) -> Vec<Result<MergedTargetConfigRecord, (MergedTargetConfigRecord, TargetError)>>
 where
     I: IntoIterator<Item = (String, String)>,
 {
@@ -220,14 +299,28 @@ where
             let redacted_config = redacted_target_config(&merged_config);
             debug!(instance_id = %id, ?redacted_config, "Merged target configuration");
         }
-        merged_configs.push(MergedTargetConfigRecord {
-            instance_id: id,
-            enabled: is_target_enabled(&merged_config),
-            effective_config: merged_config,
-            has_file_default,
-            has_file_instance,
-            has_env_default,
-            has_env_instance,
+        merged_configs.push(match is_target_enabled(&merged_config) {
+            Ok(enabled) => Ok(MergedTargetConfigRecord {
+                instance_id: id,
+                enabled,
+                effective_config: merged_config,
+                has_file_default,
+                has_file_instance,
+                has_env_default,
+                has_env_instance,
+            }),
+            Err(err) => Err((
+                MergedTargetConfigRecord {
+                    instance_id: id,
+                    enabled: false,
+                    effective_config: merged_config,
+                    has_file_default,
+                    has_file_instance,
+                    has_env_default,
+                    has_env_instance,
+                },
+                err,
+            )),
         });
     }
 
@@ -238,8 +331,9 @@ where
 mod tests {
     use super::{
         collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value,
-        redacted_target_config,
+        redacted_target_config, try_collect_target_configs_from_env,
     };
+    use crate::TargetError;
     use rustfs_config::notify::{
         ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
         ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX,
@@ -269,7 +363,7 @@ mod tests {
 
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -282,7 +376,8 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()),
                 ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()),
             ],
-        );
+        )
+        .expect("valid env target");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         assert_eq!(configs.len(), 2);
@@ -295,7 +390,7 @@ mod tests {
     #[test]
     fn collect_target_configs_discovers_enabled_instance_from_env() {
         let cfg = Config(HashMap::new());
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -307,7 +402,8 @@ mod tests {
                     "https://example.com/from-env".to_string(),
                 ),
             ],
-        );
+        )
+        .expect("valid target configs");
 
         assert_eq!(configs.len(), 1);
         assert_eq!(configs[0].0, "primary");
@@ -323,7 +419,7 @@ mod tests {
         subsystem.insert("_".to_string(), default_kvs);
         cfg.0.insert("notify_webhook".to_string(), subsystem);
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "webhook",
@@ -332,7 +428,8 @@ mod tests {
                 "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(),
                 "https://example.com/secondary".to_string(),
             )],
-        );
+        )
+        .expect("valid target configs");
 
         assert!(configs.is_empty());
     }
@@ -361,7 +458,7 @@ mod tests {
         let cfg = Config(HashMap::new());
         let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect();
 
-        let configs = collect_target_configs_from_env(
+        let configs = try_collect_target_configs_from_env(
             &cfg,
             NOTIFY_ROUTE_PREFIX,
             "redis",
@@ -372,7 +469,8 @@ mod tests {
                 (format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()),
                 (format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()),
             ],
-        );
+        )
+        .expect("valid redis target config");
 
         let configs: HashMap<String, KVS> = configs.into_iter().collect();
         let redis_config = configs.get("primary").expect("redis env target should be discovered");
@@ -383,6 +481,57 @@ mod tests {
         assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off"));
     }
 
+    #[test]
+    fn collect_target_configs_rejects_invalid_instance_enable_value() {
+        let err = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())],
+        )
+        .expect_err("invalid enable value must not look like a disabled target");
+
+        match err {
+            TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"),
+            other => panic!("expected a configuration error, got {other}"),
+        }
+    }
+
+    #[test]
+    fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() {
+        let configs = collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()),
+                ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()),
+            ],
+        );
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "good");
+    }
+
+    #[test]
+    fn collect_target_configs_preserves_whitespace_padded_legacy_value() {
+        let configs = try_collect_target_configs_from_env(
+            &Config(HashMap::new()),
+            NOTIFY_ROUTE_PREFIX,
+            "webhook",
+            &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]),
+            vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())],
+        )
+        .expect("the shared enable parser accepts surrounding whitespace");
+
+        assert_eq!(configs.len(), 1);
+        assert_eq!(configs[0].0, "primary");
+        assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on "));
+    }
+
     #[test]
     fn redact_target_field_value_redacts_sensitive_fields() {
         assert_eq!(redact_target_field_value("password", "secret"), "***redacted***");

crates/targets/src/config/mod.rs

@@ -21,10 +21,11 @@ pub use instance::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use loader::{
     collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs,
-    collect_target_configs_from_env,
+    collect_target_configs_from_env, try_collect_target_configs, try_collect_target_configs_from_env,
 };
 pub use target_args::{
     build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,

crates/targets/src/lib.rs

@@ -43,6 +43,7 @@ pub use config::{
     LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance,
     TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances,
     normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env,
+    try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env,
 };
 pub use control_plane::{
     TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError,
@@ -65,8 +66,9 @@ pub use plugin::{
     TargetPluginRegistry, TargetRequestValidator, boxed_target,
 };
 pub use runtime::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
+    OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot,
+    RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager,
+    activate_targets_with_replay,
     adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
     init_target_and_optionally_start_replay,
     ops_diagnostics::{

crates/targets/src/plugin.rs

@@ -14,8 +14,9 @@
 
 use crate::{
     PluginRuntimeAdapter, RuntimeActivation, Target, TargetError,
-    config::collect_target_configs,
+    config::try_collect_target_configs,
     manifest::{TargetPluginManifest, builtin_target_manifest},
+    target::with_deferred_queue_store_open,
 };
 use hashbrown::HashMap;
 use rustfs_config::server_config::{Config, KVS};
@@ -317,39 +318,68 @@ where
         config: &Config,
         route_prefix: &str,
     ) -> Result<Vec<BoxedTarget<E>>, TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, false)
+            .await
+            .map(|(targets, _)| targets)
+    }
+
+    /// Creates targets while deferring queue-store open until runtime handoff.
+    /// Unlike the compatibility activation API, lifecycle preparation reports
+    /// any invalid or unconstructable configured instance so the originating
+    /// Admin request cannot report a false success.
+    pub async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
+        self.create_targets_from_config_with_store_mode(config, route_prefix, true)
+            .await
+    }
+
+    async fn create_targets_from_config_with_store_mode(
+        &self,
+        config: &Config,
+        route_prefix: &str,
+        defer_store_open: bool,
+    ) -> Result<(Vec<BoxedTarget<E>>, Vec<String>), TargetError> {
         let mut successful_targets = Vec::new();
-        let mut failed_targets = 0usize;
+        let mut failures = Vec::new();
 
         for (target_type, plugin) in &self.plugins {
             info!(target_type = %target_type, "Start working on target type");
-            for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
+            for (id, merged_config) in try_collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set())? {
                 info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
-                match self.create_target(target_type, id.clone(), &merged_config) {
+                let created = if defer_store_open {
+                    with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config))
+                } else {
+                    self.create_target(target_type, id.clone(), &merged_config)
+                };
+                match created {
                     Ok(target) => {
                         info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
                         successful_targets.push(target);
                     }
-                    Err(err) => {
-                        failed_targets += 1;
-                        error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
+                    Err(_) => {
+                        failures.push(format!("{target_type}/{id}: target construction failed"));
+                        error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target");
                     }
                 }
             }
         }
 
-        if failed_targets > 0 {
+        if !failures.is_empty() {
             warn!(
                 created = successful_targets.len(),
-                failed = failed_targets,
+                failed = failures.len(),
                 "Some configured targets failed to create and were skipped"
             );
         }
         info!(
             count = successful_targets.len(),
-            failed = failed_targets,
+            failed = failures.len(),
             "All target processing completed"
         );
-        Ok(successful_targets)
+        Ok((successful_targets, failures))
     }
 
     pub async fn create_activation_from_config<A>(

crates/targets/src/runtime/adapter.rs

@@ -13,21 +13,53 @@
 // limitations under the License.
 
 use super::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+    OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
+    RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
+    start_replay_worker,
 };
 use crate::plugin::PluginEvent;
-use crate::{Target, TargetError};
+use crate::{SharedTarget, Target, TargetError};
 use async_trait::async_trait;
+use rayon::prelude::*;
 use std::future::Future;
+use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
 
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
 
+const MAX_PARALLEL_STORE_OPENS: usize = 4;
+
+static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(MAX_PARALLEL_STORE_OPENS)
+        .thread_name(|index| format!("rustfs-target-store-open-{index}"))
+        .build()
+});
+
+enum StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    Accepted(SharedTarget<E>),
+    Rejected { panicked: bool, target: SharedTarget<E> },
+}
+
+fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
+        Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
+        Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
+        Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
+    }
+}
+
 /// Shared runtime contract for target plugins.
 #[async_trait]
 pub trait PluginRuntimeAdapter<E>: Send + Sync
@@ -96,6 +128,234 @@ where
             stop_log_prefix: stop_log_prefix.into(),
         }
     }
+
+    pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, None).await
+    }
+
+    pub async fn prepare_targets_cancellable(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, Some(cancellation)).await
+    }
+
+    async fn prepare_targets_inner(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: Option<&CancellationToken>,
+    ) -> PreparedActivation<E> {
+        let mut prepared = Vec::with_capacity(targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        let mut targets = targets.into_iter();
+        while let Some(target) = targets.next() {
+            match prepare_target(target, cancellation).await {
+                PrepareTargetResult::Ready(target) => prepared.push(target),
+                PrepareTargetResult::Degraded { error, target } => {
+                    drop(error);
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: initialization failed", target.id()),
+                    });
+                    prepared.push(target);
+                }
+                PrepareTargetResult::Failed { error, target } => {
+                    drop(error);
+                    let target_id = target.id().to_string();
+                    tracing::warn!(
+                        target_id,
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: initialization failed"),
+                    });
+                    rejected_targets.push(Arc::from(target));
+                }
+                PrepareTargetResult::Cancelled(target) => {
+                    prepared.push(Arc::from(target));
+                    prepared.extend(targets.map(Arc::from));
+                    break;
+                }
+            }
+        }
+        PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: prepared,
+        }
+    }
+
+    /// Opens queue stores only after the previous runtime generation has been
+    /// quiesced. Targets whose stores cannot be opened retain the established
+    /// fault-isolation behavior and are returned for lock-free shutdown.
+    pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
+        let mut accepted = Vec::with_capacity(prepared.targets.len());
+        let mut failures = prepared.failures;
+        let mut rejected = prepared.rejected_targets;
+        let outcomes = if prepared.targets.len() < 2 {
+            prepared.targets.into_iter().map(open_target_store).collect()
+        } else {
+            match STORE_OPEN_POOL.as_ref() {
+                // Vec's indexed parallel iterator preserves configuration
+                // order in collect, keeping failure summaries deterministic.
+                Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
+                Err(err) => {
+                    tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
+                    prepared.targets.into_iter().map(open_target_store).collect()
+                }
+            }
+        };
+        for outcome in outcomes {
+            match outcome {
+                StoreOpenOutcome::Accepted(target) => accepted.push(target),
+                StoreOpenOutcome::Rejected { panicked, target } => {
+                    if panicked {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_panicked",
+                            "Target queue store panicked while opening during runtime handoff"
+                        );
+                    } else {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_failed",
+                            "Failed to open target queue store during runtime handoff"
+                        );
+                    }
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: queue store open failed", target.id()),
+                    });
+                    rejected.push(target);
+                }
+            }
+        }
+        (
+            OpenedActivation { targets: accepted },
+            PreparedActivation {
+                failures,
+                rejected_targets: rejected,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
+        let mut replay_workers = ReplayWorkerManager::new();
+        let mut accepted = Vec::with_capacity(opened.targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        for target in opened.targets {
+            let target_id = target.id().to_string();
+            let replay = catch_unwind(AssertUnwindSafe(|| {
+                target.store().filter(|_| target.is_enabled()).map(|store| {
+                    start_replay_worker(
+                        store.boxed_clone(),
+                        Arc::clone(&target),
+                        Arc::clone(&self.replay_hook),
+                        self.replay_semaphore.clone(),
+                        self.batch_timeout,
+                        self.idle_sleep,
+                    )
+                })
+            }));
+            let replay = match replay {
+                Ok(replay) => replay,
+                Err(_) => {
+                    tracing::error!(
+                        target_id,
+                        reason = "replay_activation_panicked",
+                        "Target replay activation panicked during runtime handoff"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: replay activation failed"),
+                    });
+                    rejected_targets.push(target);
+                    continue;
+                }
+            };
+            (self.replay_start_observer)(&target_id, replay.is_some());
+            if let Some((cancel_tx, join)) = replay {
+                replay_workers.insert_with_handle(target_id, cancel_tx, join);
+            }
+            accepted.push(target);
+        }
+
+        (
+            RuntimeActivation {
+                replay_workers,
+                targets: accepted,
+            },
+            PreparedActivation {
+                failures,
+                rejected_targets,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    #[doc(hidden)]
+    pub async fn prepare_dormant_compat_activation(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    ) -> RuntimeActivation<E> {
+        let PreparedActivation {
+            failures,
+            rejected_targets,
+            targets,
+        } = self.prepare_targets(targets).await;
+        let rejected = PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: Vec::new(),
+        };
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
+        }
+        RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn start_dormant_compat_activation(
+        &self,
+        activation: RuntimeActivation<E>,
+    ) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
+        let prepared = PreparedActivation {
+            failures: Vec::new(),
+            rejected_targets: Vec::new(),
+            targets: activation.targets,
+        };
+        let (opened, open_rejected) = self.open_prepared_stores(prepared);
+        let (activation, activation_rejected) = self.try_activate_prepared(opened);
+        (activation, open_rejected, activation_rejected)
+    }
+
+    #[doc(hidden)]
+    pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+        self.shutdown(&mut runtime, &mut activation.replay_workers).await
+    }
+
+    pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
+            runtime.add_arc(target);
+        }
+        let mut replay_workers = ReplayWorkerManager::new();
+        self.shutdown(&mut runtime, &mut replay_workers).await
+    }
 }
 
 #[async_trait]
@@ -104,36 +364,16 @@ where
     E: PluginEvent,
 {
     async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
-        let replay_hook = Arc::clone(&self.replay_hook);
-        let replay_start_observer = Arc::clone(&self.replay_start_observer);
-        let replay_semaphore = self.replay_semaphore.clone();
-        let batch_timeout = self.batch_timeout;
-        let idle_sleep = self.idle_sleep;
-
-        activate_targets_with_replay(targets, move |target| {
-            let replay_hook = Arc::clone(&replay_hook);
-            let replay_start_observer = Arc::clone(&replay_start_observer);
-            let replay_semaphore = replay_semaphore.clone();
-
-            async move {
-                init_target_and_optionally_start_replay(
-                    target,
-                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
-                    move |store, target| {
-                        start_replay_worker(
-                            store,
-                            target,
-                            Arc::clone(&replay_hook),
-                            replay_semaphore.clone(),
-                            batch_timeout,
-                            idle_sleep,
-                        )
-                    },
-                )
-                .await
-            }
-        })
-        .await
+        let prepared = self.prepare_targets(targets).await;
+        let (opened, rejected) = self.open_prepared_stores(prepared);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
+        }
+        let (activation, rejected) = self.try_activate_prepared(opened);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
+        }
+        activation
     }
 
     async fn replace_runtime_targets(
@@ -189,7 +429,7 @@ where
         if !close_errors.is_empty() {
             let detail = close_errors
                 .into_iter()
-                .map(|(target_id, err)| format!("{target_id}: {err}"))
+                .map(|(target_id, _)| target_id)
                 .collect::<Vec<_>>()
                 .join("; ");
             return Err(TargetError::Storage(format!("Failed to close {detail}")));
@@ -200,24 +440,143 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
     use crate::PluginEvent;
     use crate::arn::TargetID;
     use crate::store::{Key, QueueStore, Store};
     use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use crate::{StoreError, Target, TargetError};
     use async_trait::async_trait;
-    use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Condvar, Mutex};
     use std::time::Duration;
     use tempfile::tempdir;
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
+
+    type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
+
+    type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
+
+    #[derive(Clone)]
+    struct TestOpenStore {
+        before_clone: BeforeOpen,
+        before_open: BeforeOpen,
+        store: QueueStore<QueuedPayload>,
+    }
+
+    impl Store<QueuedPayload> for TestOpenStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            (self.before_open)();
+            self.store.open()
+        }
+
+        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put(item)
+        }
+
+        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put_multiple(items)
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            self.store.put_raw(data)
+        }
+
+        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            self.store.get(key)
+        }
+
+        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            self.store.get_multiple(key)
+        }
+
+        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            self.store.get_raw(key)
+        }
+
+        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+            self.store.del(key)
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            self.store.delete()
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            self.store.list()
+        }
+
+        fn len(&self) -> usize {
+            self.store.len()
+        }
+
+        fn is_empty(&self) -> bool {
+            self.store.is_empty()
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            (self.before_clone)();
+            Box::new(self.clone())
+        }
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGate {
+        changed: Condvar,
+        state: Mutex<StoreOpenGateState>,
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGateState {
+        active: usize,
+        max_active: usize,
+        released: bool,
+    }
+
+    impl StoreOpenGate {
+        fn enter(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.active += 1;
+            state.max_active = state.max_active.max(state.active);
+            self.changed.notify_all();
+            while !state.released {
+                state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
+            }
+            state.active -= 1;
+        }
+
+        fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
+            let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            let (state, _) = self
+                .changed
+                .wait_timeout_while(state, timeout, |state| state.max_active < expected)
+                .unwrap_or_else(|err| err.into_inner());
+            state.max_active >= expected
+        }
+
+        fn release(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.released = true;
+            self.changed.notify_all();
+        }
+
+        fn max_active(&self) -> usize {
+            self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
+        }
+    }
 
     #[derive(Clone)]
     struct TestTarget {
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        init_calls: Arc<AtomicUsize>,
+        init_entered: Option<Arc<Notify>>,
         init_fails: bool,
-        store: Option<QueueStore<QueuedPayload>>,
+        store: Option<Arc<TestStore>>,
     }
 
     impl TestTarget {
@@ -225,6 +584,8 @@ mod tests {
             Self {
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                init_calls: Arc::new(AtomicUsize::new(0)),
+                init_entered: None,
                 init_fails: false,
                 store: None,
             }
@@ -235,11 +596,16 @@ mod tests {
             self
         }
 
+        fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
+            self.init_entered = Some(init_entered);
+            self
+        }
+
         fn with_store(mut self) -> Self {
             let dir = tempdir().expect("tempdir should be created for queue store tests");
             let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
             store.open().expect("queue store should open");
-            self.store = Some(store);
+            self.store = Some(Arc::new(store));
             self
         }
     }
@@ -271,9 +637,7 @@ mod tests {
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            self.store
-                .as_ref()
-                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+            self.store.as_deref()
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -281,6 +645,11 @@ mod tests {
         }
 
         async fn init(&self) -> Result<(), TargetError> {
+            self.init_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(init_entered) = &self.init_entered {
+                init_entered.notify_one();
+                return std::future::pending().await;
+            }
             if self.init_fails {
                 return Err(TargetError::Configuration("forced init failure".to_string()));
             }
@@ -334,6 +703,212 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 1);
     }
 
+    #[tokio::test]
+    async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert_eq!(prepared.targets.len(), 1);
+        assert!(
+            prepared
+                .failure_summary()
+                .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
+        );
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.failure_summary().is_some());
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation.replay_workers.stop_all("stop degraded target replay worker").await;
+    }
+
+    #[tokio::test]
+    async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
+        let adapter = builtin_adapter();
+        let init_entered = Arc::new(Notify::new());
+        let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
+        let first_close_calls = first.close_calls.clone();
+        let second = TestTarget::new("second", "webhook");
+        let second_close_calls = second.close_calls.clone();
+        let second_init_calls = second.init_calls.clone();
+        let cancellation = CancellationToken::new();
+        let prepare_adapter = adapter.clone();
+        let prepare_cancellation = cancellation.clone();
+        let prepare = tokio::spawn(async move {
+            prepare_adapter
+                .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
+                .await
+        });
+
+        init_entered.notified().await;
+        cancellation.cancel();
+        let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
+            .await
+            .expect("cancellation should interrupt target initialization")
+            .expect("preparation task should finish");
+        assert_eq!(prepared.targets.len(), 2);
+
+        adapter
+            .close_prepared(prepared)
+            .await
+            .expect("cancelled targets should close");
+        assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn prepared_activation_opens_store_before_starting_replay() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let queue_path = dir.path().join("queue");
+        let mut target = TestTarget::new("primary", "webhook");
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.targets.is_empty());
+        assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation
+            .replay_workers
+            .stop_all("stop prepared activation test worker")
+            .await;
+    }
+
+    #[tokio::test]
+    async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let invalid_base = dir.path().join("not-a-directory");
+        std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
+        let mut target = TestTarget::new("primary", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
+        const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
+
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let gate = Arc::new(StoreOpenGate::default());
+        let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
+        let mut expected_ids = Vec::with_capacity(TARGETS);
+        for index in 0..TARGETS {
+            let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
+            expected_ids.push(target.id.to_string());
+            let open_gate = gate.clone();
+            target.store = Some(Arc::new(TestOpenStore {
+                before_clone: Arc::new(|| {}),
+                before_open: Arc::new(move || open_gate.enter()),
+                store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
+            }));
+            targets.push(Box::new(target));
+        }
+
+        let prepared = adapter.prepare_targets(targets).await;
+        let open_adapter = adapter.clone();
+        let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
+        let wait_gate = gate.clone();
+        let reached_bound =
+            tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
+                .await
+                .expect("store-open observer should not panic");
+        gate.release();
+        let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
+        let opened_ids = opened
+            .targets
+            .iter()
+            .map(|target| target.id().to_string())
+            .collect::<Vec<_>>();
+
+        assert!(reached_bound, "store opens did not use the configured parallelism");
+        assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
+        assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
+        assert!(rejected.targets.is_empty());
+        assert!(rejected.failure_summary().is_none());
+    }
+
+    #[tokio::test]
+    async fn panicking_store_open_rejects_and_closes_only_that_target() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| {}),
+            before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store should report a generic activation failure");
+        let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
+
+        assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
+        assert!(
+            activation.replay_workers.is_empty(),
+            "a rejected target must not publish without a replay worker"
+        );
+        assert!(activation_rejected.failure_summary().is_none());
+        assert!(summary.contains("queue store open failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected after a store panic should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking-clone", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
+            before_open: Arc::new(|| {}),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
+        assert!(open_rejected.failure_summary().is_none());
+        let (activation, rejected) = adapter.try_activate_prepared(opened);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store clone should report a generic activation failure");
+
+        assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
+        assert!(activation.replay_workers.is_empty());
+        assert!(summary.contains("replay activation failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected during replay activation should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[tokio::test]
     async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
         let adapter = builtin_adapter();

crates/targets/src/runtime/mod.rs

@@ -27,11 +27,21 @@ use crate::store::{Key, Store, ensure_store_entry_raw_readable};
 use crate::target::QueuedPayload;
 use crate::target::TargetDeliverySnapshot;
 use crate::{StoreError, TargetError};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::sync::Arc;
 use std::{collections::HashMap, fmt::Debug};
 use std::{future::Future, pin::Pin, time::Duration};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}
 
 /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
 /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
@@ -68,11 +78,21 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
 pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 
-/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
-/// replay worker to observe its cancel signal and exit before it is forcibly
-/// aborted. Workers observe cancellation promptly (including during retry
-/// backoff), so this only guards against a wedged task.
-const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
+pub(crate) enum PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    Ready(SharedTarget<E>),
+    Degraded {
+        error: TargetError,
+        target: SharedTarget<E>,
+    },
+    Failed {
+        error: TargetError,
+        target: Box<dyn Target<E> + Send + Sync>,
+    },
+    Cancelled(Box<dyn Target<E> + Send + Sync>),
+}
 
 /// Tracks a running replay worker: its cancel channel and, when the worker was
 /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
@@ -113,7 +133,7 @@ impl ReplayWorkerManager {
     }
 
     /// Registers a cancel channel together with the worker's join handle so
-    /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
+    /// `stop_all` can await the worker's exit.
     pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
         self.cancellers.insert(
             target_id,
@@ -140,37 +160,36 @@ impl ReplayWorkerManager {
     }
 
     /// Stops every replay worker: it first signals cancellation to all of them,
-    /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
-    /// which the task is aborted). Signalling before joining lets all workers
-    /// wind down concurrently, and joining guarantees no worker keeps draining
-    /// the shared store after this returns — preventing duplicate delivery and
-    /// orphaned tasks across reloads and shutdown.
+    /// then strictly awaits each worker's exit. A worker already awaiting a
+    /// delivery acknowledgement is allowed to finish; aborting it at an
+    /// arbitrary deadline could leave an acknowledged queue entry undeleted and
+    /// make the replacement worker deliver it again. Signalling before joining
+    /// lets all workers wind down concurrently. Legacy joinless registrations
+    /// can only be signalled.
     pub async fn stop_all(&mut self, log_prefix: &str) {
-        let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let mut joins = std::collections::VecDeque::new();
 
         // Phase 1: signal cancellation to all workers.
-        for (target_id, handle) in &handles {
+        for (target_id, handle) in handles {
             tracing::info!(target_id = %target_id, "{log_prefix}");
-            let _ = handle.cancel_tx.send(()).await;
+            let _ = handle.cancel_tx.try_send(());
+            if let Some(join) = handle.join {
+                joins.push_back((target_id, join));
+            } else {
+                tracing::warn!(
+                    target_id = %target_id,
+                    "Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
+                );
+            }
         }
 
-        // Phase 2: await each worker's exit, forcibly aborting any that overrun.
-        for (target_id, handle) in handles.drain(..) {
-            let Some(mut join) = handle.join else {
-                continue;
-            };
-            match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
-                Ok(Ok(())) => {}
-                Ok(Err(err)) => {
-                    tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
-                }
-                Err(_) => {
-                    join.abort();
-                    tracing::warn!(
-                        target_id = %target_id,
-                        "Timed out awaiting replay worker exit; task aborted"
-                    );
-                }
+        // Phase 2: strict join. Delivery operations own their own protocol
+        // deadlines; lifecycle must not invent a shorter deadline that turns an
+        // acknowledgement race into duplicate delivery.
+        while let Some((target_id, join)) = joins.pop_front() {
+            if let Err(err) = join.await {
+                tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
             }
         }
     }
@@ -184,6 +203,55 @@ where
     pub targets: Vec<SharedTarget<E>>,
 }
 
+/// Targets whose persistent queue stores are open and are ready to start
+/// replay. This distinct stage prevents activation from skipping store open.
+pub struct OpenedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+struct TargetActivationFailure {
+    detail: String,
+}
+
+/// Targets that have completed initialization but have not started replay
+/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
+/// the previous workers before the replacement workers are spawned.
+pub struct PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    failures: Vec<TargetActivationFailure>,
+    rejected_targets: Vec<SharedTarget<E>>,
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+impl<E> PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub fn failure_summary(&self) -> Option<String> {
+        if self.failures.is_empty() {
+            return None;
+        }
+
+        Some(
+            self.failures
+                .iter()
+                .map(|failure| failure.detail.clone())
+                .collect::<Vec<_>>()
+                .join("; "),
+        )
+    }
+
+    pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
+        self.failures
+            .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct RuntimeStatusSnapshot {
     pub replay_worker_count: usize,
@@ -339,17 +407,19 @@ where
     /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
     /// runtime.
     pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
-        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        let targets = std::mem::take(&mut self.targets);
+        let mut closes = FuturesUnordered::new();
+        for (target_id, target) in targets {
+            closes.push(async move { (target_id, target.close().await) });
+        }
+
         let mut errors = Vec::new();
-        for target_id in target_ids {
-            if let Some(target) = self.targets.remove(&target_id)
-                && let Err(err) = target.close().await
-            {
+        while let Some((target_id, result)) = closes.next().await {
+            if let Err(err) = result {
                 tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
                 errors.push((target_id, err));
             }
         }
-        self.targets.clear();
         errors
     }
 
@@ -440,21 +510,16 @@ where
         SharedTarget<E>,
     ) -> (mpsc::Sender<()>, JoinHandle<()>),
 {
-    let target_id = target.id().to_string();
-    let has_store = target.store().is_some();
-
-    if let Err(err) = target.init().await {
-        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
-        if !has_store {
+    let shared = match prepare_target(target, None).await {
+        PrepareTargetResult::Ready(target) => target,
+        PrepareTargetResult::Degraded { target, .. } => target,
+        PrepareTargetResult::Failed { target, .. } => {
+            let _ = target.close().await;
             return None;
         }
-        tracing::warn!(
-            target_id = %target_id,
-            "Proceeding with store-backed target despite init failure"
-        );
-    }
-
-    let shared: SharedTarget<E> = Arc::from(target);
+        PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
+    };
+    let target_id = shared.id().to_string();
     if !shared.is_enabled() {
         on_replay_start(&target_id, false);
         return Some((shared, None));
@@ -467,6 +532,45 @@ where
     Some((shared, cancel))
 }
 
+pub(crate) async fn prepare_target<E>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    cancellation: Option<&CancellationToken>,
+) -> PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    let init_result = match cancellation {
+        Some(cancellation) => {
+            tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
+                result = target.init() => result,
+            }
+        }
+        None => target.init().await,
+    };
+
+    if let Err(err) = init_result {
+        tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
+        if !has_store {
+            return PrepareTargetResult::Failed { error: err, target };
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+        return PrepareTargetResult::Degraded {
+            error: err,
+            target: Arc::from(target),
+        };
+    }
+
+    PrepareTargetResult::Ready(Arc::from(target))
+}
+
 type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);
 
 pub async fn activate_targets_with_replay<E, F, Fut>(
@@ -584,7 +688,11 @@ async fn stream_replay_worker<E>(
                 }
                 Ok(Ok(_)) => {}
                 Err(join_err) => {
-                    tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = join_failure_reason(&join_err),
+                        "The failed-events maintenance task failed to join"
+                    );
                 }
             }
             last_prune = tokio::time::Instant::now();
@@ -835,7 +943,8 @@ mod tests {
     use crate::{Target, TargetError};
     use async_trait::async_trait;
     use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use tokio::sync::{Notify, Semaphore};
 
     #[tokio::test(start_paused = true)]
     async fn seed_interval_start_backdates_by_one_interval() {
@@ -900,14 +1009,20 @@ mod tests {
     #[derive(Clone)]
     struct TestTarget {
         id: TargetID,
+        block_on_close: Arc<AtomicBool>,
+        close_gate: Arc<Semaphore>,
         close_calls: Arc<AtomicUsize>,
+        close_started: Arc<Notify>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
                 id: TargetID::new(id.to_string(), name.to_string()),
+                block_on_close: Arc::new(AtomicBool::new(false)),
+                close_gate: Arc::new(Semaphore::new(0)),
                 close_calls: Arc::new(AtomicUsize::new(0)),
+                close_started: Arc::new(Notify::new()),
             }
         }
     }
@@ -935,6 +1050,10 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            self.close_started.notify_one();
+            if self.block_on_close.load(Ordering::SeqCst) {
+                let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
+            }
             Ok(())
         }
 
@@ -966,6 +1085,45 @@ mod tests {
         assert_eq!(close_calls.load(Ordering::SeqCst), 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let first = TestTarget::new("first", "webhook");
+        let second = TestTarget::new("second", "webhook");
+        let first_observer = first.clone();
+        let second_observer = second.clone();
+        manager.add_boxed(Box::new(first));
+        manager.add_boxed(Box::new(second));
+
+        let first_close_key = manager
+            .keys()
+            .into_iter()
+            .next()
+            .expect("two targets should have a first close key");
+        let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
+            (first_observer, second_observer)
+        } else {
+            (second_observer, first_observer)
+        };
+        blocked.block_on_close.store(true, Ordering::SeqCst);
+
+        let close_task = tokio::spawn(async move { manager.clear_and_close().await });
+
+        tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
+            .await
+            .expect("the first target close should start");
+        tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
+            .await
+            .expect("a blocked first close must not prevent the second close from starting");
+        assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
+
+        blocked.close_gate.add_permits(1);
+        let errors = close_task.await.expect("clear_and_close task should join");
+        assert!(errors.is_empty());
+        assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[test]
     fn runtime_manager_snapshots_targets() {
         let mut manager = TargetRuntimeManager::<String>::new();
@@ -1030,6 +1188,51 @@ mod tests {
         assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
+        use super::ReplayWorkerManager;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use tokio::sync::Notify;
+
+        let mut manager = ReplayWorkerManager::new();
+        let acknowledgement = Arc::new(Notify::new());
+        let worker_started = Arc::new(Notify::new());
+        let exited = Arc::new(AtomicBool::new(false));
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
+        let worker_acknowledgement = acknowledgement.clone();
+        let worker_started_signal = worker_started.clone();
+        let worker_exited = exited.clone();
+        let join = tokio::spawn(async move {
+            worker_started_signal.notify_one();
+            let _ = cancel_rx.recv().await;
+            // Model a protocol operation that has accepted the request but has
+            // not returned its acknowledgement yet. Lifecycle must not abort
+            // this future or the same durable entry can be sent twice.
+            worker_acknowledgement.notified().await;
+            worker_exited.store(true, Ordering::SeqCst);
+        });
+        manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
+        worker_started.notified().await;
+
+        let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
+            _ = std::future::ready(()) => {}
+        }
+        tokio::time::advance(std::time::Duration::from_secs(60)).await;
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
+            _ = std::future::ready(()) => {}
+        }
+
+        acknowledgement.notify_one();
+        stop.await;
+        assert!(exited.load(Ordering::SeqCst));
+        assert!(manager.is_empty());
+    }
+
     mod classifier {
         use super::super::{ReplayEvent, stream_replay_worker};
         use crate::arn::TargetID;

crates/targets/src/target/kafka.rs

@@ -24,7 +24,7 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error,
+        TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error,
         open_target_queue_store, persist_queued_payload_to_store,
     },
 };
@@ -32,13 +32,84 @@ use async_trait::async_trait;
 use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode};
 use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig};
 use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key};
-use std::{fmt, marker::PhantomData, sync::Arc, time::Duration};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration};
 use tokio::sync::Mutex;
 use tracing::{debug, error, info, instrument, warn};
 
 pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256";
 pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512";
+const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct KafkaDeliveryAttempt<'a> {
+    armed: bool,
+    poisoned: &'a AtomicBool,
+}
+
+impl KafkaDeliveryAttempt<'_> {
+    fn disarm(&mut self) {
+        self.armed = false;
+    }
+}
+
+impl Drop for KafkaDeliveryAttempt<'_> {
+    fn drop(&mut self) {
+        if self.armed {
+            self.poisoned.store(true, Ordering::Release);
+        }
+    }
+}
+
+fn kafka_delivery_timeout() -> TargetError {
+    TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}"))
+}
+
+async fn with_serialized_kafka_delivery<P, T, Select, SelectFuture, Deliver, DeliveryFuture, Invalidate, InvalidateFuture>(
+    delivery_lock: &Mutex<()>,
+    delivery_poisoned: &AtomicBool,
+    select_producer: Select,
+    deliver: Deliver,
+    invalidate: Invalidate,
+) -> Result<T, TargetError>
+where
+    P: Send,
+    T: Send,
+    Select: FnOnce() -> SelectFuture + Send,
+    SelectFuture: Future<Output = Result<P, TargetError>> + Send,
+    Deliver: FnOnce(P) -> DeliveryFuture + Send,
+    DeliveryFuture: Future<Output = Result<T, TargetError>> + Send,
+    Invalidate: Fn() -> InvalidateFuture + Send,
+    InvalidateFuture: Future<Output = ()> + Send,
+{
+    let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT;
+    let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock())
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    let mut attempt = KafkaDeliveryAttempt {
+        armed: true,
+        poisoned: delivery_poisoned,
+    };
+
+    if delivery_poisoned.load(Ordering::Acquire) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+
+    let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await })
+        .await
+        .map_err(|_| kafka_delivery_timeout())?;
+    if result.as_ref().is_err_and(is_connectivity_error) {
+        tokio::time::timeout_at(deadline, invalidate())
+            .await
+            .map_err(|_| kafka_delivery_timeout())?;
+        delivery_poisoned.store(false, Ordering::Release);
+    }
+    attempt.disarm();
+    result
+}
 
 /// Arguments for configuring a Kafka target
 #[derive(Clone)]
@@ -233,6 +304,8 @@ where
     args: KafkaArgs,
     store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
     producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_lock: Arc<Mutex<()>>,
+    delivery_poisoned: Arc<AtomicBool>,
     tls_state: Arc<Mutex<TargetTlsState>>,
     /// Adapter that bridges this target to the TLS reload coordinator.
     /// When `Some`, the target uses coordinator-managed material; when `None`,
@@ -291,6 +364,8 @@ where
             args,
             store: queue_store,
             producer: Arc::new(Mutex::new(None)),
+            delivery_lock: Arc::new(Mutex::new(())),
+            delivery_poisoned: Arc::new(AtomicBool::new(false)),
             tls_state: Arc::new(Mutex::new(TargetTlsState::default())),
             tls_adapter: None,
             delivery_counters: Arc::new(TargetDeliveryCounters::default()),
@@ -307,7 +382,7 @@ where
         };
 
         let mut config = AsyncProducerConfig::new()
-            .with_ack_timeout(Duration::from_secs(30))
+            .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT)
             .with_required_acks(acks);
 
         if let Some(security) = self.args.security_config(true)? {
@@ -388,20 +463,26 @@ where
             "Sending Kafka payload"
         );
 
-        let producer = self.get_or_build_producer().await?;
-
-        // Use "<bucket>/<object>" as the message key so all events for the same
-        // object hash to the same partition and preserve per-object ordering
-        // across multiple partitions (backlog#983).
-        let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
-        if let Err(err) = producer
-            .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
-            .await
-        {
-            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
-            invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await;
-            return Err(mapped);
-        }
+        // rustfs-kafka-async does not validate response correlation IDs. Keep
+        // producer selection, send, and timeout invalidation serialized so a
+        // waiter cannot reuse a connection with an unread timed-out response.
+        with_serialized_kafka_delivery(
+            &self.delivery_lock,
+            &self.delivery_poisoned,
+            || self.get_or_build_producer(),
+            |producer| async move {
+                // Use "<bucket>/<object>" as the message key so all events for the same
+                // object hash to the same partition and preserve per-object ordering
+                // across multiple partitions (backlog#983).
+                let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name);
+                producer
+                    .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice()))
+                    .await
+                    .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka"))
+            },
+            || self.invalidate_cached_producer(),
+        )
+        .await?;
 
         debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
         self.delivery_counters.record_success();
@@ -415,6 +496,8 @@ where
             args: self.args.clone(),
             store: self.store.as_ref().map(|s| s.boxed_clone()),
             producer: Arc::clone(&self.producer),
+            delivery_lock: Arc::clone(&self.delivery_lock),
+            delivery_poisoned: Arc::clone(&self.delivery_poisoned),
             tls_state: Arc::clone(&self.tls_state),
             tls_adapter: self.tls_adapter.clone(),
             delivery_counters: Arc::clone(&self.delivery_counters),
@@ -559,6 +642,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::sync::atomic::AtomicUsize;
+    use tokio::sync::Notify;
 
     fn base_args() -> KafkaArgs {
         KafkaArgs {
@@ -580,6 +665,158 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    move |selected| async move {
+                        assert_eq!(selected, 1);
+                        first_entered.notify_one();
+                        std::future::pending::<Result<usize, TargetError>>().await
+                    },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        first_entered.notified().await;
+        tokio::time::advance(Duration::from_secs(1)).await;
+        let second = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let generation = Arc::clone(&generation);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    {
+                        let generation = Arc::clone(&generation);
+                        move || async move { Ok(generation.load(Ordering::SeqCst)) }
+                    },
+                    |selected| async move { Ok(selected) },
+                    move || {
+                        let generation = Arc::clone(&generation);
+                        async move { generation.store(2, Ordering::SeqCst) }
+                    },
+                )
+                .await
+            })
+        };
+
+        assert!(matches!(
+            first.await.expect("first delivery task should not panic"),
+            Err(TargetError::Timeout(_))
+        ));
+        assert_eq!(
+            second
+                .await
+                .expect("second delivery task should not panic")
+                .expect("second delivery should succeed"),
+            2,
+            "the waiter must select a fresh producer generation after timeout invalidation"
+        );
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_includes_waiting_for_the_serialization_lock() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = AtomicBool::new(false);
+        let selected = Arc::new(AtomicBool::new(false));
+        let _held = delivery_lock.lock().await;
+
+        let error = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let selected = Arc::clone(&selected);
+                move || async move {
+                    selected.store(true, Ordering::SeqCst);
+                    Ok(())
+                }
+            },
+            |()| async { Ok(()) },
+            || async {},
+        )
+        .await
+        .expect_err("lock admission must share the absolute delivery deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer");
+        assert!(!delivery_poisoned.load(Ordering::SeqCst));
+    }
+
+    #[tokio::test]
+    async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() {
+        let delivery_lock = Arc::new(Mutex::new(()));
+        let delivery_poisoned = Arc::new(AtomicBool::new(false));
+        let generation = Arc::new(AtomicUsize::new(1));
+        let first_entered = Arc::new(Notify::new());
+        let first = {
+            let delivery_lock = Arc::clone(&delivery_lock);
+            let delivery_poisoned = Arc::clone(&delivery_poisoned);
+            let first_entered = Arc::clone(&first_entered);
+            tokio::spawn(async move {
+                with_serialized_kafka_delivery(
+                    &delivery_lock,
+                    &delivery_poisoned,
+                    || async { Ok(1usize) },
+                    move |_| async move {
+                        first_entered.notify_one();
+                        std::future::pending::<Result<(), TargetError>>().await
+                    },
+                    || async {},
+                )
+                .await
+            })
+        };
+        first_entered.notified().await;
+        first.abort();
+        assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled());
+        assert!(delivery_poisoned.load(Ordering::Acquire));
+
+        let selected = with_serialized_kafka_delivery(
+            &delivery_lock,
+            &delivery_poisoned,
+            {
+                let generation = Arc::clone(&generation);
+                move || async move { Ok(generation.load(Ordering::SeqCst)) }
+            },
+            |selected| async move { Ok(selected) },
+            {
+                let generation = Arc::clone(&generation);
+                move || {
+                    let generation = Arc::clone(&generation);
+                    async move { generation.store(2, Ordering::SeqCst) }
+                }
+            },
+        )
+        .await
+        .expect("the next delivery should recover from cancellation poisoning");
+
+        assert_eq!(selected, 2);
+        assert!(!delivery_poisoned.load(Ordering::Acquire));
+    }
+
     #[test]
     fn test_validate_empty_brokers() {
         let args = KafkaArgs {

crates/targets/src/target/mod.rs

@@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog};
 use async_trait::async_trait;
 use rustfs_s3_types::EventName;
 use serde::{Deserialize, Serialize};
+use std::cell::Cell;
 use std::fmt::Formatter;
 use std::future::Future;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::thread_local;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tracing::{debug, warn};
 
 pub mod amqp;
@@ -111,7 +113,12 @@ where
     /// Checks if the target is active and reachable
     async fn is_active(&self) -> Result<bool, TargetError>;
 
-    /// Saves an event (either sends it immediately or stores it for later)
+    /// Saves an event (either sends it immediately or stores it for later).
+    ///
+    /// A target whose [`Self::store`] returns `Some` must only persist the event
+    /// here; network delivery belongs to its replay worker. Runtime lifecycle
+    /// handoff drains these durable enqueues while allowing a direct network
+    /// send to finish against a detached target.
     async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError>;
 
     /// Sends an event from the store using the queued raw body and metadata.
@@ -605,6 +612,24 @@ pub(crate) fn open_target_queue_store(
     Ok(store.map(|store| Box::new(store) as BoxedQueuedStore))
 }
 
+thread_local! {
+    static DEFER_QUEUE_STORE_OPEN: Cell<bool> = const { Cell::new(false) };
+}
+
+pub(crate) fn with_deferred_queue_store_open<T>(operation: impl FnOnce() -> T) -> T {
+    struct Reset(bool);
+
+    impl Drop for Reset {
+        fn drop(&mut self) {
+            DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0));
+        }
+    }
+
+    let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true));
+    let _reset = Reset(previous);
+    operation()
+}
+
 /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed
 /// failed-store capability holds it directly rather than through the type-erased Store handle.
 pub(crate) fn open_target_queue_store_typed(
@@ -625,9 +650,11 @@ pub(crate) fn open_target_queue_store_typed(
         TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
     };
     let store = QueueStore::<QueuedPayload>::new(queue_dir, queue_limit, extension);
-    store
-        .open()
-        .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) {
+        store
+            .open()
+            .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?;
+    }
 
     Ok(Some(store))
 }
@@ -649,6 +676,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool {
     matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
 }
 
+/// Applies an absolute deadline to one protocol delivery attempt.
+///
+/// Target clients expose different timeout controls, and several of them only
+/// place a timeout value in the wire request without bounding the local socket
+/// future. Keeping the outer deadline here gives every caller the same typed,
+/// retryable timeout without changing the target-specific error mapping.
+pub(crate) async fn with_delivery_deadline<T, F>(
+    deadline: Duration,
+    operation: &'static str,
+    delivery: F,
+) -> Result<T, TargetError>
+where
+    F: Future<Output = Result<T, TargetError>>,
+{
+    match tokio::time::timeout(deadline, delivery).await {
+        Ok(result) => result,
+        Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))),
+    }
+}
+
 pub(crate) async fn invalidate_cache_on_connectivity_error<F, Fut>(err: &TargetError, invalidate: F)
 where
     F: FnOnce() -> Fut,
@@ -1137,6 +1184,29 @@ mod tests {
         let _ = fs::remove_file(base);
     }
 
+    #[test]
+    fn deferred_queue_store_creation_does_not_touch_the_filesystem() {
+        let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4()));
+        fs::write(&base, b"not-a-directory").expect("failed to create file base");
+        let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string());
+
+        let store = with_deferred_queue_store_open(|| {
+            open_target_queue_store(
+                base.to_str().unwrap(),
+                100,
+                TargetType::NotifyEvent,
+                ChannelTargetType::Kafka.as_str(),
+                &target_id,
+                "deferred open",
+            )
+        })
+        .expect("deferred construction must not open the queue directory")
+        .expect("non-empty queue directory should create a dormant store");
+
+        assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it");
+        let _ = fs::remove_file(base);
+    }
+
     #[test]
     fn persist_queued_payload_to_store_writes_encoded_payload() {
         let store = MockQueuedStore::new(false);
@@ -1182,6 +1252,19 @@ mod tests {
         assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string())));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() {
+        let error = with_delivery_deadline(
+            Duration::from_secs(30),
+            "test delivery",
+            std::future::pending::<Result<(), TargetError>>(),
+        )
+        .await
+        .expect_err("a stalled delivery must hit its hard deadline");
+
+        assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s"));
+    }
+
     #[tokio::test]
     async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() {
         let marker = Arc::new(AtomicBool::new(false));

crates/targets/src/target/mqtt.rs

@@ -765,11 +765,6 @@ where
 
     #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
     async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
-        let client_guard = self.client.lock().await;
-        let client = client_guard
-            .as_ref()
-            .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
-
         debug!(
             event = EVENT_MQTT_DELIVERY_STATE,
             component = LOG_COMPONENT_TARGETS,
@@ -790,10 +785,22 @@ where
         // silently dropped the event while its durable copy was already deleted
         // (backlog#971). Error classification now matches on the typed error
         // instead of substring matching on the display string.
-        let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await {
-            Ok(notice) => notice,
-            Err(e) => {
-                let err = classify_mqtt_client_error(&e);
+        let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async {
+            let client_guard = self.client.lock().await;
+            let client = client_guard
+                .as_ref()
+                .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?;
+            let notice = client
+                .publish_tracked(&self.args.topic, self.args.qos, false, body)
+                .await
+                .map_err(|error| classify_mqtt_client_error(&error))?;
+            drop(client_guard);
+            Ok(notice)
+        })
+        .await
+        {
+            Ok(Ok(notice)) => notice,
+            Ok(Err(err)) => {
                 warn!(
                     event = EVENT_MQTT_DELIVERY_STATE,
                     component = LOG_COMPONENT_TARGETS,
@@ -801,18 +808,29 @@ where
                     target_id = %self.id,
                     state = "publish_failed",
                     reason = "enqueue_error",
-                    error = %e,
+                    error = %err,
                     "mqtt delivery state"
                 );
                 mark_target_disconnected_on_connectivity_error(&self.connected, &err);
                 return Err(err);
             }
+            Err(_) => {
+                warn!(
+                    event = EVENT_MQTT_DELIVERY_STATE,
+                    component = LOG_COMPONENT_TARGETS,
+                    subsystem = LOG_SUBSYSTEM_MQTT,
+                    target_id = %self.id,
+                    state = "publish_failed",
+                    reason = "enqueue_timeout",
+                    "mqtt delivery state"
+                );
+                // Admission can time out because the local bounded request
+                // channel is full while the MQTT session remains connected.
+                // Only protocol/client failures are evidence of disconnect.
+                return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string()));
+            }
         };
 
-        // Release the client lock before awaiting the broker acknowledgement so a
-        // slow/hung broker never blocks other senders from queueing publishes.
-        drop(client_guard);
-
         match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await {
             Ok(Ok(())) => {
                 debug!(
@@ -1708,9 +1726,9 @@ where
 #[cfg(test)]
 mod tests {
     use super::{
-        ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS,
-        classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor,
-        validate_mqtt_broker_url,
+        AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig,
+        MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error,
+        next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url,
     };
     use crate::error::TargetError;
     use crate::target::{REDACTED_SECRET, TargetType};
@@ -1720,6 +1738,23 @@ mod tests {
     use tokio::sync::mpsc;
     use url::Url;
 
+    fn base_mqtt_args() -> MQTTArgs {
+        MQTTArgs {
+            enable: true,
+            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
+            topic: "rustfs/events".to_string(),
+            qos: QoS::AtLeastOnce,
+            username: String::new(),
+            password: String::new(),
+            tls: MQTTTlsConfig::default(),
+            max_reconnect_interval: Duration::from_secs(1),
+            keep_alive: Duration::from_secs(30),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
     #[test]
     fn mqtt_client_error_classified_as_not_connected() {
         // A publish that cannot be handed to the event loop means the client is
@@ -1752,6 +1787,38 @@ mod tests {
         assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_)));
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn enqueue_timeout_keeps_a_live_session_connected() {
+        let target = MQTTTarget::<String>::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build");
+        let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883)))
+            .capacity(1)
+            .build();
+        client
+            .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice())
+            .await
+            .expect("first publish should fill the local channel");
+        *target.client.lock().await = Some(client);
+        target.connected.store(true, Ordering::SeqCst);
+        let meta = QueuedPayloadMeta::new(
+            rustfs_s3_types::EventName::ObjectCreatedPut,
+            "bucket".to_string(),
+            "object".to_string(),
+            "application/json",
+            2,
+        );
+
+        let error = target
+            .send_body(b"{}".to_vec(), &meta)
+            .await
+            .expect_err("a full local request channel should hit the enqueue deadline");
+
+        assert!(matches!(error, TargetError::Timeout(_)));
+        assert!(
+            target.connected.load(Ordering::SeqCst),
+            "local admission pressure is not evidence that the MQTT session disconnected"
+        );
+    }
+
     #[test]
     fn next_reconnect_backoff_doubles_until_capped() {
         let mut backoff = MQTT_RECONNECT_BACKOFF_MIN;
@@ -1864,21 +1931,13 @@ mod tests {
     #[test]
     fn debug_redacts_mqtt_secret_fields() {
         let args = MQTTArgs {
-            enable: true,
-            broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"),
-            topic: "rustfs/events".to_string(),
-            qos: QoS::AtLeastOnce,
             username: "mqtt-user".to_string(),
             password: "mqtt-password".to_string(),
             tls: MQTTTlsConfig {
                 client_key_path: "/etc/rustfs/mqtt.key".to_string(),
                 ..MQTTTlsConfig::default()
             },
-            max_reconnect_interval: Duration::from_secs(1),
-            keep_alive: Duration::from_secs(30),
-            queue_dir: String::new(),
-            queue_limit: 0,
-            target_type: TargetType::NotifyEvent,
+            ..base_mqtt_args()
         };
 
         let rendered = format!("{args:?}");

crates/targets/src/target/mysql.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret,
+        persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -47,6 +47,8 @@ use uuid::Uuid;
 /// `TargetError::Timeout`, a connectivity error, so the payload stays queued
 /// for replay.
 const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15);
+/// Absolute ceiling for one INSERT, including pool checkout and server execution.
+const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Name of the optional idempotency-key column / primary key. Present on tables
 /// created by this target; absent on legacy two-column tables.
@@ -784,29 +786,33 @@ where
             "Inserting MySQL event"
         );
 
-        let pool = self.get_or_init_pool().await?;
-        // At this point the pool has already been initialized (get_or_init_pool
-        // succeeded above), so get_conn() failures are always transient: the
-        // connection was lost or the pool is temporarily exhausted.
-        let mut conn = checkout_conn(&pool).await?;
-
         let event_time = extract_event_time(body)?;
         let event_data =
             std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?;
 
         let quoted_table = quote_table_name(&self.args.table)?;
+        with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async {
+            let pool = self.get_or_init_pool().await?;
+            // At this point the pool has already been initialized (get_or_init_pool
+            // succeeded above), so get_conn() failures are always transient: the
+            // connection was lost or the pool is temporarily exhausted.
+            let mut conn = checkout_conn(&pool).await?;
 
-        if self.idempotency_supported.load(Ordering::Relaxed) {
-            let sql = mysql_insert_sql_with_event_id(&quoted_table);
-            conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        } else {
-            let sql = mysql_insert_sql_legacy(&quoted_table);
-            conn.exec_drop(sql, (event_time.as_str(), event_data))
-                .await
-                .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
-        }
+            if self.idempotency_supported.load(Ordering::Relaxed) {
+                let sql = mysql_insert_sql_with_event_id(&quoted_table);
+                conn.exec_drop(sql, (event_id, event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            } else {
+                let sql = mysql_insert_sql_legacy(&quoted_table);
+                conn.exec_drop(sql, (event_time.as_str(), event_data))
+                    .await
+                    .map_err(|err| map_mysql_error(err, "Failed to insert event"))?;
+            }
+
+            Ok(())
+        })
+        .await?;
 
         self.delivery_counters.record_success();
         debug!(target_id = %self.id, "MySQL event inserted");

crates/targets/src/target/nats/mod.rs

@@ -25,7 +25,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
-        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret,
+        open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error};
 pub(crate) use jetstream::resolve_dedup_id;
 pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream};
 
+const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+
 #[derive(Clone)]
 pub struct NATSArgs {
     pub enable: bool,
@@ -397,9 +399,21 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        let client = self.get_or_connect().await?;
-        if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await {
-            let err = classify_nats_publish_error(&e);
+        let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async {
+            let client = self.get_or_connect().await?;
+            client
+                .publish(self.args.subject.clone(), body.into())
+                .await
+                .map_err(|err| classify_nats_publish_error(&err))?;
+
+            // publish only enqueues the message on the client's outbound channel. Flush to confirm the
+            // message reached the server before delivery is treated as successful (backlog#971).
+            client.flush().await.map_err(|err| classify_nats_flush_error(&err))?;
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = result {
             if is_connectivity_error(&err) {
                 self.invalidate_cached_client_connection().await;
                 self.connected.store(false, Ordering::SeqCst);
@@ -407,15 +421,6 @@ where
             return Err(err);
         }
 
-        // publish only enqueues the message on the client's outbound channel. Flush to confirm the
-        // message reached the server before delivery is treated as successful (backlog#971).
-        if let Err(e) = client.flush().await {
-            let err = classify_nats_flush_error(&e);
-            self.invalidate_cached_client_connection().await;
-            self.connected.store(false, Ordering::SeqCst);
-            return Err(err);
-        }
-
         self.delivery_counters.record_success();
         Ok(())
     }

crates/targets/src/target/postgres.rs

@@ -38,7 +38,7 @@ use crate::{
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret,
-        redacted_secret,
+        redacted_secret, with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
 /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio
 /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts.
 const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20);
+/// Absolute ceiling for one SQL delivery, including pool checkout and execution.
+const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
 
 /// Returns `true` for any `s3:ObjectRemoved:*` event.
 ///
@@ -730,30 +732,29 @@ where
 
         let key = resolve_payload_key(&payload, meta);
 
-        let result = match self.args.format {
-            // For the single-row `namespace` format, an object removal must
-            // delete the row rather than UPSERT it, otherwise stale state
-            // lingers in the table after the object is gone.
-            PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
-                client.execute(&self.namespace_delete_sql, &[&key]).await
+        with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async {
+            match self.args.format {
+                // For the single-row `namespace` format, an object removal must
+                // delete the row rather than UPSERT it, otherwise stale state
+                // lingers in the table after the object is gone.
+                PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => {
+                    client.execute(&self.namespace_delete_sql, &[&key]).await
+                }
+                PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
+                PostgresFormat::Access => {
+                    let event_name_str = meta.event_name.to_string();
+                    let queued_at_ms = meta.queued_at_unix_ms as i64;
+                    client
+                        .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
+                        .await
+                }
             }
-            PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await,
-            PostgresFormat::Access => {
-                let event_name_str = meta.event_name.to_string();
-                let queued_at_ms = meta.queued_at_unix_ms as i64;
-                client
-                    .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms])
-                    .await
-            }
-        };
+            .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed"))
+        })
+        .await?;
 
-        match result {
-            Ok(_) => {
-                self.delivery_counters.record_success();
-                Ok(())
-            }
-            Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")),
-        }
+        self.delivery_counters.record_success();
+        Ok(())
     }
 
     /// Probes the table from `init()`. Failure is non-fatal when a queue is

crates/targets/src/target/pulsar.rs

@@ -24,8 +24,9 @@ use crate::{
     store::{Key, Store},
     target::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
-        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store,
-        persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error,
+        open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -39,11 +40,15 @@ use std::fmt;
 use std::path::Path;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::sync::Mutex as AsyncMutex;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use url::Url;
 use uuid::Uuid;
 
+const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30);
+const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1);
+
 #[derive(Clone)]
 pub struct PulsarArgs {
     pub enable: bool,
@@ -276,6 +281,23 @@ where
         self.tls_state.lock().reset();
     }
 
+    async fn clear_failed_delivery_state(&self) {
+        match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await {
+            Ok(mut producer) => {
+                producer.take();
+            }
+            Err(_) => {
+                warn!(
+                    target_id = %self.id,
+                    reason = "producer_cleanup_lock_timeout",
+                    "Timed out clearing the Pulsar producer after a failed delivery"
+                );
+            }
+        }
+        self.clear_cached_client();
+        self.connected.store(false, Ordering::SeqCst);
+    }
+
     async fn get_or_connect_client(&self) -> Result<Pulsar<TokioExecutor>, TargetError> {
         // When a TLS reload adapter is attached, it drives client rebuilds
         // in the background. The inline per-send fingerprint check is skipped.
@@ -334,20 +356,30 @@ where
     }
 
     async fn send_body(&self, body: Vec<u8>) -> Result<(), TargetError> {
-        self.init_producer().await?;
-        let mut guard = self.producer.lock().await;
-        let producer = guard
-            .as_mut()
-            .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
-        let receipt = producer
-            .send_non_blocking(body)
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
-        receipt
-            .await
-            .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
-        self.delivery_counters.record_success();
-        Ok(())
+        let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async {
+            self.init_producer().await?;
+            let mut guard = self.producer.lock().await;
+            let producer = guard
+                .as_mut()
+                .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?;
+            let receipt = producer
+                .send_non_blocking(body)
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?;
+            receipt
+                .await
+                .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?;
+            self.delivery_counters.record_success();
+            Ok(())
+        })
+        .await;
+
+        if let Err(err) = &result
+            && is_connectivity_error(err)
+        {
+            self.clear_failed_delivery_state().await;
+        }
+        result
     }
 }
 
@@ -525,6 +557,22 @@ mod tests {
         }
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() {
+        let target = Arc::new(PulsarTarget::<String>::new("pulsar:test".to_string(), base_args()).expect("target should build"));
+        target.connected.store(true, Ordering::SeqCst);
+        let producer_guard = target.producer.lock().await;
+        let cleanup = {
+            let target = Arc::clone(&target);
+            tokio::spawn(async move { target.clear_failed_delivery_state().await })
+        };
+
+        cleanup.await.expect("cleanup task should not panic");
+
+        assert!(!target.connected.load(Ordering::SeqCst));
+        drop(producer_guard);
+    }
+
     #[test]
     fn debug_redacts_pulsar_secret_fields() {
         let args = PulsarArgs {

crates/targets/src/target/redis.rs

@@ -26,6 +26,7 @@ use crate::{
         ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
         TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error,
         mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store,
+        with_delivery_deadline,
     },
 };
 use async_trait::async_trait;
@@ -48,6 +49,19 @@ use tokio::sync::Mutex;
 use tracing::{debug, info, instrument, warn};
 use url::Url;
 
+const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5);
+
+fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration {
+    let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX);
+    let per_attempt = args
+        .connection_timeout
+        .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT)
+        .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT))
+        .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2)));
+    per_attempt.saturating_mul(attempts)
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RedisTlsPolicy {
     SystemCa,
@@ -214,6 +228,16 @@ impl RedisArgs {
             ));
         }
 
+        if self.connection_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration(
+                "Redis connection_timeout must be greater than zero".to_string(),
+            ));
+        }
+
+        if self.response_timeout == Some(Duration::ZERO) {
+            return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string()));
+        }
+
         if self.pipeline_buffer_size == Some(0) {
             return Err(TargetError::Configuration(
                 "Redis pipeline_buffer_size must be greater than zero".to_string(),
@@ -464,71 +488,97 @@ where
             "Sending Redis payload"
         );
 
-        let mut attempt = 0usize;
-        let mut last_error = None;
-        while attempt < self.args.max_retry_attempts {
-            attempt += 1;
+        let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async {
+            let mut attempt = 0usize;
+            let mut last_error = None;
+            while attempt < self.args.max_retry_attempts {
+                attempt += 1;
 
-            let mut publisher = self.get_or_create_publisher().await?;
-            match publisher
-                .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT);
+                let mut publisher = match with_delivery_deadline(
+                    connection_timeout,
+                    "Redis connection",
+                    self.get_or_create_publisher(),
+                )
                 .await
-            {
-                Ok(receiver_count) => {
-                    // PUBLISH returns the number of subscribers that received the
-                    // message. Redis pub/sub is best-effort: with zero subscribers
-                    // the event is delivered to no one, yet the durable copy is
-                    // deleted. Warn so operators relying on reliable delivery are
-                    // not silently losing events (backlog#982).
-                    if receiver_count == 0 {
+                {
+                    Ok(publisher) => publisher,
+                    Err(err) => {
+                        invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await;
+                        return Err(err);
+                    }
+                };
+                let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT);
+                match with_delivery_deadline(response_timeout, "Redis publish response", async {
+                    publisher
+                        .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice())
+                        .await
+                        .map_err(map_redis_error)
+                })
+                .await
+                {
+                    Ok(receiver_count) => {
+                        // PUBLISH returns the number of subscribers that received the
+                        // message. Redis pub/sub is best-effort: with zero subscribers
+                        // the event is delivered to no one, yet the durable copy is
+                        // deleted. Warn so operators relying on reliable delivery are
+                        // not silently losing events (backlog#982).
+                        if receiver_count == 0 {
+                            warn!(
+                                target_id = %self.id,
+                                channel = %self.args.channel,
+                                "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            );
+                        }
+                        debug!(
+                            target_id = %self.id,
+                            channel = %self.args.channel,
+                            attempt,
+                            receiver_count,
+                            "Event published to Redis channel"
+                        );
+                        self.delivery_counters.record_success();
+                        return Ok(());
+                    }
+                    Err(mapped) => {
+                        invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
+
                         warn!(
                             target_id = %self.id,
                             channel = %self.args.channel,
-                            "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)"
+                            attempt,
+                            max_attempts = self.args.max_retry_attempts,
+                            error = %mapped,
+                            "Redis publish attempt failed"
                         );
-                    }
-                    debug!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        receiver_count,
-                        "Event published to Redis channel"
-                    );
-                    self.delivery_counters.record_success();
-                    return Ok(());
-                }
-                Err(err) => {
-                    let mapped = map_redis_error(err);
-                    invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await;
 
-                    warn!(
-                        target_id = %self.id,
-                        channel = %self.args.channel,
-                        attempt,
-                        max_attempts = self.args.max_retry_attempts,
-                        error = %mapped,
-                        "Redis publish attempt failed"
-                    );
+                        if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
+                            last_error = Some(mapped);
+                            break;
+                        }
 
-                    if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts {
                         last_error = Some(mapped);
-                        break;
+                        tokio::time::sleep(compute_retry_delay(
+                            attempt,
+                            self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
+                            self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
+                        ))
+                        .await;
                     }
-
-                    last_error = Some(mapped);
-                    tokio::time::sleep(compute_retry_delay(
-                        attempt,
-                        self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)),
-                        self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)),
-                    ))
-                    .await;
                 }
             }
+
+            Err(last_error.unwrap_or(TargetError::Unknown(
+                "Redis publish failed without a captured error".to_string(),
+            )))
+        })
+        .await;
+
+        if let Err(err) = &result {
+            invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await;
+            self.connected.store(false, Ordering::SeqCst);
         }
-
-        self.connected.store(false, Ordering::SeqCst);
-
-        Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string())))
+        result
     }
 }
 
@@ -551,7 +601,7 @@ where
         // thus a fresh TCP+TLS handshake — on every health check (backlog#982).
         // ensure_publisher_ready already invalidates the cached manager on a
         // connectivity error so the next attempt rebuilds it.
-        match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await {
+        match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await {
             Ok(Ok(())) => {
                 self.connected.store(true, Ordering::SeqCst);
                 Ok(true)
@@ -917,6 +967,26 @@ mod tests {
         assert!(args.validate().is_err());
     }
 
+    #[test]
+    fn validate_rejects_zero_connection_timeout() {
+        let args = RedisArgs {
+            connection_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn validate_rejects_zero_response_timeout() {
+        let args = RedisArgs {
+            response_timeout: Some(Duration::ZERO),
+            ..base_args()
+        };
+
+        assert!(args.validate().is_err());
+    }
+
     #[test]
     fn validate_accepts_custom_ca_tls_policy() {
         let args = RedisArgs {
@@ -1237,6 +1307,23 @@ mod tests {
         assert_eq!(target.delivery_snapshot().total_messages, 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() {
+        let mut args = base_args();
+        args.max_retry_attempts = 1;
+        args.response_timeout = Some(Duration::from_secs(90));
+        let manager_config = build_redis_connection_manager_config(&args);
+
+        assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90)));
+        assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97));
+        with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async {
+            tokio::time::sleep(Duration::from_secs(70)).await;
+            Ok::<_, TargetError>(())
+        })
+        .await
+        .expect("the configured delivery budget must not impose a fixed sixty-second cap");
+    }
+
     #[tokio::test]
     async fn send_body_sets_connected_false_after_retry_exhaustion() {
         let listener = match TcpListener::bind("127.0.0.1:0").await {

crates/targets/src/target/webhook.rs

@@ -952,7 +952,7 @@ mod tests {
             .expect("https webhook probe should trust configured ca");
 
         assert_eq!(resp.status(), reqwest::StatusCode::OK);
-        assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), "");
+        assert!(resp.bytes().await.expect("read response body").is_empty());
         handle.join().expect("tls server thread");
     }
 }

docs/testing/e2e-suite-inventory.md

@@ -19,19 +19,24 @@
 |---|---|---|
 | admin_auth_test | 3 | ✅ |
 | admin_iam_crud_test | 2 | ✅ |
+| admin_pools_test | 1 | ✅ |
 | admin_timeout_regression_test | 1 |  |
 | anonymous_access_test | 3 | ✅ |
+| api_rate_limit_test | 3 |  |
 | archive_download_integrity_test | 13 |  |
 | bucket_logging_test | 3 |  |
 | bucket_policy_check_test | 1 | ✅ |
 | checksum_upload_test | 7 |  |
 | cluster_concurrency_test | 2 |  |
-| common | 3 |  |
+| cluster_multidrive_pool_test | 2 |  |
+| common | 10 |  |
 | compression_test | 1 |  |
+| connection_cap_test | 2 |  |
 | console_smoke_test | 1 | ✅ |
 | content_encoding_test | 3 | ✅ |
+| copy_object_checksum_test | 3 |  |
 | copy_object_metadata_test | 1 | ✅ |
-| copy_object_version_restore_test | 1 |  |
+| copy_object_version_restore_test | 2 |  |
 | copy_source_invalid_date_test | 1 | ✅ |
 | create_bucket_region_test | 2 | ✅ |
 | degraded_read_eof_regression_test | 3 |  |
@@ -40,6 +45,7 @@
 | delete_objects_versioning_test | 2 | ✅ |
 | existing_object_tag_policy_test | 4 |  |
 | fake_s3_target | 4 | ✅ |
+| fault_proxy | 7 |  |
 | get_codec_streaming_compat_test | 1 |  |
 | head_object_consistency_test | 1 | ✅ |
 | head_object_range_test | 1 | ✅ |
@@ -56,13 +62,13 @@
 | multipart_auth_test | 109 |  |
 | namespace_lock_quorum_test | 2 |  |
 | negative_sigv4_test | 6 | ✅ |
-| notification_webhook_test | 1 | ✅ |
+| notification_webhook_test | 2 | ✅ |
 | object_lambda_test | 16 |  |
 | object_lock | 33 |  |
 | overwrite_cleanup_regression_test | 1 |  |
 | presigned_negative_test | 7 | ✅ |
 | protocols | 16 |  |
-| quota_test | 13 |  |
+| quota_test | 14 |  |
 | reliability_disk_fault_test | 3 |  |
 | reliant | 10 | 4 ✅ |
 | replication_extension_test | 47 | 20 ✅ +27 🌙 |
@@ -75,4 +81,6 @@
 | tls_hot_reload_test | 1 | ✅ |
 | version_id_regression_test | 10 | ✅ |
 
-**Total listed: 439 tests across 57 modules · PR smoke subset: 112 tests / 27 modules** (25 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-16.
+`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above.
+
+**Total listed: 467 tests across 63 modules · PR smoke subset: 114 tests / 28 modules** (26 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-21.

rustfs/src/admin/handlers/audit.rs

@@ -221,7 +221,7 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
     validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }
 
-fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
     shared_target_mutation_block_reason(
         audit_target_specs(),
         AUDIT_ROUTE_PREFIX,
@@ -248,16 +248,18 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
     target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action)
 }
 
-fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
-    shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
-        .into_iter()
-        .map(|endpoint| AuditEndpoint {
-            account_id: endpoint.account_id,
-            service: endpoint.service,
-            status: endpoint.status,
-            source: endpoint.source,
-        })
-        .collect()
+fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> S3Result<Vec<AuditEndpoint>> {
+    Ok(
+        shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)?
+            .into_iter()
+            .map(|endpoint| AuditEndpoint {
+                account_id: endpoint.account_id,
+                service: endpoint.service,
+                status: endpoint.status,
+                source: endpoint.source,
+            })
+            .collect(),
+    )
 }
 
 fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
@@ -279,7 +281,7 @@ impl Operation for AuditTargetConfig {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
         let config_snapshot = load_server_config_from_store().await?;
-        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
             log_audit_target_operation_blocked!("set_audit_target_config", Some(target_type), Some(target_name), &reason);
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
@@ -305,12 +307,14 @@ impl Operation for AuditTargetConfig {
         )
         .await?;
 
-        update_audit_config_and_reload(audit_target_specs(), |config| {
+        let mutation_target_type = target_type.to_lowercase();
+        let mutation_target_name = target_name.to_lowercase();
+        update_audit_config_and_reload(audit_target_specs(), move |config| {
             config
                 .0
-                .entry(target_type.to_lowercase())
+                .entry(mutation_target_type.clone())
                 .or_default()
-                .insert(target_name.to_lowercase(), kvs.clone());
+                .insert(mutation_target_name.clone(), kvs.clone());
             true
         })
         .await
@@ -344,7 +348,7 @@ impl Operation for ListAuditTargets {
         }
 
         let config = load_server_config_from_store().await?;
-        let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses);
+        let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses)?;
         let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }).map_err(|e| {
             log_audit_target_request_failed!("list_audit_targets", "serialize_audit_targets_failed", None, None, e);
             s3_error!(InternalError, "failed to serialize audit targets: {}", e)
@@ -369,19 +373,21 @@ impl Operation for RemoveAuditTarget {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
         let config_snapshot = load_server_config_from_store().await?;
-        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
             log_audit_target_operation_blocked!("remove_audit_target_config", Some(target_type), Some(target_name), &reason);
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
 
-        update_audit_config_and_reload(audit_target_specs(), |config| {
+        let mutation_target_type = target_type.to_lowercase();
+        let mutation_target_name = target_name.to_lowercase();
+        update_audit_config_and_reload(audit_target_specs(), move |config| {
             let mut changed = false;
-            if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
-                if targets.remove(&target_name.to_lowercase()).is_some() {
+            if let Some(targets) = config.0.get_mut(&mutation_target_type) {
+                if targets.remove(&mutation_target_name).is_some() {
                     changed = true;
                 }
                 if targets.is_empty() {
-                    config.0.remove(&target_type.to_lowercase());
+                    config.0.remove(&mutation_target_type);
                 }
             }
             changed
@@ -469,7 +475,7 @@ mod tests {
                     (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
                     (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
                 let mixed = merged
                     .iter()
@@ -512,7 +518,7 @@ mod tests {
                     (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
                     (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
                 let mixed = merged
                     .iter()
@@ -549,7 +555,7 @@ mod tests {
                     (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
                     (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
 
                 let mixed = merged
                     .iter()
@@ -576,7 +582,8 @@ mod tests {
             ],
             || {
                 let config = Config(HashMap::new());
-                let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+                let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                    .expect("audit target mutation block reason");
                 assert!(reason.is_some());
                 assert!(reason.unwrap().contains("managed by environment variables"));
             },
@@ -613,7 +620,8 @@ mod tests {
                 AUDIT_WEBHOOK_SUB_SYS.to_string(),
                 HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
             )]));
-            let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary");
+            let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                .expect("audit target mutation block reason");
             assert!(reason.is_some());
             assert!(reason.unwrap().contains("both persisted config and environment variables"));
         });
@@ -633,7 +641,7 @@ mod tests {
                 ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
             ],
             || {
-                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
                 let mixed = merged
                     .iter()
                     .find(|entry| entry.account_id == "mixed-disabled")
@@ -655,7 +663,7 @@ mod tests {
                 ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
             ],
             || {
-                let merged = merge_audit_endpoints(&config, HashMap::new());
+                let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints");
                 let env_only = merged
                     .iter()
                     .find(|entry| entry.account_id == "env-only")
@@ -768,7 +776,7 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
-                let merged = merge_audit_endpoints(&config, runtime);
+                let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints");
                 let mixed = merged
                     .iter()
                     .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -787,7 +795,11 @@ mod tests {
         )]));
 
         with_audit_webhook_target_env_cleared("primarycase", || {
-            assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none());
+            assert!(
+                audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase")
+                    .expect("audit target mutation block reason")
+                    .is_none()
+            );
         });
     }
 
@@ -795,7 +807,11 @@ mod tests {
     fn audit_target_mutation_block_reason_allows_runtime_only_target() {
         with_audit_webhook_target_env_cleared("primary", || {
             let config = Config(HashMap::new());
-            assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary").is_none());
+            assert!(
+                audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary")
+                    .expect("audit target mutation block reason")
+                    .is_none()
+            );
         });
     }
 

rustfs/src/admin/handlers/audit_runtime_config.rs

@@ -14,11 +14,15 @@
 
 use crate::admin::handlers::target_descriptor::AdminTargetSpec;
 use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context};
-use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
+use crate::admin::storage_api::config::{
+    read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
+};
 use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
 use rustfs_config::DEFAULT_DELIMITER;
 use rustfs_config::server_config::Config;
 use s3s::{S3Result, s3_error};
+use tracing::warn;
 
 pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result<Config> {
     let Some(store) = current_object_store_handle_for_context(context) else {
@@ -51,30 +55,31 @@ pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config
         match system.get_state().await {
             AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
                 if has_targets {
-                    system
-                        .reload_config(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
+                    system.reload_config(config).await.map_err(|_| {
+                        warn!(reason = "reload_failed", "Failed to reload local audit runtime");
+                        s3_error!(InternalError, "failed to reload audit config")
+                    })?;
                 } else {
-                    system
-                        .close()
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
+                    system.close().await.map_err(|_| {
+                        warn!(reason = "stop_failed", "Failed to stop local audit runtime");
+                        s3_error!(InternalError, "failed to stop audit system")
+                    })?;
                 }
             }
             AuditSystemState::Stopped | AuditSystemState::Stopping => {
                 if has_targets {
-                    system
-                        .start(config)
-                        .await
-                        .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+                    system.start(config).await.map_err(|_| {
+                        warn!(reason = "start_failed", "Failed to start local audit runtime");
+                        s3_error!(InternalError, "failed to start audit system")
+                    })?;
                 }
             }
         }
     } else if has_targets {
-        start_global_audit_system(config)
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
+        start_global_audit_system(config).await.map_err(|_| {
+            warn!(reason = "start_failed", "Failed to start global audit runtime");
+            s3_error!(InternalError, "failed to start audit system")
+        })?;
     }
 
     Ok(())
@@ -86,30 +91,40 @@ async fn update_audit_config_and_reload_for_context<F>(
     mut modifier: F,
 ) -> S3Result<()>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
     let Some(store) = current_object_store_handle_for_context(context) else {
         return Err(s3_error!(InternalError, "server storage not initialized"));
     };
 
-    let mut config = read_admin_config_without_migrate(store.clone())
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
+    let specs = specs.to_vec();
+    let lock_store = store.clone();
+    with_admin_server_config_write_lock(lock_store, move || async move {
+        let mut config = read_admin_config_without_migrate_no_lock(store.clone())
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
 
-    if !modifier(&mut config) {
-        return Ok(());
-    }
+        if !modifier(&mut config) {
+            return Ok(());
+        }
 
-    save_admin_server_config(store, &config)
-        .await
-        .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+        save_admin_server_config_no_lock(store, &config)
+            .await
+            .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
 
-    apply_audit_runtime_config(specs, config).await
+        // Keep persistence and runtime publication in one detached, serialized
+        // mutation. Otherwise a cancelled caller or two concurrent updates can
+        // leave the persisted config and active audit generation disagreeing.
+        apply_audit_runtime_config(&specs, config).await
+    })
+    .await
+    .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+    Ok(())
 }
 
 pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
     let context = current_app_context();
     update_audit_config_and_reload_for_context(context.as_deref(), specs, modifier).await
@@ -121,26 +136,30 @@ pub(crate) async fn set_audit_target_config(
     target_name: &str,
     kvs: rustfs_config::server_config::KVS,
 ) -> S3Result<()> {
-    update_audit_config_and_reload(specs, |config| {
+    let subsystem = subsystem.to_lowercase();
+    let target_name = target_name.to_lowercase();
+    update_audit_config_and_reload(specs, move |config| {
         config
             .0
-            .entry(subsystem.to_lowercase())
+            .entry(subsystem.clone())
             .or_default()
-            .insert(target_name.to_lowercase(), kvs.clone());
+            .insert(target_name.clone(), kvs.clone());
         true
     })
     .await
 }
 
 pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
-    update_audit_config_and_reload(specs, |config| {
+    let subsystem = subsystem.to_lowercase();
+    let target_name = target_name.to_lowercase();
+    update_audit_config_and_reload(specs, move |config| {
         let mut changed = false;
-        if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
-            if targets.remove(&target_name.to_lowercase()).is_some() {
+        if let Some(targets) = config.0.get_mut(&subsystem) {
+            if targets.remove(&target_name).is_some() {
                 changed = true;
             }
             if targets.is_empty() {
-                config.0.remove(&subsystem.to_lowercase());
+                config.0.remove(&subsystem);
             }
         }
         changed

rustfs/src/admin/handlers/config_admin.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::admin::auth::validate_admin_request;
+use crate::admin::handlers::supervise_admin_mutation;
 use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::admin::runtime_sources::{
     current_app_context, current_object_store_handle_for_context, current_server_config_for_context, publish_server_config,
@@ -20,12 +21,14 @@ use crate::admin::runtime_sources::{
 use crate::admin::service::config::{
     CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN,
     LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem,
-    prepare_server_config, signal_config_snapshot_reload, signal_dynamic_config_reload,
+    preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked,
+    signal_dynamic_config_reload_checked,
 };
 use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
 use crate::admin::storage_api::config::{
     RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate,
-    save_admin_config, save_admin_server_config,
+    read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
 };
 use crate::admin::storage_api::contract::list::ListOperations as _;
 use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
@@ -47,7 +50,7 @@ use rustfs_config::notify::{
     ENV_NOTIFY_MQTT_QOS, ENV_NOTIFY_MQTT_QUEUE_DIR, ENV_NOTIFY_MQTT_QUEUE_LIMIT, ENV_NOTIFY_MQTT_RECONNECT_INTERVAL,
     ENV_NOTIFY_MQTT_TOPIC, ENV_NOTIFY_MQTT_USERNAME, ENV_NOTIFY_WEBHOOK_AUTH_TOKEN, ENV_NOTIFY_WEBHOOK_CLIENT_CERT,
     ENV_NOTIFY_WEBHOOK_CLIENT_KEY, ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, ENV_NOTIFY_WEBHOOK_QUEUE_DIR,
-    ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
+    ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_SUB_SYSTEMS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::oidc::{
     ENV_IDENTITY_OPENID_CLAIM_NAME, ENV_IDENTITY_OPENID_CLAIM_PREFIX, ENV_IDENTITY_OPENID_CLIENT_ID,
@@ -726,6 +729,14 @@ async fn load_server_config_from_store() -> S3Result<ServerConfig> {
         .map_err(Into::into)
 }
 
+async fn load_server_config_from_store_locked() -> S3Result<ServerConfig> {
+    let store = object_store()?;
+    read_admin_config_without_migrate_no_lock(store)
+        .await
+        .map_err(ApiError::from)
+        .map_err(Into::into)
+}
+
 async fn load_active_server_config() -> S3Result<ServerConfig> {
     if let Ok(config) = load_server_config_from_store().await {
         return Ok(config);
@@ -736,9 +747,9 @@ async fn load_active_server_config() -> S3Result<ServerConfig> {
         .ok_or_else(|| s3_error!(InternalError, "server config is not initialized"))
 }
 
-async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
+async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> {
     let store = object_store()?;
-    save_admin_server_config(store, config)
+    save_admin_server_config_no_lock(store, config)
         .await
         .map_err(ApiError::from)
         .map_err(Into::into)
@@ -1563,20 +1574,144 @@ async fn commit_prepared_config(
 /// Re-apply local mutable worker families after a full-config replacement.
 /// Peers receive one full-snapshot signal after this returns; signaling each
 /// family here as well would recreate audit/scanner targets twice per peer.
-async fn apply_dynamic_subsystems(config: &ServerConfig) {
+fn publish_notify_config_intent(
+    config: &ServerConfig,
+    sub_system: Option<&str>,
+) -> Option<rustfs_notify::NotificationLifecycleTransition> {
+    (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)))
+        .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone()))
+}
+
+async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> {
+    if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) {
+        preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?;
+    }
+    Ok(())
+}
+
+async fn wait_notify_config_intent(transition: Option<rustfs_notify::NotificationLifecycleTransition>) -> S3Result<bool> {
+    let Some(transition) = transition else {
+        return Ok(false);
+    };
+    transition.wait().await.map_err(|err| {
+        warn!(error = %err, "Failed to apply local notification config");
+        s3_error!(InternalError, "failed to apply notification config")
+    })?;
+    Ok(true)
+}
+
+async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec<String> {
+    let mut failures = Vec::new();
     for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
-        if let Err(err) = apply_dynamic_config_for_subsystem(config, sub_system).await {
+        if NOTIFY_SUB_SYSTEMS.contains(&sub_system) {
+            continue;
+        }
+        if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() {
+            failures.push(format!("local {sub_system}"));
             warn!(
                 event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
                 component = LOG_COMPONENT_ADMIN,
                 subsystem = LOG_SUBSYSTEM_CONFIG,
                 config_subsystem = sub_system,
                 state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-                error = %err,
+                reason = "apply_failed",
                 "Published server config but failed to reload a local worker subsystem"
             );
         }
     }
+    failures
+}
+
+fn finish_config_reconciliation(errors: Vec<String>) -> S3Result<()> {
+    if errors.is_empty() {
+        Ok(())
+    } else {
+        Err(s3_error!(
+            InternalError,
+            "server config persisted but runtime convergence failed: {}",
+            errors.join("; ")
+        ))
+    }
+}
+
+async fn reconcile_targeted_config(
+    config: ServerConfig,
+    sub_system: Option<String>,
+    storage_class_applied: bool,
+    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
+) -> S3Result<bool> {
+    let mut errors = Vec::new();
+    let notify_applied = notify_transition.is_some();
+    if let Err(err) = wait_notify_config_intent(notify_transition).await {
+        warn!(error = %err, "Local notification config failed to converge");
+        errors.push("local notify".to_string());
+    }
+
+    let config_applied = if notify_applied {
+        if let Some(sub_system) = sub_system.as_deref()
+            && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await
+        {
+            warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
+            errors.push(format!("peer {sub_system}"));
+        }
+        true
+    } else if storage_class_applied {
+        if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
+            warn!(error = %err, "Peer storage-class reload failed");
+            errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
+        }
+        true
+    } else if let Some(sub_system) = sub_system.as_deref()
+        && is_dynamic_config_subsystem(sub_system)
+    {
+        let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await {
+            Ok(applied) => applied,
+            Err(_) => {
+                warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed");
+                errors.push(format!("local {sub_system}"));
+                false
+            }
+        };
+        if let Err(err) = signal_dynamic_config_reload_checked(sub_system).await {
+            warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed");
+            errors.push(format!("peer {sub_system}"));
+        }
+        config_applied
+    } else {
+        if let Err(err) = signal_config_snapshot_reload_checked().await {
+            warn!(error = %err, "Peer config snapshot reload failed");
+            errors.push("peer config snapshot".to_string());
+        }
+        false
+    };
+
+    finish_config_reconciliation(errors)?;
+    Ok(config_applied)
+}
+
+async fn reconcile_full_config(
+    config: ServerConfig,
+    notify_transition: Option<rustfs_notify::NotificationLifecycleTransition>,
+) -> S3Result<()> {
+    let mut errors = Vec::new();
+    if let Err(err) = wait_notify_config_intent(notify_transition).await {
+        warn!(error = %err, "Local notification config failed to converge");
+        errors.push("local notify".to_string());
+    }
+    if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await {
+        warn!(error = %err, "Peer storage-class reload failed");
+        errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}"));
+    }
+    errors.extend(apply_non_notify_dynamic_subsystems(&config).await);
+    if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await {
+        warn!(error = %err, "Peer notification config reload failed");
+        errors.push("peer notify".to_string());
+    }
+    if let Err(err) = signal_config_snapshot_reload_checked().await {
+        warn!(error = %err, "Peer config snapshot reload failed");
+        errors.push("peer config snapshot".to_string());
+    }
+    finish_config_reconciliation(errors)
 }
 
 pub struct GetConfigKVHandler {}
@@ -1611,37 +1746,39 @@ impl Operation for SetConfigKVHandler {
         }
         validate_config_directives(&directives)?;
 
-        let sub_system = config_update_sub_system(&directives)?;
-        let mut config = load_server_config_from_store().await?;
-        apply_set_directives(&mut config, &directives)?;
-        let prepared = prepare_server_config(&config, sub_system).await?;
-        save_server_config_history(&body).await?;
-        let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
-            commit_prepared_config(
-                config.clone(),
-                prepared,
-                save_server_config_to_store(&config),
-                publish_prepared_config_snapshots,
-            )
-            .await?;
-            signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-            true
-        } else {
-            save_server_config_to_store(&config).await?;
-            publish_server_config(config.clone());
-            if let Some(sub_system) = sub_system
-                && is_dynamic_config_subsystem(sub_system)
-            {
-                let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
-                if config_applied {
-                    signal_dynamic_config_reload(sub_system).await;
-                }
-                config_applied
-            } else {
-                signal_config_snapshot_reload().await;
-                false
-            }
-        };
+        let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
+        let transaction_sub_system = sub_system.clone();
+        let config_store = object_store()?;
+        let config_applied = supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(sub_system.as_deref()).await?;
+            let (config, storage_class_applied, notify_transition) =
+                with_admin_server_config_write_lock(config_store, move || async move {
+                    let sub_system = transaction_sub_system.as_deref();
+                    let mut config = load_server_config_from_store_locked().await?;
+                    apply_set_directives(&mut config, &directives)?;
+                    let prepared = prepare_server_config(&config, sub_system).await?;
+                    save_server_config_history(&body).await?;
+                    if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
+                        commit_prepared_config(
+                            config.clone(),
+                            prepared,
+                            save_server_config_to_store_locked(&config),
+                            publish_prepared_config_snapshots,
+                        )
+                        .await?;
+                    } else {
+                        save_server_config_to_store_locked(&config).await?;
+                        publish_server_config(config.clone());
+                    }
+                    let notify_transition = publish_notify_config_intent(&config, sub_system);
+                    Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
+                })
+                .await
+                .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+
+            reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
+        })
+        .await?;
 
         success_response(config_applied)
     }
@@ -1660,37 +1797,39 @@ impl Operation for DelConfigKVHandler {
         }
         validate_config_directives(&directives)?;
 
-        let sub_system = config_update_sub_system(&directives)?;
-        let mut config = load_server_config_from_store().await?;
-        apply_delete_directives(&mut config, &directives);
-        let prepared = prepare_server_config(&config, sub_system).await?;
-        save_server_config_history(&body).await?;
-        let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
-            commit_prepared_config(
-                config.clone(),
-                prepared,
-                save_server_config_to_store(&config),
-                publish_prepared_config_snapshots,
-            )
-            .await?;
-            signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-            true
-        } else {
-            save_server_config_to_store(&config).await?;
-            publish_server_config(config.clone());
-            if let Some(sub_system) = sub_system
-                && is_dynamic_config_subsystem(sub_system)
-            {
-                let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?;
-                if config_applied {
-                    signal_dynamic_config_reload(sub_system).await;
-                }
-                config_applied
-            } else {
-                signal_config_snapshot_reload().await;
-                false
-            }
-        };
+        let sub_system = config_update_sub_system(&directives)?.map(str::to_owned);
+        let transaction_sub_system = sub_system.clone();
+        let config_store = object_store()?;
+        let config_applied = supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(sub_system.as_deref()).await?;
+            let (config, storage_class_applied, notify_transition) =
+                with_admin_server_config_write_lock(config_store, move || async move {
+                    let sub_system = transaction_sub_system.as_deref();
+                    let mut config = load_server_config_from_store_locked().await?;
+                    apply_delete_directives(&mut config, &directives);
+                    let prepared = prepare_server_config(&config, sub_system).await?;
+                    save_server_config_history(&body).await?;
+                    if sub_system == Some(STORAGE_CLASS_SUB_SYS) {
+                        commit_prepared_config(
+                            config.clone(),
+                            prepared,
+                            save_server_config_to_store_locked(&config),
+                            publish_prepared_config_snapshots,
+                        )
+                        .await?;
+                    } else {
+                        save_server_config_to_store_locked(&config).await?;
+                        publish_server_config(config.clone());
+                    }
+                    let notify_transition = publish_notify_config_intent(&config, sub_system);
+                    Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition))
+                })
+                .await
+                .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
+
+            reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await
+        })
+        .await?;
 
         success_response(config_applied)
     }
@@ -1776,16 +1915,26 @@ impl Operation for RestoreConfigHistoryKVHandler {
         let mut config = ServerConfig::new();
         apply_set_directives(&mut config, &directives)?;
         let prepared = prepare_server_config(&config, None).await?;
-        commit_prepared_config(
-            config.clone(),
-            prepared,
-            save_server_config_to_store(&config),
-            publish_prepared_config_snapshots,
-        )
+        let config_store = object_store()?;
+        supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(None).await?;
+            let persisted_config = config.clone();
+            let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
+                commit_prepared_config(
+                    persisted_config.clone(),
+                    prepared,
+                    save_server_config_to_store_locked(&persisted_config),
+                    publish_prepared_config_snapshots,
+                )
+                .await?;
+                Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
+            })
+            .await
+            .map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??;
+
+            reconcile_full_config(config, notify_transition).await
+        })
         .await?;
-        signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-        apply_dynamic_subsystems(&config).await;
-        signal_config_snapshot_reload().await;
 
         success_response(false)
     }
@@ -1820,17 +1969,27 @@ impl Operation for SetConfigHandler {
         let mut config = ServerConfig::new();
         apply_set_directives(&mut config, &directives)?;
         let prepared = prepare_server_config(&config, None).await?;
-        save_server_config_history(&body).await?;
-        commit_prepared_config(
-            config.clone(),
-            prepared,
-            save_server_config_to_store(&config),
-            publish_prepared_config_snapshots,
-        )
+        let config_store = object_store()?;
+        supervise_admin_mutation("config mutation", async move {
+            preflight_notify_config_intent(None).await?;
+            save_server_config_history(&body).await?;
+            let persisted_config = config.clone();
+            let notify_transition = with_admin_server_config_write_lock(config_store, move || async move {
+                commit_prepared_config(
+                    persisted_config.clone(),
+                    prepared,
+                    save_server_config_to_store_locked(&persisted_config),
+                    publish_prepared_config_snapshots,
+                )
+                .await?;
+                Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None))
+            })
+            .await
+            .map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??;
+
+            reconcile_full_config(config, notify_transition).await
+        })
         .await?;
-        signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await;
-        apply_dynamic_subsystems(&config).await;
-        signal_config_snapshot_reload().await;
 
         success_response(false)
     }
@@ -1890,92 +2049,6 @@ mod tests {
         assert_eq!(*events.lock().expect("result events lock"), ["persist"]);
     }
 
-    #[test]
-    fn storage_config_write_handlers_persist_before_publishing() {
-        const SOURCE: &str = include_str!("config_admin.rs");
-
-        for (handler, next_handler, follow_up) in [
-            (
-                "SetConfigKVHandler",
-                "DelConfigKVHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "DelConfigKVHandler",
-                "HelpConfigKVHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "RestoreConfigHistoryKVHandler",
-                "GetConfigHandler",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-            (
-                "SetConfigHandler",
-                "#[cfg(test)]",
-                "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await",
-            ),
-        ] {
-            let start_marker = format!("impl Operation for {handler}");
-            let start = SOURCE
-                .find(&start_marker)
-                .unwrap_or_else(|| panic!("missing {handler} implementation"));
-            let tail = &SOURCE[start..];
-            let end = tail
-                .find(next_handler)
-                .unwrap_or_else(|| panic!("missing {next_handler} after {handler}"));
-            let implementation = &tail[..end];
-            let prepared_commit_path = if matches!(handler, "SetConfigKVHandler" | "DelConfigKVHandler") {
-                let branch_start = implementation
-                    .find("if sub_system == Some(STORAGE_CLASS_SUB_SYS)")
-                    .unwrap_or_else(|| panic!("missing storage-class branch in {handler}"));
-                let branch = &implementation[branch_start..];
-                let branch_end = branch
-                    .find("} else {")
-                    .unwrap_or_else(|| panic!("missing non-storage branch in {handler}"));
-                &branch[..branch_end]
-            } else {
-                implementation
-            };
-
-            assert_eq!(prepared_commit_path.matches("commit_prepared_config(").count(), 1, "{handler}");
-            let commit_start = prepared_commit_path.find("commit_prepared_config(").expect("commit call");
-            let commit_end = prepared_commit_path[commit_start..].find(';').expect("commit terminator") + commit_start;
-            let commit_statement = &prepared_commit_path[commit_start..=commit_end];
-            assert!(commit_statement.contains(".await?;"), "{handler} must propagate commit failure");
-
-            let follow_up_start = prepared_commit_path
-                .find(follow_up)
-                .unwrap_or_else(|| panic!("missing follow-up in {handler}"));
-            assert!(follow_up_start > commit_end, "{handler} must run follow-up only after commit");
-            assert!(
-                !prepared_commit_path.contains("publish_server_config("),
-                "{handler} must not publish directly"
-            );
-            assert!(
-                !prepared_commit_path.contains(".publish_storage_class("),
-                "{handler} must not publish directly"
-            );
-
-            if matches!(handler, "RestoreConfigHistoryKVHandler" | "SetConfigHandler") {
-                let worker_start = prepared_commit_path
-                    .find("apply_dynamic_subsystems(&config).await")
-                    .unwrap_or_else(|| panic!("missing local worker apply in {handler}"));
-                let signal_start = prepared_commit_path
-                    .find("signal_config_snapshot_reload().await")
-                    .unwrap_or_else(|| panic!("missing snapshot signal in {handler}"));
-                assert!(
-                    worker_start > follow_up_start,
-                    "{handler} must converge peer parity before local worker apply"
-                );
-                assert!(
-                    signal_start > worker_start,
-                    "{handler} must signal the full snapshot after local worker apply"
-                );
-            }
-        }
-    }
-
     #[test]
     fn tokenize_config_line_handles_quotes_and_escapes() {
         let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#)

rustfs/src/admin/handlers/event.rs

@@ -15,6 +15,7 @@
 use crate::admin::{
     auth::validate_admin_request,
     handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
+    handlers::supervise_admin_mutation,
     handlers::target_descriptor::{
         AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
         build_json_response, collect_runtime_statuses, extract_supported_target_params,
@@ -22,6 +23,8 @@ use crate::admin::{
         target_mutation_block_reason as shared_target_mutation_block_reason,
     },
     router::{AdminOperation, Operation, S3Router},
+    runtime_sources::{AppContext, app_context_from_req},
+    service::config::{preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context},
 };
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{
@@ -43,6 +46,45 @@ use std::sync::LazyLock;
 use tracing::{Span, error, info, warn};
 
 const LOG_COMPONENT_ADMIN_API: &str = "admin_api";
+
+async fn converge_target_mutation_on_cluster(
+    context: Option<&AppContext>,
+    target_type: &str,
+    local_result: Result<(), rustfs_notify::NotificationError>,
+) -> S3Result<()> {
+    let subsystem = notification_target_subsystem(target_type)?;
+    let peer_result = signal_dynamic_config_reload_checked_for_context(context, subsystem).await;
+
+    match (local_result, peer_result) {
+        (Ok(()), Ok(())) => Ok(()),
+        (Err(local), Ok(())) => {
+            warn!(target_type, error = %local, "Local notification runtime failed to converge");
+            Err(s3_error!(InternalError, "local notification runtime failed to converge"))
+        }
+        (Ok(()), Err(peer)) => Err(peer),
+        (Err(local), Err(peer)) => {
+            warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed");
+            Err(s3_error!(
+                InternalError,
+                "local notification runtime and peer convergence failed: {}",
+                peer
+            ))
+        }
+    }
+}
+
+fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> {
+    notification_target_specs()
+        .iter()
+        .find(|spec| spec.subsystem == target_type)
+        .map(|spec| spec.subsystem)
+        .ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type))
+}
+
+async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> {
+    preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await
+}
+
 const LOG_SUBSYSTEM_NOTIFICATION_TARGET: &str = "notification_target";
 const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
 const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
@@ -203,6 +245,7 @@ struct NotificationEndpoint {
 
 #[derive(Serialize, Debug)]
 struct NotificationEndpointsResponse {
+    notify_enabled: bool,
     notification_endpoints: Vec<NotificationEndpoint>,
 }
 
@@ -229,7 +272,7 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
     validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }
 
-fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
+fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result<Option<String>> {
     shared_target_mutation_block_reason(
         notification_target_specs(),
         NOTIFY_ROUTE_PREFIX,
@@ -256,16 +299,21 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
     target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action)
 }
 
-fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
-    shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
-        .into_iter()
-        .map(|endpoint| NotificationEndpoint {
-            account_id: endpoint.account_id,
-            service: endpoint.service,
-            status: endpoint.status,
-            source: endpoint.source,
-        })
-        .collect()
+fn merge_notification_endpoints(
+    config: &Config,
+    runtime_statuses: HashMap<EndpointKey, String>,
+) -> S3Result<Vec<NotificationEndpoint>> {
+    Ok(
+        shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)?
+            .into_iter()
+            .map(|endpoint| NotificationEndpoint {
+                account_id: endpoint.account_id,
+                service: endpoint.service,
+                status: endpoint.status,
+                source: endpoint.source,
+            })
+            .collect(),
+    )
 }
 
 fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
@@ -284,6 +332,7 @@ impl Operation for NotificationTarget {
         let span = Span::current();
         let _enter = span.enter();
         let (target_type, target_name) = extract_target_params(&params)?;
+        let context = app_context_from_req(&req);
 
         authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
         if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -291,7 +340,7 @@ impl Operation for NotificationTarget {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
         let (ns, config_snapshot) = load_notification_config_snapshot().await?;
-        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
             log_notification_target_operation_blocked!("set_target_config", Some(target_type), Some(target_name), &reason);
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
@@ -320,7 +369,15 @@ impl Operation for NotificationTarget {
         )
         .await?;
 
-        ns.set_target_config(target_type, target_name, kvs).await.map_err(|e| {
+        let mutation_target_type = target_type.to_owned();
+        let mutation_target_name = target_name.to_owned();
+        supervise_admin_mutation("notification target mutation", async move {
+            preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
+            let local_result = ns.set_target_config(&mutation_target_type, &mutation_target_name, kvs).await;
+            converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
+        })
+        .await
+        .map_err(|e| {
             log_notification_target_request_failed!(
                 "set_target_config",
                 "set_target_config_failed",
@@ -328,7 +385,7 @@ impl Operation for NotificationTarget {
                 Some(target_name),
                 e
             );
-            s3_error!(InternalError, "failed to set target config: {}", e)
+            e
         })?;
         log_notification_target_config_updated!("set_target_config", target_type, target_name);
 
@@ -343,11 +400,29 @@ impl Operation for ListNotificationTargets {
         let span = Span::current();
         let _enter = span.enter();
         authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
+        refresh_persisted_module_switches_from_store().await.map_err(|err| {
+            warn!(
+                event = EVENT_ADMIN_REQUEST_FAILED,
+                component = LOG_COMPONENT_ADMIN_API,
+                subsystem = LOG_SUBSYSTEM_NOTIFICATION_TARGET,
+                operation = "list_targets",
+                result = "failed",
+                reason = "module_switch_refresh_failed",
+                error = %err,
+                "admin request failed"
+            );
+            s3_error!(InternalError, "failed to refresh notification module state")
+        })?;
+        let notify_enabled = refresh_notify_module_enabled();
         let (ns, config) = load_notification_config_snapshot().await?;
         let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
-        let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
+        let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses)?;
 
-        let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }).map_err(|e| {
+        let data = serde_json::to_vec(&NotificationEndpointsResponse {
+            notify_enabled,
+            notification_endpoints,
+        })
+        .map_err(|e| {
             log_notification_target_request_failed!("list_targets", "serialize_targets_failed", None, None, e);
             s3_error!(InternalError, "failed to serialize targets: {}", e)
         })?;
@@ -409,6 +484,7 @@ impl Operation for RemoveNotificationTarget {
         let span = Span::current();
         let _enter = span.enter();
         let (target_type, target_name) = extract_target_params(&params)?;
+        let context = app_context_from_req(&req);
 
         authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
         if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
@@ -416,12 +492,20 @@ impl Operation for RemoveNotificationTarget {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
         let (ns, config_snapshot) = load_notification_config_snapshot().await?;
-        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
+        if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? {
             log_notification_target_operation_blocked!("remove_target_config", Some(target_type), Some(target_name), &reason);
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
 
-        ns.remove_target_config(target_type, target_name).await.map_err(|e| {
+        let mutation_target_type = target_type.to_owned();
+        let mutation_target_name = target_name.to_owned();
+        supervise_admin_mutation("notification target mutation", async move {
+            preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?;
+            let local_result = ns.remove_target_config(&mutation_target_type, &mutation_target_name).await;
+            converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await
+        })
+        .await
+        .map_err(|e| {
             log_notification_target_request_failed!(
                 "remove_target_config",
                 "remove_target_config_failed",
@@ -429,7 +513,7 @@ impl Operation for RemoveNotificationTarget {
                 Some(target_name),
                 e
             );
-            s3_error!(InternalError, "failed to remove target config: {}", e)
+            e
         })?;
         log_notification_target_config_updated!("remove_target_config", target_type, target_name);
 
@@ -464,6 +548,41 @@ mod tests {
         }])
     }
 
+    #[test]
+    fn notification_target_subsystem_resolves_admin_route_type() {
+        assert_eq!(
+            notification_target_subsystem(NOTIFY_WEBHOOK_SUB_SYS).expect("webhook subsystem should resolve"),
+            NOTIFY_WEBHOOK_SUB_SYS
+        );
+        assert!(notification_target_subsystem("webhook").is_err());
+    }
+
+    #[test]
+    fn notification_endpoints_response_includes_required_module_state() {
+        let response = NotificationEndpointsResponse {
+            notify_enabled: true,
+            notification_endpoints: vec![NotificationEndpoint {
+                account_id: "primary".to_string(),
+                service: "webhook".to_string(),
+                status: "online".to_string(),
+                source: TargetEndpointSource::Config,
+            }],
+        };
+
+        assert_eq!(
+            serde_json::to_value(response).expect("notification target response should serialize"),
+            serde_json::json!({
+                "notify_enabled": true,
+                "notification_endpoints": [{
+                    "account_id": "primary",
+                    "service": "webhook",
+                    "status": "online",
+                    "source": "config"
+                }]
+            })
+        );
+    }
+
     #[test]
     fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() {
         let mut cfg_map = HashMap::new();
@@ -478,7 +597,7 @@ mod tests {
         let config = Config(cfg_map);
 
         let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]);
-        let merged = merge_notification_endpoints(&config, runtime);
+        let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
         let mqtt = merged
             .iter()
@@ -507,7 +626,7 @@ mod tests {
             (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()),
             (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()),
         ]);
-        let merged = merge_notification_endpoints(&config, runtime);
+        let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
         let env_only = merged
             .iter()
@@ -549,7 +668,7 @@ mod tests {
                     (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()),
                     (("env-only".to_string(), "webhook".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
                 let mixed = merged
                     .iter()
@@ -592,7 +711,7 @@ mod tests {
                     (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
                     (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
                 let mixed = merged
                     .iter()
@@ -629,7 +748,7 @@ mod tests {
                     (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
                     (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
                 ]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
 
                 let mixed = merged
                     .iter()
@@ -656,7 +775,8 @@ mod tests {
             ],
             || {
                 let config = Config(HashMap::new());
-                let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+                let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary")
+                    .expect("target mutation block reason");
                 assert!(reason.is_some());
                 assert!(reason.unwrap().contains("managed by environment variables"));
             },
@@ -696,7 +816,8 @@ mod tests {
                 NOTIFY_WEBHOOK_SUB_SYS.to_string(),
                 HashMap::from([("primary".to_string(), enabled_kvs("on"))]),
             )]));
-            let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary");
+            let reason =
+                target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary").expect("target mutation block reason");
             assert!(reason.is_some());
             assert!(reason.unwrap().contains("both persisted config and environment variables"));
         });
@@ -709,7 +830,11 @@ mod tests {
             NOTIFY_WEBHOOK_SUB_SYS.to_string(),
             HashMap::from([(target_name.to_string(), enabled_kvs("on"))]),
         )]));
-        assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none());
+        assert!(
+            target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name)
+                .expect("target mutation block reason")
+                .is_none()
+        );
     }
 
     #[test]
@@ -726,7 +851,7 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")),
             ],
             || {
-                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
                 let mixed = merged
                     .iter()
                     .find(|entry| entry.account_id == "mixed-disabled")
@@ -748,7 +873,7 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")),
             ],
             || {
-                let merged = merge_notification_endpoints(&config, HashMap::new());
+                let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints");
                 let env_only = merged
                     .iter()
                     .find(|entry| entry.account_id == "env-only")
@@ -798,7 +923,7 @@ mod tests {
             ],
             || {
                 let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]);
-                let merged = merge_notification_endpoints(&config, runtime);
+                let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints");
                 let mixed = merged
                     .iter()
                     .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
@@ -835,7 +960,11 @@ mod tests {
                 ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>),
             ],
             || {
-                assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none());
+                assert!(
+                    target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase")
+                        .expect("target mutation block reason")
+                        .is_none()
+                );
             },
         );
     }
@@ -864,6 +993,22 @@ mod tests {
             list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
             "notification target list should require GetBucketTargetAction"
         );
+        let authorize_index = list_block
+            .find("authorize_notification_admin_request")
+            .expect("target list should authorize the request");
+        let refresh_index = list_block
+            .find("refresh_persisted_module_switches_from_store().await.map_err")
+            .expect("target list should fail when persisted module state cannot be refreshed");
+        let effective_state_index = list_block
+            .find("let notify_enabled = refresh_notify_module_enabled();")
+            .expect("target list should resolve the effective env and persisted module state");
+        let load_index = list_block
+            .find("load_notification_config_snapshot().await?")
+            .expect("target list should load notification config");
+        assert!(
+            authorize_index < refresh_index && refresh_index < effective_state_index && effective_state_index < load_index,
+            "target list should authorize, refresh persisted state, resolve effective state, then load targets"
+        );
         assert!(
             arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"),
             "notification target arn listing should require GetBucketTargetAction"

rustfs/src/admin/handlers/mod.rs

@@ -63,6 +63,19 @@ pub mod user_iam;
 pub mod user_lifecycle;
 pub mod user_policy_binding;
 
+pub(crate) async fn supervise_admin_mutation<T>(
+    operation: &'static str,
+    mutation: impl std::future::Future<Output = s3s::S3Result<T>> + Send + 'static,
+) -> s3s::S3Result<T>
+where
+    T: Send + 'static,
+{
+    tokio::spawn(mutation).await.map_err(|err| {
+        let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" };
+        s3s::s3_error!(InternalError, "{} task {}", operation, outcome)
+    })?
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -114,6 +127,44 @@ mod tests {
         // Test passes if we reach this point without panicking
     }
 
+    #[tokio::test]
+    async fn supervised_admin_mutation_survives_waiter_cancellation() {
+        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
+        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+        let (completed_tx, completed_rx) = tokio::sync::oneshot::channel();
+
+        let waiter = tokio::spawn(async move {
+            supervise_admin_mutation("test mutation", async move {
+                let _ = started_tx.send(());
+                let _ = release_rx.await;
+                let _ = completed_tx.send(());
+                Ok(())
+            })
+            .await
+        });
+
+        started_rx.await.expect("mutation started");
+        waiter.abort();
+        release_tx.send(()).expect("release mutation");
+        tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx)
+            .await
+            .expect("detached mutation should complete")
+            .expect("completion signal");
+    }
+
+    #[tokio::test]
+    async fn supervised_admin_mutation_does_not_expose_panic_payload() {
+        let error = supervise_admin_mutation::<()>("test mutation", async {
+            panic!("do-not-expose-payload");
+        })
+        .await
+        .expect_err("panicking mutation should fail");
+
+        let rendered = error.to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+
     // Note: Testing the actual async handler implementations requires:
     // 1. S3Request setup with proper headers, URI, and credentials
     // 2. Global object store initialization

rustfs/src/admin/handlers/module_switch.rs

@@ -12,26 +12,31 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::admin::runtime_sources::default_admin_usecase;
+use crate::admin::runtime_sources::{AppContext, app_context_from_req, default_admin_usecase};
+use crate::admin::service::config::{
+    preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context,
+};
 use crate::admin::{
     auth::validate_admin_request,
+    handlers::supervise_admin_mutation,
     router::{AdminOperation, Operation, S3Router},
 };
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{
-    ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot,
-    init_event_notifier, refresh_audit_module_enabled, refresh_notify_module_enabled,
-    refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, shutdown_event_notifier,
-    start_audit_system, stop_audit_system, validate_module_switch_update,
+    ADMIN_PREFIX, MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
+    RemoteAddr, apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled,
+    mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled,
+    refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to,
+    validate_module_switch_update,
 };
 use http::{HeaderMap, StatusCode};
 use hyper::Method;
 use matchit::Params;
-use rustfs_audit::AuditError;
 use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
 use rustfs_policy::policy::action::{Action, AdminAction};
 use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::{Deserialize, Serialize};
+use std::sync::Arc;
 
 pub fn register_module_switch_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
     r.insert(
@@ -139,6 +144,74 @@ async fn refresh_module_switch_snapshot() -> S3Result<ModuleSwitchSnapshot> {
     Ok(current_module_switch_snapshot())
 }
 
+async fn apply_module_switch_update(context: Arc<AppContext>, switches: PersistedModuleSwitches) -> S3Result<()> {
+    preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?;
+    let store = context.object_store();
+    mark_event_notifier_unreconciled();
+    let notification_system = rustfs_notify::ensure_live_events();
+    if switches.notify_enabled {
+        notification_system
+            .reload_persisted_config_from_store(store.clone())
+            .await
+            .map_err(|err| {
+                tracing::warn!(error = %err, "Failed to load notification config for module switch update");
+                s3_error!(InternalError, "failed to load notification config")
+            })?;
+    }
+
+    let transition_system = notification_system.clone();
+    let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || {
+        let enabled = refresh_notify_module_enabled();
+        transition_system.publish_targets_enabled(enabled, None)
+    })
+    .await
+    .map_err(|err| {
+        tracing::warn!(error = %err, "Failed to save module switches");
+        s3_error!(InternalError, "failed to save module switches")
+    })?;
+
+    let mut failures = Vec::new();
+    let mut notify_converged = true;
+    if let Err(err) = notify_transition.wait().await {
+        tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update");
+        notify_converged = false;
+        failures.push("local notify");
+    }
+    if !switches.notify_enabled
+        && let Err(err) = notification_system.reload_persisted_config_from_store(store).await
+    {
+        tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable");
+        notify_converged = false;
+        failures.push("local notify config cache");
+    }
+    if notify_converged && notification_system.runtime_lifecycle_is_converged() {
+        mark_event_notifier_reconciled();
+    } else if notify_converged {
+        failures.push("local notify convergence");
+    }
+
+    if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() {
+        tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update");
+        failures.push("local audit");
+    }
+    if let Err(err) =
+        signal_dynamic_config_reload_checked_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await
+    {
+        tracing::warn!(error = %err, "Peer nodes failed to apply module switch update");
+        failures.push("peer module switches");
+    }
+
+    if failures.is_empty() {
+        Ok(())
+    } else {
+        Err(s3_error!(
+            InternalError,
+            "module switches persisted but runtime convergence failed: {}",
+            failures.join("; ")
+        ))
+    }
+}
+
 pub struct GetModuleSwitchesHandler {}
 
 #[async_trait::async_trait]
@@ -156,7 +229,9 @@ pub struct UpdateModuleSwitchesHandler {}
 impl Operation for UpdateModuleSwitchesHandler {
     async fn call(&self, mut req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
         authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;
-        refresh_persisted_module_switches_from_store()
+        let context = app_context_from_req(&req).ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))?;
+        let store = context.object_store();
+        refresh_persisted_module_switches_from(store.clone())
             .await
             .map_err(|e| s3_error!(InternalError, "failed to reload persisted module switches: {}", e))?;
 
@@ -183,28 +258,7 @@ impl Operation for UpdateModuleSwitchesHandler {
             return Err(s3_error!(InvalidRequest, "{err}"));
         }
 
-        save_persisted_module_switches_to_store(switches)
-            .await
-            .map_err(|e| s3_error!(InternalError, "failed to save module switches: {}", e))?;
-
-        // Apply the new effective values immediately on this node so the console
-        // response reflects the runtime state after to write completes.
-        if refresh_notify_module_enabled() {
-            init_event_notifier().await;
-        } else {
-            shutdown_event_notifier().await;
-        }
-
-        if refresh_audit_module_enabled() {
-            match start_audit_system().await {
-                Ok(()) | Err(AuditError::AlreadyInitialized) => {}
-                Err(e) => return Err(s3_error!(InternalError, "failed to apply audit module switch: {}", e)),
-            }
-        } else {
-            stop_audit_system()
-                .await
-                .map_err(|e| s3_error!(InternalError, "failed to stop audit module after switch update: {}", e))?;
-        }
+        supervise_admin_mutation("module switch update", apply_module_switch_update(context, switches)).await?;
 
         let snapshot = current_module_switch_snapshot();
         build_response(StatusCode::OK, &ModuleSwitchesResponse::from(snapshot), req.headers.get("x-request-id"))
@@ -233,6 +287,12 @@ mod tests {
             put_block.contains("authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;"),
             "module switch PUT should require ConfigUpdateAdminAction"
         );
+        assert!(
+            put_block.contains(
+                "supervise_admin_mutation(\"module switch update\", apply_module_switch_update(context, switches)).await?;"
+            ),
+            "module switch PUT must delegate the complete mutation to its supervisor"
+        );
     }
 
     #[test]

rustfs/src/admin/handlers/notify_runtime_access.rs

@@ -12,31 +12,33 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::server::init_event_notifier;
+use crate::server::{init_event_notifier, is_event_notifier_reconciled};
 use rustfs_config::server_config::Config;
 use s3s::{S3Result, s3_error};
 use std::sync::Arc;
-use tokio::sync::Mutex;
-
-static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(());
 
 pub(crate) async fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
-    if let Some(system) = rustfs_notify::notification_system() {
+    if is_event_notifier_reconciled()
+        && let Some(system) = rustfs_notify::notification_system()
+        && system.runtime_lifecycle_is_converged()
+    {
         return Ok(system);
     }
 
-    let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await;
-    if let Some(system) = rustfs_notify::notification_system() {
-        return Ok(system);
+    init_event_notifier()
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to reconcile notification runtime: {}", err))?;
+    let system =
+        rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))?;
+    if !system.runtime_lifecycle_is_converged() {
+        return Err(s3_error!(InternalError, "latest notification lifecycle generation has not converged"));
     }
-
-    init_event_notifier().await;
-    rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
+    Ok(system)
 }
 
 pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
     let system = get_notification_system().await?;
-    let config = system.config.read().await.clone();
+    let config = system.config_snapshot().await;
     Ok((system, config))
 }
 

rustfs/src/admin/handlers/oidc.rs

@@ -19,7 +19,10 @@ use crate::admin::runtime_sources::{
     current_server_config_for_context,
 };
 use crate::admin::service::federated_identity::DefaultFederatedSessionBinding;
-use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config};
+use crate::admin::storage_api::config::{
+    read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock,
+    with_admin_server_config_write_lock,
+};
 use crate::auth::{check_key_valid, get_session_token};
 use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
 use http::StatusCode;
@@ -368,13 +371,20 @@ impl Operation for PutOidcConfigHandler {
         if is_env_managed_provider(provider_id) {
             return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
         }
+        let provider_id = provider_id.to_owned();
 
         let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?;
-        let mut config = load_server_config_from_store().await?;
-        let existing_secret = persisted_provider_secret(&config, provider_id);
-        let provider_config = build_provider_config_from_upsert(provider_id, request, existing_secret)?;
-        upsert_persisted_provider_config(&mut config, &provider_config);
-        save_server_config_to_store(&config).await?;
+        let store = oidc_config_store()?;
+        let lock_store = store.clone();
+        with_admin_server_config_write_lock(lock_store, move || async move {
+            let mut config = load_server_config_from_store_locked(store.clone()).await?;
+            let existing_secret = persisted_provider_secret(&config, &provider_id);
+            let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?;
+            upsert_persisted_provider_config(&mut config, &provider_config);
+            save_server_config_to_store_locked(store, &config).await
+        })
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
 
         json_response(
             StatusCode::OK,
@@ -403,10 +413,17 @@ impl Operation for DeleteOidcConfigHandler {
         if is_env_managed_provider(provider_id) {
             return Err(s3_error!(AccessDenied, "provider is managed by environment variables"));
         }
+        let provider_id = provider_id.to_owned();
 
-        let mut config = load_server_config_from_store().await?;
-        delete_persisted_provider_config(&mut config, provider_id)?;
-        save_server_config_to_store(&config).await?;
+        let store = oidc_config_store()?;
+        let lock_store = store.clone();
+        with_admin_server_config_write_lock(lock_store, move || async move {
+            let mut config = load_server_config_from_store_locked(store.clone()).await?;
+            delete_persisted_provider_config(&mut config, &provider_id)?;
+            save_server_config_to_store_locked(store, &config).await
+        })
+        .await
+        .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??;
 
         json_response(
             StatusCode::OK,
@@ -881,23 +898,32 @@ fn json_response<T: Serialize>(status: StatusCode, payload: &T) -> S3Result<S3Re
 }
 
 async fn load_server_config_from_store() -> S3Result<ServerConfig> {
-    let context = current_app_context();
-    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
-        return Err(s3_error!(InternalError, "storage layer not initialized"));
-    };
+    let store = oidc_config_store()?;
 
     read_admin_config_without_migrate(store)
         .await
         .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
 }
 
-async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> {
+fn oidc_config_store() -> S3Result<std::sync::Arc<crate::admin::storage_api::runtime::ECStore>> {
     let context = current_app_context();
-    let Some(store) = current_object_store_handle_for_context(context.as_deref()) else {
-        return Err(s3_error!(InternalError, "storage layer not initialized"));
-    };
+    current_object_store_handle_for_context(context.as_deref())
+        .ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))
+}
 
-    save_admin_server_config(store, config)
+async fn load_server_config_from_store_locked(
+    store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
+) -> S3Result<ServerConfig> {
+    read_admin_config_without_migrate_no_lock(store)
+        .await
+        .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}")))
+}
+
+async fn save_server_config_to_store_locked(
+    store: std::sync::Arc<crate::admin::storage_api::runtime::ECStore>,
+    config: &ServerConfig,
+) -> S3Result<()> {
+    save_admin_server_config_no_lock(store, config)
         .await
         .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}")))
 }

rustfs/src/admin/handlers/plugins_instances.rs

@@ -605,7 +605,7 @@ fn plugin_instance_mutation_block_reason(
     target_type: &str,
     target_name: &str,
     target_label: &str,
-) -> Option<String> {
+) -> S3Result<Option<String>> {
     shared_target_mutation_block_reason(context.specs, context.route_prefix, config, target_type, target_name, target_label)
 }
 
@@ -667,7 +667,7 @@ async fn collect_domain_instances(context: PluginInstanceDomainContext) -> S3Res
     let config = plugin_instance_config_snapshot(context).await?;
     let module_disabled_reason = module_disabled_block_reason(context.domain, "listing plugin instances");
     let mut entries = Vec::new();
-    for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses) {
+    for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses)? {
         entries.push(plugin_instance_list_entry(instance, module_disabled_reason.clone()));
     }
     Ok(entries)
@@ -688,13 +688,7 @@ async fn find_plugin_instance(instance_id: &str) -> S3Result<Option<TargetInstan
     let context = plugin_instance_domain_context(parse_plugin_instance_id(instance_id)?.1);
     let runtime_statuses = plugin_instance_runtime_statuses(context).await?;
     let config = plugin_instance_config_snapshot(context).await?;
-    Ok(find_target_instance(
-        context.specs,
-        context.route_prefix,
-        &config,
-        runtime_statuses,
-        instance_id,
-    ))
+    find_target_instance(context.specs, context.route_prefix, &config, runtime_statuses, instance_id)
 }
 
 async fn set_plugin_instance_config(
@@ -800,7 +794,7 @@ impl Operation for PutPluginInstanceHandler {
             resolved.target_spec.subsystem,
             &resolved.target_name,
             "plugin instance",
-        ) {
+        )? {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
 
@@ -852,7 +846,7 @@ impl Operation for DeletePluginInstanceHandler {
             resolved.target_spec.subsystem,
             &resolved.target_name,
             "plugin instance",
-        ) {
+        )? {
             return Err(s3_error!(InvalidRequest, "{reason}"));
         }
 
@@ -965,7 +959,8 @@ mod tests {
         )]));
 
         let instances =
-            collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new());
+            collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new())
+                .expect("collect target instances");
         let primary = instances
             .into_iter()
             .find(|instance| instance.account_id == "primary" && instance.service == "webhook")
@@ -988,7 +983,8 @@ mod tests {
                     NOTIFY_ROUTE_PREFIX,
                     &Config(HashMap::new()),
                     HashMap::new(),
-                );
+                )
+                .expect("collect target instances");
                 let env_only = instances
                     .into_iter()
                     .find(|instance| instance.account_id == "env-only")
@@ -1008,7 +1004,8 @@ mod tests {
             NOTIFY_ROUTE_PREFIX,
             &Config(HashMap::new()),
             runtime_statuses,
-        );
+        )
+        .expect("collect target instances");
 
         let runtime_only = instances
             .into_iter()

rustfs/src/admin/handlers/target_descriptor.rs

@@ -30,7 +30,7 @@ use rustfs_targets::{
     check_redis_server_available,
     config::{
         TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
-        build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances,
+        build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, try_normalize_target_plugin_instances,
         validate_redis_config,
     },
     manifest::builtin_target_manifest,
@@ -223,11 +223,11 @@ pub(crate) fn endpoint_source(
     config: &Config,
     target_type: &str,
     target_name: &str,
-) -> TargetEndpointSource {
-    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
+) -> S3Result<TargetEndpointSource> {
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
     let service = target_service_name(specs, target_type).unwrap_or_default();
     let key = normalized_endpoint_key(target_name, service);
-    classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)
+    Ok(classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key))
 }
 
 pub(crate) fn target_mutation_block_reason(
@@ -237,8 +237,8 @@ pub(crate) fn target_mutation_block_reason(
     target_type: &str,
     target_name: &str,
     target_label: &str,
-) -> Option<String> {
-    match endpoint_source(specs, route_prefix, config, target_type, target_name) {
+) -> S3Result<Option<String>> {
+    Ok(match endpoint_source(specs, route_prefix, config, target_type, target_name)? {
         TargetEndpointSource::Env => Some(format!(
             "{} '{}' is managed by environment variables and cannot be modified from the console",
             target_label, target_name
@@ -248,7 +248,7 @@ pub(crate) fn target_mutation_block_reason(
             target_label, target_name
         )),
         TargetEndpointSource::Config | TargetEndpointSource::Runtime => None,
-    }
+    })
 }
 
 pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, enabled: bool, action: &str) -> Option<String> {
@@ -304,10 +304,10 @@ pub(crate) fn merge_target_endpoints(
     route_prefix: &str,
     config: &Config,
     runtime_statuses: HashMap<EndpointKey, String>,
-) -> Vec<MergedTargetEndpoint> {
+) -> S3Result<Vec<MergedTargetEndpoint>> {
     let mut endpoints = Vec::new();
     let mut seen = HashSet::new();
-    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
     let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
 
     for ((account_id, service), status) in runtime_statuses {
@@ -361,7 +361,7 @@ pub(crate) fn merge_target_endpoints(
     }
 
     endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
-    endpoints
+    Ok(endpoints)
 }
 
 pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String {
@@ -373,12 +373,12 @@ pub(crate) fn collect_target_instances(
     route_prefix: &str,
     config: &Config,
     runtime_statuses: HashMap<EndpointKey, String>,
-) -> Vec<TargetInstanceReadModel> {
+) -> S3Result<Vec<TargetInstanceReadModel>> {
     let mut instances = Vec::new();
     let mut seen = HashSet::new();
     let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
     let domain = inferred_target_domain(route_prefix);
-    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
+    let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?;
 
     for ((account_id, service), status) in runtime_statuses {
         let normalized = normalized_endpoint_key(&account_id, &service);
@@ -439,7 +439,7 @@ pub(crate) fn collect_target_instances(
     }
 
     instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
-    instances
+    Ok(instances)
 }
 
 pub(crate) fn find_target_instance(
@@ -448,10 +448,10 @@ pub(crate) fn find_target_instance(
     config: &Config,
     runtime_statuses: HashMap<EndpointKey, String>,
     canonical_id: &str,
-) -> Option<TargetInstanceReadModel> {
-    collect_target_instances(specs, route_prefix, config, runtime_statuses)
+) -> S3Result<Option<TargetInstanceReadModel>> {
+    Ok(collect_target_instances(specs, route_prefix, config, runtime_statuses)?
         .into_iter()
-        .find(|instance| instance.canonical_id == canonical_id)
+        .find(|instance| instance.canonical_id == canonical_id))
 }
 
 pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
@@ -559,11 +559,11 @@ fn normalized_target_instances(
     specs: &[AdminTargetSpec],
     route_prefix: &str,
     config: &Config,
-) -> Vec<TargetPluginInstanceRecord> {
-    specs
-        .iter()
-        .flat_map(|spec| {
-            normalize_target_plugin_instances(
+) -> S3Result<Vec<TargetPluginInstanceRecord>> {
+    let mut instances = Vec::new();
+    for spec in specs {
+        instances.extend(
+            try_normalize_target_plugin_instances(
                 config,
                 &TargetPluginInstanceCompatDescriptor {
                     domain: inferred_target_domain(route_prefix),
@@ -574,8 +574,10 @@ fn normalized_target_instances(
                     valid_fields: spec.valid_keys,
                 },
             )
-        })
-        .collect()
+            .map_err(|err| s3_error!(InvalidRequest, "invalid {} target environment: {}", spec.service, err))?,
+        );
+    }
+    Ok(instances)
 }
 
 fn inferred_target_domain(route_prefix: &str) -> TargetDomain {
@@ -597,8 +599,8 @@ fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Op
     specs.iter().find(|spec| spec.service == service)
 }
 
-fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot {
-    let normalized_instances = normalized_target_instances(specs, route_prefix, config);
+fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> S3Result<TargetEndpointSnapshot> {
+    let normalized_instances = normalized_target_instances(specs, route_prefix, config)?;
     let mut configured_keys = Vec::new();
     let mut config_targets = HbHashSet::new();
     let mut env_targets = HbHashSet::new();
@@ -618,12 +620,12 @@ fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, conf
         }
     }
 
-    TargetEndpointSnapshot {
+    Ok(TargetEndpointSnapshot {
         normalized_instances,
         configured_keys,
         config_targets,
         env_targets,
-    }
+    })
 }
 
 async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>

rustfs/src/admin/router.rs

@@ -628,7 +628,7 @@ fn resolve_object_lambda_webhook_config_from_server_config(
 
 async fn load_current_server_config() -> S3Result<Config> {
     if let Some(system) = notification_system() {
-        return Ok(system.config.read().await.clone());
+        return Ok(system.config_snapshot().await);
     }
 
     if let Some(store) = current_object_store_handle() {

rustfs/src/admin/service/config.rs

@@ -19,18 +19,21 @@ use crate::admin::runtime_sources::{
 use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass};
 use crate::admin::storage_api::contract::admin::StorageAdminApi;
 use crate::admin::storage_api::runtime::ECStore;
+use crate::server::{
+    MODULE_SWITCHES_SIGNAL_SUBSYSTEM, apply_audit_module_switch_for_context, reconcile_event_notifier_from_store,
+};
 use rustfs_audit::reload_audit_config;
+use rustfs_config::AUDIT_DEFAULT_DIR;
 use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
-use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_WEBHOOK_SUB_SYS};
+use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_SUB_SYSTEMS};
 use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
 use rustfs_config::server_config::{Config as ServerConfig, KVS};
-use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR};
 use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
 use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
 use rustfs_iam::oidc::load_oidc_provider_configs_from_server_config;
 use rustfs_targets::config::{
-    validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, validate_nats_config,
-    validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
+    try_collect_target_configs, validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
+    validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
 };
 use s3s::{S3Error, S3ErrorCode, S3Result};
 use std::future::Future;
@@ -38,10 +41,11 @@ use tracing::warn;
 use url::Url;
 
 pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool {
-    matches!(
-        sub_system,
-        STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
-    )
+    NOTIFY_SUB_SYSTEMS.contains(&sub_system)
+        || matches!(
+            sub_system,
+            STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
+        )
 }
 
 pub(crate) const FULL_CONFIG_WORKER_SUBSYSTEMS: [&str; 2] = [AUDIT_WEBHOOK_SUB_SYS, SCANNER_SUB_SYS];
@@ -179,30 +183,20 @@ fn target_kvs(config: &ServerConfig, sub_system: &str, target: &str) -> KVS {
 }
 
 fn validate_notify_subsystem_config(config: &ServerConfig, sub_system: &str) -> S3Result<()> {
-    let Some(targets) = config.0.get(sub_system) else {
+    let Some(descriptor) = rustfs_notify::factory::builtin_target_descriptors()
+        .into_iter()
+        .find(|descriptor| descriptor.subsystem() == sub_system)
+    else {
         return Ok(());
     };
+    let plugin = descriptor.plugin();
 
-    for target in targets.keys() {
-        let kvs = target_kvs(config, sub_system, target);
-        if !target_enabled(&kvs) {
-            continue;
-        }
-
-        let result = match sub_system {
-            "notify_webhook" => validate_webhook_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_amqp" => validate_amqp_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_kafka" => validate_kafka_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_mqtt" => validate_mqtt_config(&kvs),
-            "notify_mysql" => validate_mysql_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_nats" => validate_nats_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_postgres" => validate_postgres_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_pulsar" => validate_pulsar_config(&kvs, EVENT_DEFAULT_DIR),
-            "notify_redis" => validate_redis_config(&kvs, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
-            _ => return Ok(()),
-        };
-
-        result.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
+    for (target, kvs) in try_collect_target_configs(config, NOTIFY_ROUTE_PREFIX, plugin.target_type(), plugin.valid_fields_set())
+        .map_err(|err| invalid_request(format!("invalid {sub_system} config: {err}")))?
+    {
+        plugin
+            .validate_config(&kvs)
+            .map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?;
     }
 
     Ok(())
@@ -314,8 +308,7 @@ pub(crate) async fn prepare_server_config_for_context(
         Some(STORAGE_CLASS_SUB_SYS) => {
             prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
         }
-        Some(NOTIFY_WEBHOOK_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?,
-        Some(NOTIFY_MQTT_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?,
+        Some(sub_system) if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => validate_notify_subsystem_config(config, sub_system)?,
         Some(AUDIT_WEBHOOK_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?,
         Some(AUDIT_MQTT_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?,
         Some(IDENTITY_OPENID_SUB_SYS) => validate_identity_openid_config(config)?,
@@ -324,8 +317,9 @@ pub(crate) async fn prepare_server_config_for_context(
         Some(_) => {}
         None => {
             prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?);
-            validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?;
-            validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?;
+            for sub_system in NOTIFY_SUB_SYSTEMS {
+                validate_notify_subsystem_config(config, sub_system)?;
+            }
             validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?;
             validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?;
             validate_identity_openid_config(config)?;
@@ -371,6 +365,18 @@ pub async fn apply_dynamic_config_for_subsystem_for_context(
         AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS => reload_audit_config(config.clone())
             .await
             .map_err(|err| internal_error(format!("failed to reload audit config: {err}")))?,
+        sub_system if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => {
+            // The notify lifecycle is intentionally process-wide. The explicit
+            // store keeps the persisted snapshot bound to the request context;
+            // the ECStore NotificationSys resolved below is the peer-broadcast
+            // service, not the notify target runtime.
+            let system = rustfs_notify::ensure_live_events();
+            let store = resolve_runtime_config_store_for_context(context)?;
+            system
+                .reload_persisted_config_from_store(store)
+                .await
+                .map_err(|err| internal_error(format!("failed to reload notification config: {err}")))?;
+        }
         SCANNER_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
             .map_err(|err| internal_error(format!("failed to reload scanner config: {err}")))?,
         HEAL_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config)
@@ -387,6 +393,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste
 }
 
 pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
+    if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM {
+        let store = resolve_runtime_config_store_for_context(context)?;
+        let notify_result = reconcile_event_notifier_from_store(store).await;
+        let audit_result = apply_audit_module_switch_for_context(context).await;
+        return match (notify_result, audit_result) {
+            (Ok(()), Ok(())) => Ok(()),
+            (Err(_), Ok(())) => Err(internal_error("failed to reconcile notification module switch")),
+            (Ok(()), Err(_)) => Err(internal_error("failed to reconcile audit module switch")),
+            (Err(_), Err(_)) => Err(internal_error("failed to reconcile notification and audit module switches")),
+        };
+    }
+
     if !is_dynamic_config_subsystem(sub_system) {
         return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
     }
@@ -398,9 +416,8 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap
     })?;
     apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
         .await
-        .map_err(|err| {
-            warn!("peer reload_dynamic_config: failed to apply {sub_system}: {err}");
-            err
+        .inspect_err(|_| {
+            warn!(config_subsystem = sub_system, reason = "apply_failed", "Peer dynamic config apply failed");
         })?;
     Ok(())
 }
@@ -428,19 +445,19 @@ where
     let (config, prepared) = prepare(config).await?;
     publish(&config, prepared)?;
 
-    // Worker reloads mutate live state and have no rollback contract. They are
-    // therefore best-effort after the validated storage/server snapshots are
-    // published; a transient worker failure must not leave this peer on stale
-    // erasure geometry.
+    // Worker reloads mutate live state and have no rollback contract, so the
+    // validated snapshots stay published. The RPC still reports convergence
+    // failure explicitly so the originating Admin request cannot claim success.
     if let Err(err) = apply_workers(config).await {
         warn!(
             event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
             component = LOG_COMPONENT_ADMIN,
             subsystem = LOG_SUBSYSTEM_CONFIG,
             state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-            error = ?err,
+            reason = "apply_failed",
             "Runtime config snapshot was published but a worker reload failed"
         );
+        return Err(err);
     }
     Ok(())
 }
@@ -468,20 +485,29 @@ pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppCont
             Ok(())
         },
         |config| async move {
+            let mut failures = Vec::new();
             for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS {
-                if let Err(err) = apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system).await {
+                if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system)
+                    .await
+                    .is_err()
+                {
+                    failures.push(sub_system);
                     warn!(
                         event = EVENT_CONFIG_WORKER_RELOAD_FAILED,
                         component = LOG_COMPONENT_ADMIN,
                         subsystem = LOG_SUBSYSTEM_CONFIG,
                         config_subsystem = sub_system,
                         state = CONFIG_WORKER_RELOAD_FAILURE_STATE,
-                        error = ?err,
+                        reason = "apply_failed",
                         "Peer runtime config snapshot was published but a subsystem worker reload failed"
                     );
                 }
             }
-            Ok(())
+            if failures.is_empty() {
+                Ok(())
+            } else {
+                Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; "))))
+            }
         },
     )
     .await
@@ -492,20 +518,77 @@ pub async fn reload_runtime_config_snapshot() -> S3Result<()> {
     reload_runtime_config_snapshot_for_context(context.as_deref()).await
 }
 
-pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
-    if !is_dynamic_config_subsystem(sub_system) {
-        return;
+pub async fn preflight_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
+    if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
+        return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
     }
 
     let Some(notification_sys) = current_notification_system_for_context(context) else {
-        return;
+        return Ok(());
     };
 
-    for failure in notification_sys.reload_dynamic_config(sub_system).await {
-        if let Some(err) = failure.err {
-            tracing::warn!("peer {} dynamic config reload for {} failed: {}", failure.host, sub_system, err);
+    let mut failed = 0usize;
+    for failure in notification_sys.preflight_dynamic_config(sub_system).await {
+        if failure.err.is_some() {
+            failed += 1;
+            let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
+            warn!(
+                peer = host,
+                config_subsystem = sub_system,
+                reason = "unsupported",
+                "Peer does not support dynamic config convergence"
+            );
         }
     }
+
+    if failed == 0 {
+        Ok(())
+    } else {
+        Err(internal_error(format!(
+            "{failed} peer(s) do not support dynamic config convergence for {sub_system}"
+        )))
+    }
+}
+
+pub async fn preflight_dynamic_config_reload(sub_system: &str) -> S3Result<()> {
+    let context = current_app_context();
+    preflight_dynamic_config_reload_for_context(context.as_deref(), sub_system).await
+}
+
+pub async fn signal_dynamic_config_reload_checked_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> {
+    if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) {
+        return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}")));
+    }
+
+    let Some(notification_sys) = current_notification_system_for_context(context) else {
+        return Ok(());
+    };
+
+    let mut failed = 0usize;
+    for failure in notification_sys.reload_dynamic_config(sub_system).await {
+        if failure.err.is_some() {
+            failed += 1;
+            let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
+            warn!(
+                peer = host,
+                config_subsystem = sub_system,
+                reason = "reload_failed",
+                "Peer dynamic config convergence failed"
+            );
+        }
+    }
+
+    if failed == 0 {
+        Ok(())
+    } else {
+        Err(internal_error(format!("{failed} peer(s) failed dynamic config reload for {sub_system}")))
+    }
+}
+
+pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) {
+    if let Err(err) = signal_dynamic_config_reload_checked_for_context(context, sub_system).await {
+        tracing::warn!("peer dynamic config reload for {} failed: {}", sub_system, err);
+    }
 }
 
 pub async fn signal_dynamic_config_reload(sub_system: &str) {
@@ -513,16 +596,36 @@ pub async fn signal_dynamic_config_reload(sub_system: &str) {
     signal_dynamic_config_reload_for_context(context.as_deref(), sub_system).await;
 }
 
-pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
+pub async fn signal_dynamic_config_reload_checked(sub_system: &str) -> S3Result<()> {
+    let context = current_app_context();
+    signal_dynamic_config_reload_checked_for_context(context.as_deref(), sub_system).await
+}
+
+pub async fn signal_config_snapshot_reload_checked_for_context(context: Option<&AppContext>) -> S3Result<()> {
     let Some(notification_sys) = current_notification_system_for_context(context) else {
-        return;
+        return Ok(());
     };
 
+    let mut failed = 0usize;
     for failure in notification_sys.refresh_config_snapshot().await {
-        if let Some(err) = failure.err {
-            tracing::warn!("peer config snapshot refresh failed for {}: {}", failure.host, err);
+        if failure.err.is_some() {
+            failed += 1;
+            let host = if failure.host.is_empty() { "<unknown>" } else { &failure.host };
+            warn!(peer = host, reason = "reload_failed", "Peer config snapshot refresh failed");
         }
     }
+
+    if failed == 0 {
+        Ok(())
+    } else {
+        Err(internal_error(format!("{failed} peer(s) failed config snapshot reload")))
+    }
+}
+
+pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) {
+    if let Err(err) = signal_config_snapshot_reload_checked_for_context(context).await {
+        tracing::warn!("peer config snapshot refresh failed: {err}");
+    }
 }
 
 pub async fn signal_config_snapshot_reload() {
@@ -530,30 +633,46 @@ pub async fn signal_config_snapshot_reload() {
     signal_config_snapshot_reload_for_context(context.as_deref()).await;
 }
 
+pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> {
+    let context = current_app_context();
+    signal_config_snapshot_reload_checked_for_context(context.as_deref()).await
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface};
     use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
-    use crate::admin::storage_api::config::save_admin_server_config;
+    use crate::admin::storage_api::config::{
+        read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config,
+        save_admin_server_config_no_lock, with_admin_server_config_write_lock,
+    };
+    use crate::admin::storage_api::error::StorageError;
+    use crate::server::{
+        ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled,
+        refresh_persisted_module_switches_from, save_persisted_module_switches_to,
+    };
     use crate::storage_api::cluster::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx};
-    use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
+    use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS};
     use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
     use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
     use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR};
     use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
     use rustfs_kms::KmsServiceManager;
     use std::collections::HashMap;
+    use std::future::{Future, poll_fn};
     use std::path::Path;
     use std::sync::{
         Arc, Mutex,
         atomic::{AtomicUsize, Ordering},
     };
+    use std::task::Poll;
     use tempfile::TempDir;
     use tokio_util::sync::CancellationToken;
 
     const LIFECYCLE_RELOAD_LABEL: &str = "lifecycle";
+    const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
     const REPLICATION_RELOAD_LABEL: &str = "replication";
 
     fn without_storage_class_env<R>(f: impl FnOnce() -> R) -> R {
@@ -748,6 +867,139 @@ mod tests {
         }
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    async fn concurrent_real_store_config_rmw_preserves_notify_and_oidc_updates() {
+        let temp_dir = TempDir::new().expect("server config RMW temp dir");
+        let store = build_isolated_heterogeneous_store(temp_dir.path()).await;
+        save_admin_server_config(store.clone(), &ServerConfig::new())
+            .await
+            .expect("persist baseline server config");
+
+        let notify_entered = Arc::new(tokio::sync::Notify::new());
+        let release_notify = Arc::new(tokio::sync::Notify::new());
+        let (oidc_polled_tx, oidc_polled_rx) = tokio::sync::oneshot::channel();
+        let (oidc_entered_tx, mut oidc_entered_rx) = tokio::sync::oneshot::channel();
+        let transaction_order = Arc::new(Mutex::new(Vec::new()));
+
+        let notify_task = {
+            let store = store.clone();
+            let notify_entered = notify_entered.clone();
+            let release_notify = release_notify.clone();
+            let transaction_order = transaction_order.clone();
+            tokio::spawn(async move {
+                let transaction_store = store.clone();
+                with_admin_server_config_write_lock(store, move || async move {
+                    let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
+                    transaction_order.lock().expect("transaction order lock").push("notify-read");
+                    notify_entered.notify_one();
+                    release_notify.notified().await;
+
+                    let mut target = KVS::new();
+                    target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
+                    target.insert(WEBHOOK_ENDPOINT.to_string(), "https://notify.example.test/hook".to_string());
+                    config
+                        .0
+                        .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+                        .or_default()
+                        .insert("concurrent-notify".to_string(), target);
+                    save_admin_server_config_no_lock(transaction_store, &config).await?;
+                    transaction_order.lock().expect("transaction order lock").push("notify-save");
+                    Ok::<(), StorageError>(())
+                })
+                .await
+                .expect("notify transaction should acquire config locks")
+                .expect("notify transaction should persist");
+            })
+        };
+
+        tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, notify_entered.notified())
+            .await
+            .expect("notify transaction should enter");
+
+        let oidc_task = {
+            let store = store.clone();
+            let transaction_order = transaction_order.clone();
+            tokio::spawn(async move {
+                let transaction_store = store.clone();
+                let transaction = with_admin_server_config_write_lock(store, move || async move {
+                    let _ = oidc_entered_tx.send(());
+                    let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?;
+                    transaction_order.lock().expect("transaction order lock").push("oidc-read");
+
+                    let mut provider = KVS::new();
+                    provider.insert(
+                        OIDC_CONFIG_URL.to_string(),
+                        "https://identity.example.test/.well-known/openid-configuration".to_string(),
+                    );
+                    provider.insert(OIDC_CLIENT_ID.to_string(), "console".to_string());
+                    config
+                        .0
+                        .entry(IDENTITY_OPENID_SUB_SYS.to_string())
+                        .or_default()
+                        .insert("concurrent-oidc".to_string(), provider);
+                    save_admin_server_config_no_lock(transaction_store, &config).await?;
+                    transaction_order.lock().expect("transaction order lock").push("oidc-save");
+                    Ok::<(), StorageError>(())
+                });
+                tokio::pin!(transaction);
+                let mut oidc_polled_tx = Some(oidc_polled_tx);
+                poll_fn(|cx| match transaction.as_mut().poll(cx) {
+                    Poll::Pending => {
+                        if let Some(tx) = oidc_polled_tx.take() {
+                            let _ = tx.send(());
+                        }
+                        Poll::Ready(())
+                    }
+                    Poll::Ready(_) => panic!("OIDC transaction entered while notify held the config lock"),
+                })
+                .await;
+                transaction
+                    .await
+                    .expect("OIDC transaction should acquire config locks")
+                    .expect("OIDC transaction should persist");
+            })
+        };
+
+        tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, oidc_polled_rx)
+            .await
+            .expect("OIDC transaction should be polled")
+            .expect("OIDC transaction poll signal should be delivered");
+        assert!(
+            matches!(oidc_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
+            "OIDC RMW must not enter while the notify RMW owns the server-config transaction"
+        );
+
+        release_notify.notify_one();
+        tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async {
+            notify_task.await.expect("notify task should not panic");
+            oidc_task.await.expect("OIDC task should not panic");
+        })
+        .await
+        .expect("both server-config transactions should finish");
+
+        let persisted = read_admin_config_without_migrate(store)
+            .await
+            .expect("read final server config");
+        assert_eq!(
+            persisted
+                .get_value(NOTIFY_WEBHOOK_SUB_SYS, "concurrent-notify")
+                .expect("notify update should survive")
+                .get(WEBHOOK_ENDPOINT),
+            "https://notify.example.test/hook"
+        );
+        assert_eq!(
+            persisted
+                .get_value(IDENTITY_OPENID_SUB_SYS, "concurrent-oidc")
+                .expect("OIDC update should survive")
+                .get(OIDC_CLIENT_ID),
+            "console"
+        );
+        assert_eq!(
+            *transaction_order.lock().expect("transaction order lock"),
+            vec!["notify-read", "notify-save", "oidc-read", "oidc-save"]
+        );
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn peer_dynamic_reload_rejects_later_pool_without_publishing() {
@@ -811,6 +1063,67 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn peer_module_switch_reload_uses_its_explicit_context_store() {
+        temp_env::async_with_vars(
+            [
+                (rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>),
+                (rustfs_config::ENV_AUDIT_ENABLE, None::<&str>),
+            ],
+            async {
+                let selected = runtime_config_reload_fixture().await;
+                let fallback = runtime_config_reload_fixture().await;
+                save_persisted_module_switches_to(
+                    selected.context.object_store(),
+                    PersistedModuleSwitches {
+                        notify_enabled: true,
+                        audit_enabled: true,
+                    },
+                    || (),
+                )
+                .await
+                .expect("persist selected module switches");
+
+                refresh_persisted_module_switches_from(fallback.context.object_store())
+                    .await
+                    .expect("refresh absent fallback module switches");
+                assert!(
+                    !current_module_switch_snapshot().notify_enabled,
+                    "fallback store should resolve the default"
+                );
+
+                reload_dynamic_config_runtime_state_for_context(Some(&selected.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
+                    .await
+                    .expect("peer module switch reload should converge");
+
+                let selected_snapshot = current_module_switch_snapshot();
+                assert!(selected_snapshot.notify_enabled);
+                assert!(selected_snapshot.persisted_audit_enabled);
+                assert_eq!(selected_snapshot.notify_source, ModuleSwitchSource::Console);
+                assert!(matches!(
+                    rustfs_notify::notification_system()
+                        .expect("notification singleton should exist")
+                        .runtime_lifecycle_state(),
+                    rustfs_notify::NotificationRuntimeState::TargetsEnabled { .. }
+                ));
+                assert!(is_event_notifier_reconciled());
+
+                reload_dynamic_config_runtime_state_for_context(Some(&fallback.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM)
+                    .await
+                    .expect("restore default module switch state");
+                assert!(!current_module_switch_snapshot().notify_enabled);
+                assert_eq!(
+                    rustfs_notify::notification_system()
+                        .expect("notification singleton should remain stable")
+                        .runtime_lifecycle_state(),
+                    rustfs_notify::NotificationRuntimeState::LiveOnly
+                );
+            },
+        )
+        .await;
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn peer_full_reload_rejects_later_pool_without_publishing() {
@@ -835,12 +1148,12 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn full_reload_publishes_snapshots_before_best_effort_worker_failure() {
+    async fn full_reload_publishes_snapshots_before_reporting_worker_failure() {
         let events = Arc::new(Mutex::new(Vec::new()));
         let publish_events = events.clone();
         let worker_events = events.clone();
 
-        reload_runtime_config_snapshot_with(
+        let err = reload_runtime_config_snapshot_with(
             async { Ok(ServerConfig::new()) },
             |config| async { Ok((config, PreparedRuntimeConfig::default())) },
             move |_config, _prepared| {
@@ -855,8 +1168,9 @@ mod tests {
             },
         )
         .await
-        .expect("worker failure must not roll back validated storage/server snapshots");
+        .expect_err("worker failure must be reported after publishing validated snapshots");
 
+        assert_eq!(err.message(), Some("injected worker reload failure"));
         assert_eq!(
             *events.lock().expect("reload result lock"),
             ["publish", "worker-1-applied", "worker-2-failed"]
@@ -871,7 +1185,9 @@ mod tests {
         assert!(is_dynamic_config_subsystem(HEAL_SUB_SYS));
         assert!(is_dynamic_config_subsystem(STORAGE_CLASS_SUB_SYS));
         assert!(!is_dynamic_config_subsystem("identity_openid"));
-        assert!(!is_dynamic_config_subsystem("notify_webhook"));
+        for sub_system in NOTIFY_SUB_SYSTEMS {
+            assert!(is_dynamic_config_subsystem(sub_system));
+        }
     }
 
     #[test]
@@ -885,6 +1201,7 @@ mod tests {
             STORAGE_CLASS_SUB_SYS,
             AUDIT_WEBHOOK_SUB_SYS,
             AUDIT_MQTT_SUB_SYS,
+            NOTIFY_WEBHOOK_SUB_SYS,
             SCANNER_SUB_SYS,
             HEAL_SUB_SYS,
         ] {
@@ -1014,17 +1331,52 @@ mod tests {
     }
 
     #[test]
+    #[serial_test::serial(notify_config_env)]
     fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
         crate::admin::storage_api::config::init_admin_config_defaults();
         let mut config = ServerConfig::new();
         let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
-        let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
+        let kvs = targets.entry("primary".to_string()).or_default();
         kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
         kvs.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
         kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
 
-        let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
-        assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+        temp_env::with_vars_unset(
+            [
+                format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"),
+                format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"),
+            ],
+            || {
+                let err =
+                    validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail");
+                assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+            },
+        );
+    }
+
+    #[test]
+    #[serial_test::serial(notify_config_env)]
+    fn validate_notify_subsystem_config_uses_enabled_env_overlay() {
+        crate::admin::storage_api::config::init_admin_config_defaults();
+        let mut config = ServerConfig::new();
+        let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
+        let kvs = targets.entry("primary".to_string()).or_default();
+        kvs.insert(ENABLE_KEY.to_string(), EnableState::Off.to_string());
+        kvs.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/hook".to_string());
+        kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string());
+
+        temp_env::with_vars(
+            [
+                (format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"), Some("on")),
+                (format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"), Some("not-a-url")),
+            ],
+            || {
+                let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS)
+                    .expect_err("invalid environment endpoint should fail");
+                assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
+                assert!(err.message().is_some_and(|message| message.contains("target 'primary'")));
+            },
+        );
     }
 
     #[test]

rustfs/src/admin/storage_api.rs

@@ -412,6 +412,10 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc<ECStore>) -> Resu
     ecstore_config::com::read_config_without_migrate(api).await
 }
 
+pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
+    ecstore_config::com::read_config_without_migrate_no_lock(api).await
+}
+
 pub(crate) async fn save_admin_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
     ecstore_config::com::save_config(api, file, data).await
 }
@@ -420,10 +424,27 @@ pub(crate) async fn delete_admin_config(api: Arc<ECStore>, file: &str) -> Result
     ecstore_config::com::delete_config(api, file).await
 }
 
+#[cfg(test)]
 pub(crate) async fn save_admin_server_config(api: Arc<ECStore>, cfg: &rustfs_config::server_config::Config) -> Result<()> {
     ecstore_config::com::save_server_config(api, cfg).await
 }
 
+pub(crate) async fn save_admin_server_config_no_lock(
+    api: Arc<ECStore>,
+    cfg: &rustfs_config::server_config::Config,
+) -> Result<()> {
+    ecstore_config::com::save_server_config_no_lock(api, cfg).await
+}
+
+pub(crate) async fn with_admin_server_config_write_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    ecstore_config::com::with_server_config_write_lock(api, operation).await
+}
+
 pub(crate) fn init_admin_config_defaults() {
     ecstore_config::init();
 }
@@ -521,10 +542,13 @@ pub(crate) mod cluster {
 }
 
 pub(crate) mod config {
+    #[cfg(test)]
+    pub(crate) use super::save_admin_server_config;
     pub(crate) use super::storageclass;
     pub(crate) use super::{
         RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config,
-        read_admin_config_without_migrate, save_admin_config, save_admin_server_config,
+        read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config,
+        save_admin_server_config_no_lock, with_admin_server_config_write_lock,
     };
 }
 

rustfs/src/embedded.rs

@@ -52,7 +52,10 @@ use crate::server::ShutdownHandle;
 use crate::startup_embedded::{EmbeddedStartedServer, EmbeddedStartupArgs, EmbeddedStartupError, run_embedded_startup};
 use crate::startup_lifecycle::embedded_endpoint_address;
 use crate::startup_server::find_embedded_available_port;
-use crate::startup_shutdown::{run_embedded_server_drop_cleanup, run_embedded_server_shutdown};
+use crate::startup_shutdown::{
+    EmbeddedRuntimeOwner, register_embedded_runtime_owner, run_embedded_server_drop_cleanup, run_embedded_server_shutdown,
+    run_embedded_shutdown_cleanup,
+};
 use std::net::SocketAddr;
 use std::path::PathBuf;
 use tokio_util::sync::CancellationToken;
@@ -215,14 +218,21 @@ impl RustFSServerBuilder {
     }
 
     async fn do_build(self) -> Result<RustFSServer, ServerError> {
-        let started = run_embedded_startup(self.startup_args).await?;
-
-        Ok(started.into())
+        let mut runtime_owner = register_embedded_runtime_owner().await;
+        match run_embedded_startup(self.startup_args).await {
+            Ok(started) => Ok(RustFSServer::from_started(started, runtime_owner)),
+            Err(err) => {
+                if let Some(cleanup) = runtime_owner.release() {
+                    run_embedded_shutdown_cleanup(cleanup).await;
+                }
+                Err(err.into())
+            }
+        }
     }
 }
 
-impl From<EmbeddedStartedServer> for RustFSServer {
-    fn from(started: EmbeddedStartedServer) -> Self {
+impl RustFSServer {
+    fn from_started(started: EmbeddedStartedServer, runtime_owner: EmbeddedRuntimeOwner) -> Self {
         Self {
             address: started.bound_addr,
             access_key: started.access_key,
@@ -231,6 +241,7 @@ impl From<EmbeddedStartedServer> for RustFSServer {
             shutdown_handle: Some(started.shutdown_handle),
             cancel_token: started.cancel_token,
             temp_dir: started.temp_dir,
+            runtime_owner: Some(runtime_owner),
         }
     }
 }
@@ -250,6 +261,7 @@ pub struct RustFSServer {
     shutdown_handle: Option<ShutdownHandle>,
     cancel_token: CancellationToken,
     temp_dir: Option<PathBuf>,
+    runtime_owner: Option<EmbeddedRuntimeOwner>,
 }
 
 impl RustFSServer {
@@ -290,13 +302,34 @@ impl RustFSServer {
     }
 
     async fn do_shutdown(&mut self) {
-        run_embedded_server_shutdown(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref()).await;
+        let Some(runtime_owner) = self.runtime_owner.take() else {
+            return;
+        };
+        let runtime = runtime_owner.cleanup_runtime_handle();
+        run_embedded_server_shutdown(
+            &runtime,
+            &self.cancel_token,
+            &mut self.shutdown_handle,
+            &mut self.temp_dir,
+            Some(runtime_owner),
+        )
+        .await;
     }
 }
 
 impl Drop for RustFSServer {
     fn drop(&mut self) {
-        run_embedded_server_drop_cleanup(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref());
+        let Some(runtime_owner) = self.runtime_owner.take() else {
+            return;
+        };
+        let runtime = runtime_owner.cleanup_runtime_handle();
+        run_embedded_server_drop_cleanup(
+            &runtime,
+            &self.cancel_token,
+            &mut self.shutdown_handle,
+            &mut self.temp_dir,
+            Some(runtime_owner),
+        );
     }
 }
 

rustfs/src/server/audit.rs

@@ -12,7 +12,11 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::{module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from_store, runtime_sources};
+use super::{
+    module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from,
+    refresh_persisted_module_switches_from_store, runtime_sources,
+};
+use crate::runtime_sources::AppContext;
 use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState};
 use std::sync::atomic::{AtomicBool, Ordering};
 use tracing::{info, warn};
@@ -23,6 +27,13 @@ fn server_config_from_context() -> Option<rustfs_config::server_config::Config>
     runtime_sources::current_server_config()
 }
 
+fn server_config_for_context(context: Option<&AppContext>) -> Option<rustfs_config::server_config::Config> {
+    match context {
+        Some(context) => context.server_config().get(),
+        None => server_config_from_context(),
+    }
+}
+
 pub fn refresh_audit_module_enabled() -> bool {
     let enabled = resolve_audit_module_state().enabled;
     AUDIT_MODULE_ENABLED.store(enabled, Ordering::Relaxed);
@@ -51,7 +62,15 @@ fn has_any_audit_targets(config: &rustfs_config::server_config::Config) -> bool
 /// If not configured, it skips the initialization.
 /// It also handles cases where the audit system is already running or if the global configuration is not loaded.
 pub async fn start_audit_system() -> AuditResult<()> {
-    if let Err(err) = refresh_persisted_module_switches_from_store().await {
+    start_audit_system_for_context(None).await
+}
+
+pub(crate) async fn start_audit_system_for_context(context: Option<&AppContext>) -> AuditResult<()> {
+    let refresh_result = match context {
+        Some(context) => refresh_persisted_module_switches_from(context.object_store()).await,
+        None => refresh_persisted_module_switches_from_store().await,
+    };
+    if let Err(err) = refresh_result {
         warn!("Failed to refresh persisted audit module switch from store: {}", err);
     }
 
@@ -70,7 +89,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
     );
 
     // 1. Get the global configuration loaded by ecstore
-    let server_config = match server_config_from_context() {
+    let server_config = match server_config_for_context(context) {
         Some(config) => config,
         None => {
             warn!(
@@ -165,3 +184,14 @@ pub async fn stop_audit_system() -> AuditResult<()> {
         Ok(())
     }
 }
+
+pub(crate) async fn apply_audit_module_switch_for_context(context: Option<&AppContext>) -> AuditResult<()> {
+    if refresh_audit_module_enabled() {
+        match start_audit_system_for_context(context).await {
+            Ok(()) | Err(AuditError::AlreadyInitialized) => Ok(()),
+            Err(err) => Err(err),
+        }
+    } else {
+        stop_audit_system().await
+    }
+}

rustfs/src/server/event.rs

@@ -12,20 +12,48 @@
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
 
-use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store, runtime_sources};
-use crate::storage_api::server::event::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook};
+use super::{
+    module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
+    refresh_persisted_module_switches_from_store, runtime_sources,
+};
+use crate::storage_api::server::event::{
+    EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
+    with_server_config_read_lock,
+};
 use chrono::{DateTime, Utc};
-use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo};
+use rustfs_notify::{
+    EventArgs as NotifyEventArgs, NotificationError, NotificationRuntimeState, NotificationSystem, NotifyObjectInfo,
+};
 use rustfs_s3_types::EventName;
+use std::future::Future;
 use std::net::SocketAddr;
+use std::sync::OnceLock;
 use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
 use tokio::spawn;
-use tracing::{error, info, instrument, warn};
+use tokio::task::JoinHandle;
+use tokio::time::{Instant, MissedTickBehavior};
+use tokio_util::sync::CancellationToken;
+use tracing::{info, instrument, warn};
 
 static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
+static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false);
+static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new();
 
-fn server_config_from_context() -> Option<rustfs_config::server_config::Config> {
-    runtime_sources::current_server_config()
+const EVENT_NOTIFIER_RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
+const EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120);
+const EVENT_NOTIFY_RUNTIME_RECONCILE: &str = "notify_runtime_reconcile";
+
+pub(crate) fn is_event_notifier_reconciled() -> bool {
+    NOTIFY_RUNTIME_RECONCILED.load(Ordering::Acquire)
+}
+
+pub(crate) fn mark_event_notifier_reconciled() {
+    NOTIFY_RUNTIME_RECONCILED.store(true, Ordering::Release);
+}
+
+pub(crate) fn mark_event_notifier_unreconciled() {
+    NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release);
 }
 
 pub fn refresh_notify_module_enabled() -> bool {
@@ -112,79 +140,201 @@ fn parse_host_and_port(host: String) -> (String, u16) {
 }
 
 fn install_ecstore_event_dispatch_hook() {
-    let installed = register_event_dispatch_hook(|args| {
-        let Some(notify_args) = convert_ecstore_event_args(args) else {
-            return;
-        };
-        spawn(async move {
-            runtime_sources::current_notify_interface().notify(notify_args).await;
+    ECSTORE_EVENT_DISPATCH_HOOK.get_or_init(|| {
+        let installed = register_event_dispatch_hook(|args| {
+            let Some(notify_args) = convert_ecstore_event_args(args) else {
+                return;
+            };
+            spawn(async move {
+                runtime_sources::current_notify_interface().notify(notify_args).await;
+            });
         });
+
+        if !installed {
+            warn!("ECStore event dispatch hook was already registered");
+        }
     });
+}
 
-    if !installed {
-        warn!("ECStore event dispatch hook was already registered");
+fn ensure_live_events_initialized() -> std::sync::Arc<NotificationSystem> {
+    let system = rustfs_notify::ensure_live_events();
+    install_ecstore_event_dispatch_hook();
+    system
+}
+
+fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> {
+    if system.runtime_lifecycle_is_converged() {
+        Ok(())
+    } else {
+        Err(NotificationError::Initialization(
+            "Latest notification lifecycle generation has not converged".to_string(),
+        ))
     }
 }
 
-fn ensure_live_events_initialized() -> bool {
-    if rustfs_notify::notification_system().is_some() {
-        return true;
-    }
+pub(crate) async fn reconcile_event_notifier_from_store(
+    store: std::sync::Arc<rustfs_notify::NotifyStore>,
+) -> Result<(), NotificationError> {
+    let result = async {
+        validate_notify_module_env().map_err(NotificationError::Initialization)?;
+        let system = ensure_live_events_initialized();
+        let transition_system = system.clone();
+        let transition_store = store.clone();
+        let transition = with_refreshed_notify_module_state_from(store, move |resolution| async move {
+            NOTIFY_MODULE_ENABLED.store(resolution.enabled, Ordering::Relaxed);
+            let read_store = transition_store.clone();
+            let config_system = transition_system.clone();
+            with_server_config_read_lock(transition_store, move || async move {
+                let config = read_existing_server_config_no_lock(read_store)
+                    .await
+                    .map_err(|err| NotificationError::ReadConfig(err.to_string()))?;
+                let mode_matches = match config_system.runtime_lifecycle_state() {
+                    NotificationRuntimeState::LiveOnly => !resolution.enabled,
+                    NotificationRuntimeState::TargetsEnabled { .. } => resolution.enabled,
+                    NotificationRuntimeState::Terminated => false,
+                };
+                if config_system.config_snapshot().await == config
+                    && mode_matches
+                    && config_system.runtime_lifecycle_is_converged()
+                {
+                    Ok::<_, NotificationError>(None)
+                } else {
+                    Ok(Some(config_system.publish_targets_enabled(resolution.enabled, Some(config))))
+                }
+            })
+            .await
+            .map_err(|err| NotificationError::StorageNotAvailable(err.to_string()))?
+        })
+        .await
+        .map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??;
 
-    match rustfs_notify::initialize_live_events() {
-        Ok(()) => {
-            install_ecstore_event_dispatch_hook();
-            true
+        if let Some(transition) = transition {
+            transition.wait().await?;
         }
-        Err(e) => {
-            error!("Failed to initialize live event stream support: {}", e);
-            false
+
+        ensure_event_notifier_converged(&system)
+    }
+    .await;
+
+    if result.is_ok() {
+        mark_event_notifier_reconciled();
+    } else {
+        mark_event_notifier_unreconciled();
+    }
+    result
+}
+
+pub(crate) fn start_persisted_event_notifier_reconciler(
+    store: std::sync::Arc<rustfs_notify::NotifyStore>,
+    cancellation: CancellationToken,
+) -> JoinHandle<()> {
+    spawn(run_persisted_event_notifier_reconciler(
+        cancellation,
+        EVENT_NOTIFIER_RECONCILE_INTERVAL,
+        move || {
+            let store = store.clone();
+            async move { reconcile_event_notifier_from_store(store).await }
+        },
+    ))
+}
+
+async fn run_persisted_event_notifier_reconciler<Reconcile, ReconcileFuture>(
+    cancellation: CancellationToken,
+    reconcile_interval: Duration,
+    mut reconcile: Reconcile,
+) where
+    Reconcile: FnMut() -> ReconcileFuture,
+    ReconcileFuture: Future<Output = Result<(), NotificationError>>,
+{
+    let first_tick = Instant::now() + reconcile_interval;
+    let mut ticker = tokio::time::interval_at(first_tick, reconcile_interval);
+    ticker.set_missed_tick_behavior(MissedTickBehavior::Skip);
+    let mut failure_reported = false;
+
+    loop {
+        tokio::select! {
+            biased;
+            _ = cancellation.cancelled() => break,
+            _ = ticker.tick() => {}
+        }
+
+        let result = tokio::select! {
+            biased;
+            _ = cancellation.cancelled() => break,
+            result = tokio::time::timeout(EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT, reconcile()) => result,
+        };
+
+        match result {
+            Ok(Ok(())) => {
+                if failure_reported {
+                    info!(
+                        event = EVENT_NOTIFY_RUNTIME_RECONCILE,
+                        component = "notify",
+                        subsystem = "lifecycle",
+                        state = "recovered",
+                        "Persisted notification runtime reconciliation recovered"
+                    );
+                }
+                failure_reported = false;
+            }
+            Ok(Err(_)) | Err(_) => {
+                if !failure_reported {
+                    warn!(
+                        event = EVENT_NOTIFY_RUNTIME_RECONCILE,
+                        component = "notify",
+                        subsystem = "lifecycle",
+                        state = "degraded",
+                        reason = "reconcile_failed_or_timed_out",
+                        "Persisted notification runtime reconciliation failed"
+                    );
+                }
+                failure_reported = true;
+            }
         }
     }
 }
 
-/// Shuts down the event notifier system gracefully
-pub async fn shutdown_event_notifier() {
+/// Irreversibly shuts down the event notifier target runtime for process exit.
+pub async fn shutdown_event_notifier() -> Result<(), NotificationError> {
     info!("Shutting down event notifier system...");
-
-    if !rustfs_notify::is_notification_system_initialized() {
+    let Some(system) = rustfs_notify::notification_system() else {
         info!("Event notifier system is not initialized, nothing to shut down.");
-        return;
-    }
-
-    let system = match rustfs_notify::notification_system() {
-        Some(sys) => sys,
-        None => {
-            info!("Event notifier system is not initialized.");
-            return;
-        }
+        return Ok(());
     };
 
-    // Call the shutdown function from the rustfs_notify module
-    system.shutdown().await;
+    system.shutdown_checked().await?;
     info!("Event notifier system shut down successfully.");
+    Ok(())
 }
 
 #[instrument]
-pub async fn init_event_notifier() {
-    if let Err(err) = refresh_persisted_module_switches_from_store().await {
-        warn!("Failed to refresh persisted notify module switch from store: {}", err);
-    }
+pub async fn init_event_notifier() -> Result<(), NotificationError> {
+    mark_event_notifier_unreconciled();
+    validate_notify_module_env().map_err(NotificationError::Initialization)?;
+    let system = ensure_live_events_initialized();
+    refresh_persisted_module_switches_from_store()
+        .await
+        .map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?;
 
     let enabled = refresh_notify_module_enabled();
+
     if !enabled {
         info!(
             target: "rustfs::main::init_event_notifier",
             "Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
             rustfs_config::ENV_NOTIFY_ENABLE
         );
-        if ensure_live_events_initialized() {
-            info!(
-                target: "rustfs::main::init_event_notifier",
-                "Live event stream support initialized successfully."
-            );
+        if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly {
+            system.set_targets_enabled(false, None).await?;
         }
-        return;
+        system.reload_persisted_config().await?;
+        info!(
+            target: "rustfs::main::init_event_notifier",
+            "Live event stream support initialized successfully."
+        );
+        ensure_event_notifier_converged(&system)?;
+        mark_event_notifier_reconciled();
+        return Ok(());
     }
 
     info!(
@@ -192,53 +342,44 @@ pub async fn init_event_notifier() {
         "Initializing event notifier..."
     );
 
-    // 1. Get the global configuration loaded by ecstore
-    let server_config = match server_config_from_context() {
-        Some(config) => config,
-        None => {
-            warn!("Event notifier initialization failed: Global server config not loaded.");
-            return;
-        }
-    };
-
     info!(
         target: "rustfs::main::init_event_notifier",
         "Event notifier configuration found, proceeding with initialization."
     );
 
-    if let Some(system) = rustfs_notify::notification_system() {
-        // Reuse the existing global system on re-enable so bucket rules, metrics,
-        // and stream lifecycle stay aligned with the current process singleton.
-        if let Err(e) = system.reload_config(server_config).await {
-            error!("Failed to reload event notifier system: {}", e);
-        } else {
-            info!(
-                target: "rustfs::main::init_event_notifier",
-                "Event notifier system reloaded successfully."
-            );
-        }
-    } else {
-        match rustfs_notify::initialize(server_config).await {
-            Ok(()) => {
-                install_ecstore_event_dispatch_hook();
-                info!(
-                    target: "rustfs::main::init_event_notifier",
-                    "Event notifier system initialized successfully."
-                );
-            }
-            Err(e) => error!("Failed to initialize event notifier system: {}", e),
-        }
+    system.reload_persisted_config().await?;
+    let runtime_state = system.runtime_lifecycle_state();
+    if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) {
+        system.set_targets_enabled(true, None).await?;
     }
+    info!(
+        target: "rustfs::main::init_event_notifier",
+        "Event notifier system initialized successfully."
+    );
+    ensure_event_notifier_converged(&system)?;
+    mark_event_notifier_reconciled();
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
-    use super::{convert_ecstore_object_info, parse_host_and_port};
+    use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler};
     use crate::storage_api::server::event::StorageObjectInfo;
     use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
     use chrono::{DateTime, Utc};
-    use std::{collections::HashMap, sync::Arc};
+    use rustfs_notify::NotificationError;
+    use std::{
+        collections::HashMap,
+        future::pending,
+        sync::{
+            Arc,
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+        },
+        time::Duration as StdDuration,
+    };
     use time::{Duration, OffsetDateTime};
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
 
     #[test]
     fn parse_host_and_port_with_ipv4_and_port() {
@@ -303,4 +444,88 @@ mod tests {
         assert_eq!(converted.storage_class.as_deref(), Some("GLACIER"));
         assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
     }
+
+    #[tokio::test(start_paused = true)]
+    async fn persisted_reconciler_converges_after_one_injected_tick() {
+        let persisted_generation = Arc::new(AtomicUsize::new(1));
+        let runtime_generation = Arc::new(AtomicUsize::new(1));
+        let runtime_converged = Arc::new(AtomicBool::new(true));
+        let reconcile_calls = Arc::new(AtomicUsize::new(0));
+        let reconciled = Arc::new(Notify::new());
+        let cancellation = CancellationToken::new();
+
+        let task = tokio::spawn(run_persisted_event_notifier_reconciler(
+            cancellation.clone(),
+            StdDuration::from_secs(5),
+            {
+                let persisted_generation = persisted_generation.clone();
+                let runtime_generation = runtime_generation.clone();
+                let runtime_converged = runtime_converged.clone();
+                let reconcile_calls = reconcile_calls.clone();
+                let reconciled = reconciled.clone();
+                move || {
+                    let persisted_generation = persisted_generation.clone();
+                    let runtime_generation = runtime_generation.clone();
+                    let runtime_converged = runtime_converged.clone();
+                    let reconcile_calls = reconcile_calls.clone();
+                    let reconciled = reconciled.clone();
+                    async move {
+                        runtime_generation.store(persisted_generation.load(Ordering::SeqCst), Ordering::SeqCst);
+                        runtime_converged.store(true, Ordering::SeqCst);
+                        reconcile_calls.fetch_add(1, Ordering::SeqCst);
+                        reconciled.notify_one();
+                        Ok(())
+                    }
+                }
+            },
+        ));
+        tokio::task::yield_now().await;
+
+        persisted_generation.store(2, Ordering::SeqCst);
+        runtime_converged.store(false, Ordering::SeqCst);
+        assert_eq!(
+            reconcile_calls.load(Ordering::SeqCst),
+            0,
+            "the first tick must wait for the configured interval"
+        );
+
+        tokio::time::advance(StdDuration::from_secs(5)).await;
+        reconciled.notified().await;
+
+        assert_eq!(reconcile_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(runtime_generation.load(Ordering::SeqCst), 2);
+        assert!(runtime_converged.load(Ordering::SeqCst));
+
+        cancellation.cancel();
+        task.await.expect("persisted reconciler should stop after cancellation");
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn persisted_reconciler_cancellation_interrupts_an_inflight_attempt() {
+        let entered = Arc::new(Notify::new());
+        let cancellation = CancellationToken::new();
+        let task = tokio::spawn(run_persisted_event_notifier_reconciler(
+            cancellation.clone(),
+            StdDuration::from_secs(5),
+            {
+                let entered = entered.clone();
+                move || {
+                    let entered = entered.clone();
+                    async move {
+                        entered.notify_one();
+                        pending::<Result<(), NotificationError>>().await
+                    }
+                }
+            },
+        ));
+        tokio::task::yield_now().await;
+        tokio::time::advance(StdDuration::from_secs(5)).await;
+        entered.notified().await;
+
+        cancellation.cancel();
+        tokio::time::timeout(StdDuration::from_secs(1), task)
+            .await
+            .expect("cancellation should stop an in-flight reconciliation attempt")
+            .expect("persisted reconciler should not panic");
+    }
 }

rustfs/src/server/mod.rs

@@ -32,6 +32,7 @@ pub mod tls_material;
 use tracing::warn;
 
 // Items used by main.rs (binary crate) and/or embedded.rs — must be fully pub.
+pub(crate) use audit::apply_audit_module_switch_for_context;
 pub use audit::{is_audit_module_enabled, refresh_audit_module_enabled, start_audit_system, stop_audit_system};
 pub use event::{init_event_notifier, is_notify_module_enabled, refresh_notify_module_enabled, shutdown_event_notifier};
 pub use http::start_http_server;
@@ -45,6 +46,10 @@ pub use service_state::wait_for_shutdown;
 
 // Items only used within the library crate (admin handlers, server/http.rs, etc.).
 pub(crate) use event::convert_ecstore_object_info;
+pub(crate) use event::{
+    is_event_notifier_reconciled, mark_event_notifier_reconciled, mark_event_notifier_unreconciled,
+    reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler,
+};
 #[cfg(test)]
 pub(crate) use health::{
     HealthPayloadContext, HealthReadinessSource, build_component_details, build_health_payload, health_check_state,
@@ -55,8 +60,9 @@ pub(crate) use http::HeaderMapCarrier;
 pub(crate) use http::active_http_requests;
 pub(crate) use layer::RequestContextLayer;
 pub(crate) use module_switch::{
-    ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot,
-    refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,
+    MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
+    current_module_switch_snapshot, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store,
+    save_persisted_module_switches_to, validate_module_switch_update,
 };
 pub(crate) use prefix::{
     ADMIN_PREFIX, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE,

rustfs/src/server/module_switch.rs

@@ -13,17 +13,35 @@
 // limitations under the License.
 
 use super::runtime_sources;
-use crate::storage_api::server::module_switch::{Error as StorageError, read_config, save_config};
+use crate::storage_api::server::module_switch::{
+    Error as StorageError, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
+    with_config_object_write_lock,
+};
+use crate::storage_api::server::runtime_sources::ECStore;
 use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use std::sync::LazyLock;
 use std::sync::atomic::{AtomicBool, Ordering};
+use tokio::sync::Mutex;
 
 const MODULE_SWITCH_CONFIG_PATH: &str = "config/module_switches.json";
+pub(crate) const MODULE_SWITCHES_SIGNAL_SUBSYSTEM: &str = "module_switches";
 
 // Keep a cheap in-process snapshot so hot-path checks do not need to read
 // cluster metadata after startup or console-triggered refresh.
 static PERSISTED_NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
 static PERSISTED_AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE);
 static PERSISTED_MODULE_SWITCH_CONFIGURED: AtomicBool = AtomicBool::new(false);
+static MODULE_SWITCH_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
+
+async fn serialize_module_switch_rmw<F, Fut, T>(operation: F) -> T
+where
+    F: FnOnce() -> Fut,
+    Fut: std::future::Future<Output = T>,
+{
+    let _rmw_guard = MODULE_SWITCH_RMW_LOCK.lock().await;
+    operation().await
+}
 
 #[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
 pub(crate) struct PersistedModuleSwitches {
@@ -80,6 +98,14 @@ fn env_override_value(key: &str) -> Option<bool> {
     rustfs_utils::get_env_opt_bool(key)
 }
 
+pub(crate) fn validate_notify_module_env() -> Result<(), String> {
+    let key = rustfs_config::ENV_NOTIFY_ENABLE;
+    if env_override_exists(key) && env_override_value(key).is_none() {
+        return Err(format!("{key} is not a valid boolean"));
+    }
+    Ok(())
+}
+
 fn effective_module_switch_state(env_key: &str, persisted_enabled: bool, default_enabled: bool) -> ModuleSwitchResolution {
     // Explicit env remains the highest-priority source so process-level bootstrap
     // cannot be silently overridden by a later console write.
@@ -162,42 +188,157 @@ pub(crate) async fn refresh_persisted_module_switches_from_store() -> Result<Per
     let Some(store) = runtime_sources::current_object_store_handle() else {
         return Err("storage layer not initialized".to_string());
     };
+    refresh_persisted_module_switches_from(store).await
+}
 
-    let (config, configured) = match read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
-        Ok(data) => (
+pub(crate) async fn refresh_persisted_module_switches_from(store: Arc<ECStore>) -> Result<PersistedModuleSwitches, String> {
+    refresh_persisted_module_switches_with(|| async move {
+        match read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
+            Ok(data) => Ok(Some(data)),
+            Err(StorageError::ConfigNotFound) => Ok(None),
+            Err(err) => Err(format!("failed to load module switch config: {err}")),
+        }
+    })
+    .await
+}
+
+pub(crate) async fn with_refreshed_notify_module_state_from<T, Publish, PublishFuture>(
+    store: Arc<ECStore>,
+    publish: Publish,
+) -> Result<T, String>
+where
+    Publish: FnOnce(ModuleSwitchResolution) -> PublishFuture + Send + 'static,
+    PublishFuture: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    // Lock order: local module RMW -> distributed module-object read -> the
+    // callback's server-config read lock. Server-config writers never acquire
+    // the module-object lock, and module writers release any server-config
+    // read before taking the module-object write lock.
+    serialize_module_switch_rmw(|| async move {
+        let read_store = store.clone();
+        with_config_object_read_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
+            let persisted = match read_config_no_lock(read_store, MODULE_SWITCH_CONFIG_PATH).await {
+                Ok(data) => Some(data),
+                Err(StorageError::ConfigNotFound) => None,
+                Err(err) => return Err(format!("failed to load module switch config: {err}")),
+            };
+            let config = decode_persisted_module_switches(persisted)?;
+            set_persisted_module_switches(config.0, config.1);
+            Ok(publish(resolve_notify_module_state()).await)
+        })
+        .await
+        .map_err(|err| format!("failed to lock module switch refresh: {err}"))?
+    })
+    .await
+}
+
+fn decode_persisted_module_switches(data: Option<Vec<u8>>) -> Result<(PersistedModuleSwitches, bool), String> {
+    match data {
+        Some(data) => Ok((
             serde_json::from_slice::<PersistedModuleSwitches>(&data)
                 .map_err(|e| format!("failed to deserialize module switch config: {e}"))?,
             true,
-        ),
-        Err(StorageError::ConfigNotFound) => (PersistedModuleSwitches::default(), false),
-        Err(err) => return Err(format!("failed to load module switch config: {err}")),
-    };
-
-    // Track whether the persisted file exists so the effective state can
-    // distinguish "console configured false" from "never configured, use default".
-    set_persisted_module_switches(config, configured);
-    Ok(config)
+        )),
+        None => Ok((PersistedModuleSwitches::default(), false)),
+    }
 }
 
-pub(crate) async fn save_persisted_module_switches_to_store(config: PersistedModuleSwitches) -> Result<(), String> {
-    let Some(store) = runtime_sources::current_object_store_handle() else {
-        return Err("storage layer not initialized".to_string());
-    };
+async fn refresh_persisted_module_switches_with<F, Fut>(read: F) -> Result<PersistedModuleSwitches, String>
+where
+    F: FnOnce() -> Fut,
+    Fut: std::future::Future<Output = Result<Option<Vec<u8>>, String>>,
+{
+    serialize_module_switch_rmw(|| async move {
+        let (config, configured) = decode_persisted_module_switches(read().await?)?;
 
-    let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?;
-    save_config(store, MODULE_SWITCH_CONFIG_PATH, data)
+        // Track whether the persisted file exists so the effective state can
+        // distinguish "console configured false" from "never configured, use default".
+        set_persisted_module_switches(config, configured);
+        Ok(config)
+    })
+    .await
+}
+
+pub(crate) async fn save_persisted_module_switches_to<T, F>(
+    store: Arc<ECStore>,
+    config: PersistedModuleSwitches,
+    publish: F,
+) -> Result<T, String>
+where
+    F: FnOnce() -> T + Send + 'static,
+    T: Send + 'static,
+{
+    // Lock order matches refresh_persisted_module_switches_from: local RMW
+    // guard before the namespace lock. Taking these in the opposite order lets
+    // a reader hold the local guard while waiting for a writer that is itself
+    // waiting for the local guard.
+    serialize_module_switch_rmw(|| async move {
+        let save_store = store.clone();
+        with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
+            save_persisted_module_switches_inner(
+                config,
+                move |data| async move {
+                    save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data)
+                        .await
+                        .map_err(|e| format!("failed to save module switch config: {e}"))
+                },
+                publish,
+            )
+            .await
+        })
         .await
-        .map_err(|e| format!("failed to save module switch config: {e}"))?;
+        .map_err(|err| format!("failed to lock module switch update: {err}"))?
+    })
+    .await
+}
+
+#[cfg(test)]
+async fn save_persisted_module_switches_with<T, F, Save, SaveFuture>(
+    config: PersistedModuleSwitches,
+    save: Save,
+    publish: F,
+) -> Result<T, String>
+where
+    F: FnOnce() -> T,
+    Save: FnOnce(Vec<u8>) -> SaveFuture,
+    SaveFuture: std::future::Future<Output = Result<(), String>>,
+{
+    serialize_module_switch_rmw(|| save_persisted_module_switches_inner(config, save, publish)).await
+}
+
+async fn save_persisted_module_switches_inner<T, F, Save, SaveFuture>(
+    config: PersistedModuleSwitches,
+    save: Save,
+    publish: F,
+) -> Result<T, String>
+where
+    F: FnOnce() -> T,
+    Save: FnOnce(Vec<u8>) -> SaveFuture,
+    SaveFuture: std::future::Future<Output = Result<(), String>>,
+{
+    let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?;
+    save(data).await?;
 
     set_persisted_module_switches(config, true);
-    Ok(())
+    Ok(publish())
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::storage_api::startup::storage::{EndpointServerPools, init_local_disks_with_instance_ctx, new_instance_ctx};
     use serial_test::serial;
+    use std::future::{Future, poll_fn};
+    use std::sync::{
+        Arc, Mutex as StdMutex,
+        atomic::{AtomicBool, Ordering as AtomicOrdering},
+    };
+    use std::task::Poll;
     use temp_env::{with_var, with_vars};
+    use tempfile::TempDir;
+    use tokio::sync::{Notify, oneshot};
+    use tokio_util::sync::CancellationToken;
 
     #[test]
     #[serial]
@@ -299,6 +440,15 @@ mod tests {
         });
     }
 
+    #[test]
+    #[serial]
+    fn invalid_notify_env_is_rejected_before_runtime_reconcile() {
+        with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("invalid"), || {
+            let err = validate_notify_module_env().expect_err("invalid notify env must fail closed");
+            assert!(err.contains(rustfs_config::ENV_NOTIFY_ENABLE));
+        });
+    }
+
     #[test]
     #[serial]
     fn validate_module_switch_update_allows_matching_env_override() {
@@ -330,4 +480,200 @@ mod tests {
             assert!(err.contains("not a valid boolean"));
         });
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn save_publish_and_refresh_share_the_rmw_lock() {
+        let previous = current_persisted_module_switches();
+        let previous_configured = persisted_module_switches_configured();
+
+        save_persisted_module_switches_with(
+            PersistedModuleSwitches {
+                notify_enabled: false,
+                audit_enabled: true,
+            },
+            |_| async { Ok(()) },
+            || {
+                assert!(MODULE_SWITCH_RMW_LOCK.try_lock().is_err(), "publish must run while the RMW lock is held");
+            },
+        )
+        .await
+        .expect("save and publish should succeed");
+
+        let events = Arc::new(StdMutex::new(Vec::new()));
+        let release_save = Arc::new(Notify::new());
+        let (save_started_tx, save_started_rx) = oneshot::channel();
+
+        let save_events = events.clone();
+        let publish_events = events.clone();
+        let save_release = release_save.clone();
+        let save_task = tokio::spawn(async move {
+            save_persisted_module_switches_with(
+                PersistedModuleSwitches {
+                    notify_enabled: true,
+                    audit_enabled: false,
+                },
+                move |_| async move {
+                    save_events.lock().expect("lock event log").push("save");
+                    save_started_tx.send(()).expect("signal save start");
+                    save_release.notified().await;
+                    Ok(())
+                },
+                move || {
+                    publish_events.lock().expect("lock event log").push("publish");
+                },
+            )
+            .await
+        });
+
+        save_started_rx.await.expect("save should reach its persistence step");
+
+        let refresh_events = events.clone();
+        let (refresh_waiting_tx, refresh_waiting_rx) = oneshot::channel();
+        let refresh_task = tokio::spawn(async move {
+            let mut refresh = Box::pin(refresh_persisted_module_switches_with(move || async move {
+                refresh_events.lock().expect("lock event log").push("refresh");
+                Ok(None)
+            }));
+
+            poll_fn(|cx| match refresh.as_mut().poll(cx) {
+                Poll::Pending => Poll::Ready(()),
+                Poll::Ready(_) => panic!("refresh must wait for an in-flight save"),
+            })
+            .await;
+            refresh_waiting_tx.send(()).expect("signal refresh wait");
+            refresh.await
+        });
+
+        refresh_waiting_rx.await.expect("refresh should be waiting on the RMW lock");
+        release_save.notify_one();
+
+        save_task.await.expect("join save task").expect("save should succeed");
+        refresh_task
+            .await
+            .expect("join refresh task")
+            .expect("refresh should succeed");
+
+        assert_eq!(*events.lock().expect("lock event log"), ["save", "publish", "refresh"]);
+        set_persisted_module_switches(previous, previous_configured);
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn distributed_refresh_orders_old_true_before_new_false_publish() {
+        temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>)], async {
+            let previous = current_persisted_module_switches();
+            let previous_configured = persisted_module_switches_configured();
+            let temp_dir = TempDir::new().expect("module switch ordering temp dir");
+            let volume = temp_dir.path().join("disk");
+            tokio::fs::create_dir_all(&volume)
+                .await
+                .expect("create module switch test disk");
+            let (endpoint_pools, _) =
+                EndpointServerPools::from_volumes("127.0.0.1:29131", vec![volume.to_string_lossy().into_owned()])
+                    .await
+                    .expect("create module switch test endpoints");
+            let instance_ctx = new_instance_ctx();
+            init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+                .await
+                .expect("initialize module switch test disk");
+            let shutdown = CancellationToken::new();
+            let store = ECStore::new_with_instance_ctx(
+                "127.0.0.1:29131".parse().expect("module switch test address"),
+                endpoint_pools,
+                shutdown.clone(),
+                instance_ctx,
+            )
+            .await
+            .expect("create module switch test store");
+
+            save_persisted_module_switches_to(
+                store.clone(),
+                PersistedModuleSwitches {
+                    notify_enabled: true,
+                    audit_enabled: false,
+                },
+                || (),
+            )
+            .await
+            .expect("persist initial enabled module state");
+
+            let publish_order = Arc::new(StdMutex::new(Vec::new()));
+            let runtime_enabled = Arc::new(AtomicBool::new(false));
+            let reader_entered = Arc::new(Notify::new());
+            let release_reader = Arc::new(Notify::new());
+            let reader = tokio::spawn({
+                let store = store.clone();
+                let publish_order = publish_order.clone();
+                let runtime_enabled = runtime_enabled.clone();
+                let reader_entered = reader_entered.clone();
+                let release_reader = release_reader.clone();
+                async move {
+                    with_refreshed_notify_module_state_from(store, move |resolution| async move {
+                        assert!(resolution.enabled, "the old persisted snapshot should be enabled");
+                        reader_entered.notify_one();
+                        release_reader.notified().await;
+                        publish_order.lock().expect("module publish order lock").push("old-true");
+                        runtime_enabled.store(true, AtomicOrdering::SeqCst);
+                    })
+                    .await
+                }
+            });
+            reader_entered.notified().await;
+
+            let writer_entered = Arc::new(AtomicBool::new(false));
+            let (writer_started_tx, writer_started_rx) = oneshot::channel();
+            let writer = tokio::spawn({
+                let store = store.clone();
+                let publish_order = publish_order.clone();
+                let runtime_enabled = runtime_enabled.clone();
+                let writer_entered = writer_entered.clone();
+                async move {
+                    writer_started_tx.send(()).expect("signal remote writer start");
+                    let save_store = store.clone();
+                    with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move {
+                        writer_entered.store(true, AtomicOrdering::SeqCst);
+                        let data = serde_json::to_vec(&PersistedModuleSwitches {
+                            notify_enabled: false,
+                            audit_enabled: false,
+                        })
+                        .expect("serialize disabled module state");
+                        save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data)
+                            .await
+                            .map_err(|err| err.to_string())?;
+                        publish_order.lock().expect("module publish order lock").push("new-false");
+                        runtime_enabled.store(false, AtomicOrdering::SeqCst);
+                        Ok::<(), String>(())
+                    })
+                    .await
+                    .expect("remote writer should acquire module object lock")
+                    .expect("remote writer should persist disabled state");
+                }
+            });
+            writer_started_rx.await.expect("remote writer should start");
+            for _ in 0..10 {
+                tokio::task::yield_now().await;
+            }
+            assert!(
+                !writer_entered.load(AtomicOrdering::SeqCst),
+                "the remote writer must wait until the old read has synchronously published"
+            );
+
+            release_reader.notify_one();
+            reader
+                .await
+                .expect("join module reader")
+                .expect("module reader should succeed");
+            writer.await.expect("join remote module writer");
+
+            assert_eq!(
+                *publish_order.lock().expect("module publish order result lock"),
+                ["old-true", "new-false"]
+            );
+            assert!(!runtime_enabled.load(AtomicOrdering::SeqCst), "the latest persisted false state must win");
+            shutdown.cancel();
+            set_persisted_module_switches(previous, previous_configured);
+        })
+        .await;
+    }
 }

rustfs/src/startup_audit.rs

@@ -20,6 +20,7 @@ use tracing::{error, info};
 const LOG_COMPONENT_MAIN: &str = "main";
 const LOG_SUBSYSTEM_STARTUP: &str = "startup";
 const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state";
+const EVENT_NOTIFY_SYSTEM_STATE: &str = "notify_system_state";
 
 pub(crate) async fn init_audit_runtime() {
     match init_event_notifier_and_audit().await {
@@ -47,17 +48,28 @@ pub(crate) async fn init_event_notifier_and_audit() -> AuditResult<()> {
     init_event_notifier_and_audit_with(init_event_notifier, start_audit_system).await
 }
 
-async fn init_event_notifier_and_audit_with<NotifyFn, NotifyFuture, AuditFn, AuditFuture>(
+async fn init_event_notifier_and_audit_with<NotifyFn, NotifyFuture, NotifyError, AuditFn, AuditFuture>(
     notify: NotifyFn,
     start_audit: AuditFn,
 ) -> AuditResult<()>
 where
     NotifyFn: FnOnce() -> NotifyFuture,
-    NotifyFuture: Future<Output = ()>,
+    NotifyFuture: Future<Output = Result<(), NotifyError>>,
+    NotifyError: std::fmt::Display,
     AuditFn: FnOnce() -> AuditFuture,
     AuditFuture: Future<Output = AuditResult<()>>,
 {
-    notify().await;
+    if let Err(err) = notify().await {
+        error!(
+            target: "rustfs::main::run",
+            event = EVENT_NOTIFY_SYSTEM_STATE,
+            component = LOG_COMPONENT_MAIN,
+            subsystem = LOG_SUBSYSTEM_STARTUP,
+            state = "degraded",
+            error = %err,
+            "Notification runtime failed to start; continuing in degraded mode"
+        );
+    }
     start_audit().await
 }
 
@@ -75,6 +87,7 @@ mod tests {
         let result = init_event_notifier_and_audit_with(
             move || async move {
                 notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
+                Ok::<(), &'static str>(())
             },
             move || async move {
                 audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
@@ -97,6 +110,7 @@ mod tests {
         let result = init_event_notifier_and_audit_with(
             move || async move {
                 notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
+                Ok::<(), &'static str>(())
             },
             move || async move {
                 audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
@@ -109,4 +123,26 @@ mod tests {
         let events = events.lock().unwrap_or_else(|err| err.into_inner()).clone();
         assert_eq!(events, ["notify", "audit"]);
     }
+
+    #[tokio::test]
+    async fn notify_failure_still_starts_audit() {
+        let events = Arc::new(Mutex::new(Vec::new()));
+        let notify_events = events.clone();
+        let audit_events = events.clone();
+
+        let result = init_event_notifier_and_audit_with(
+            move || async move {
+                notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify");
+                Err::<(), &'static str>("notify failed")
+            },
+            move || async move {
+                audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit");
+                Ok(())
+            },
+        )
+        .await;
+
+        assert!(result.is_ok());
+        assert_eq!(*events.lock().unwrap_or_else(|err| err.into_inner()), ["notify", "audit"]);
+    }
 }

rustfs/src/startup_lifecycle.rs

@@ -14,7 +14,7 @@
 
 use crate::storage_api::startup::lifecycle::ECStore;
 use crate::{
-    server::{ServiceStateManager, ShutdownHandle, wait_for_shutdown},
+    server::{ServiceStateManager, ShutdownHandle, start_persisted_event_notifier_reconciler, wait_for_shutdown},
     startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap},
     startup_runtime_sources,
     startup_services::StartupServiceRuntime,
@@ -142,6 +142,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
     );
     publish_ready_for_iam_bootstrap(iam_bootstrap, readiness.as_ref(), Some(state_manager.as_ref())).await?;
     startup_runtime_sources::publish_init_time_now().await;
+    let event_notifier_reconciler = start_persisted_event_notifier_reconciler(store.clone(), shutdown_token.clone());
 
     if enable_scanner {
         init_data_scanner(shutdown_token.clone(), store).await;
@@ -157,6 +158,17 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec
         shutdown_token,
     )
     .await;
+    if let Err(err) = event_notifier_reconciler.await {
+        tracing::warn!(
+            target: "rustfs::main::run",
+            event = "notify_runtime_reconcile",
+            component = LOG_COMPONENT_MAIN,
+            subsystem = LOG_SUBSYSTEM_STARTUP,
+            state = "join_failed",
+            reason = if err.is_cancelled() { "task_cancelled" } else { "task_panicked" },
+            "Persisted notification runtime reconciler task failed to join"
+        );
+    }
 
     info!(
         target: "rustfs::main::run",

rustfs/src/startup_shutdown.rs

@@ -21,8 +21,11 @@ use crate::{
     startup_runtime_sources,
 };
 use rustfs_heal::shutdown_ahm_services;
+use rustfs_notify::NotificationLifecycleTransition;
 use rustfs_utils::get_env_bool_with_aliases;
-use std::path::Path;
+use std::future::Future;
+use std::path::PathBuf;
+use std::sync::Mutex;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -43,6 +46,137 @@ const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown";
 const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown";
 const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state";
 
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}
+
+struct EmbeddedRuntimeOwnerState {
+    owners: usize,
+    pending_cleanup: Option<CancellationToken>,
+}
+
+struct EmbeddedRuntimeOwners {
+    state: Mutex<EmbeddedRuntimeOwnerState>,
+}
+
+impl EmbeddedRuntimeOwners {
+    const fn new() -> Self {
+        Self {
+            state: Mutex::new(EmbeddedRuntimeOwnerState {
+                owners: 0,
+                pending_cleanup: None,
+            }),
+        }
+    }
+
+    fn register(&self) -> Option<CancellationToken> {
+        let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+        state.owners += 1;
+        state
+            .pending_cleanup
+            .as_ref()
+            .filter(|cleanup| !cleanup.is_cancelled())
+            .cloned()
+    }
+
+    fn release_with<T>(&self, prepare_cleanup: impl FnOnce(CancellationToken) -> T) -> Option<T> {
+        let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+        if state.owners == 0 {
+            return None;
+        }
+        state.owners -= 1;
+        if state.owners != 0 {
+            return None;
+        }
+        if state.pending_cleanup.as_ref().is_some_and(|cleanup| !cleanup.is_cancelled()) {
+            return None;
+        }
+
+        let completion = CancellationToken::new();
+        // The disable generation must be accepted while registration is excluded;
+        // otherwise a concurrently starting owner can enable targets first and be
+        // overwritten by this last-owner release.
+        let cleanup = prepare_cleanup(completion.clone());
+        state.pending_cleanup = Some(completion);
+        Some(cleanup)
+    }
+
+    #[cfg(test)]
+    fn owner_count(&self) -> usize {
+        self.state.lock().unwrap_or_else(|err| err.into_inner()).owners
+    }
+}
+
+static EMBEDDED_RUNTIME_OWNERS: EmbeddedRuntimeOwners = EmbeddedRuntimeOwners::new();
+
+pub(crate) struct EmbeddedRuntimeOwner {
+    active: bool,
+    runtime: tokio::runtime::Handle,
+}
+
+pub(crate) struct EmbeddedRuntimeCleanup {
+    completion: CancellationToken,
+    notification: Option<NotificationLifecycleTransition>,
+    runtime: tokio::runtime::Handle,
+}
+
+impl EmbeddedRuntimeCleanup {
+    fn prepare(completion: CancellationToken, runtime: tokio::runtime::Handle) -> Self {
+        let _runtime_guard = runtime.enter();
+        let system = rustfs_notify::ensure_live_events();
+        Self {
+            completion,
+            notification: Some(system.publish_targets_enabled(false, None)),
+            runtime,
+        }
+    }
+}
+
+impl Drop for EmbeddedRuntimeCleanup {
+    fn drop(&mut self) {
+        self.completion.cancel();
+    }
+}
+
+impl EmbeddedRuntimeOwner {
+    pub(crate) fn cleanup_runtime_handle(&self) -> tokio::runtime::Handle {
+        tokio::runtime::Handle::try_current().unwrap_or_else(|_| self.runtime.clone())
+    }
+
+    pub(crate) fn release(&mut self) -> Option<EmbeddedRuntimeCleanup> {
+        if !self.active {
+            return None;
+        }
+        self.active = false;
+        let runtime = self.cleanup_runtime_handle();
+        EMBEDDED_RUNTIME_OWNERS.release_with(move |completion| EmbeddedRuntimeCleanup::prepare(completion, runtime))
+    }
+}
+
+impl Drop for EmbeddedRuntimeOwner {
+    fn drop(&mut self) {
+        if let Some(cleanup) = self.release() {
+            schedule_embedded_runtime_cleanup(cleanup);
+        }
+    }
+}
+
+pub(crate) async fn register_embedded_runtime_owner() -> EmbeddedRuntimeOwner {
+    let runtime = tokio::runtime::Handle::current();
+    let pending_cleanup = EMBEDDED_RUNTIME_OWNERS.register();
+    let owner = EmbeddedRuntimeOwner { active: true, runtime };
+    // Reserve ownership before waiting so another release cannot start a second
+    // process-runtime cleanup while this startup is queued behind the first.
+    if let Some(cleanup) = pending_cleanup {
+        cleanup.cancelled().await;
+    }
+    owner
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum BackgroundShutdownStep {
     DataScanner,
@@ -153,7 +287,17 @@ pub(crate) async fn run_startup_shutdown_sequence(
         state = "stopping",
         "Event notifier shutdown started"
     );
-    shutdown_event_notifier().await;
+    if let Err(err) = shutdown_event_notifier().await {
+        error!(
+            target: "rustfs::main::handle_shutdown",
+            event = EVENT_EVENT_NOTIFIER_SHUTDOWN,
+            component = LOG_COMPONENT_MAIN,
+            subsystem = LOG_SUBSYSTEM_STARTUP,
+            state = "stop_failed",
+            error = %err,
+            "Event notifier shutdown failed"
+        );
+    }
 
     info!(
         target: "rustfs::main::handle_shutdown",
@@ -222,9 +366,19 @@ pub(crate) async fn run_startup_shutdown_sequence(
     );
 }
 
-pub(crate) async fn run_embedded_shutdown_cleanup() {
-    shutdown_event_notifier().await;
-
+async fn run_embedded_runtime_cleanup(mut cleanup: EmbeddedRuntimeCleanup) {
+    if let Some(notification) = cleanup.notification.take()
+        && let Err(err) = notification.wait().await
+    {
+        warn!(
+            component = LOG_COMPONENT_EMBEDDED,
+            subsystem = LOG_SUBSYSTEM_EMBEDDED,
+            event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
+            service = "notification",
+            error = %err,
+            "Embedded shutdown cleanup failed"
+        );
+    }
     if let Err(err) = stop_audit_system().await {
         warn!(
             component = LOG_COMPONENT_EMBEDDED,
@@ -235,6 +389,38 @@ pub(crate) async fn run_embedded_shutdown_cleanup() {
             "Embedded shutdown cleanup failed"
         );
     }
+    if let Err(err) = startup_runtime_sources::shutdown_observability_guard() {
+        warn!(
+            component = LOG_COMPONENT_EMBEDDED,
+            subsystem = LOG_SUBSYSTEM_EMBEDDED,
+            event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
+            service = "observability",
+            error = %err,
+            "Embedded shutdown cleanup failed"
+        );
+    }
+    cleanup.completion.cancel();
+}
+
+pub(crate) async fn run_embedded_shutdown_cleanup(cleanup: EmbeddedRuntimeCleanup) {
+    let completion = cleanup.completion.clone();
+    let runtime = cleanup.runtime.clone();
+    if let Err(err) = runtime.spawn(run_embedded_runtime_cleanup(cleanup)).await {
+        completion.cancel();
+        warn!(
+            component = LOG_COMPONENT_EMBEDDED,
+            subsystem = LOG_SUBSYSTEM_EMBEDDED,
+            event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
+            service = "process_runtime",
+            reason = join_failure_reason(&err),
+            "Embedded shutdown cleanup failed"
+        );
+    }
+}
+
+fn schedule_embedded_runtime_cleanup(cleanup: EmbeddedRuntimeCleanup) {
+    let runtime = cleanup.runtime.clone();
+    runtime.spawn(run_embedded_shutdown_cleanup(cleanup));
 }
 
 pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle, ctx: &CancellationToken) {
@@ -242,24 +428,66 @@ pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle,
     ctx.cancel();
 }
 
+async fn release_embedded_runtime_after_drain(mut runtime_owner: Option<EmbeddedRuntimeOwner>) {
+    if let Some(runtime_owner) = runtime_owner.as_mut()
+        && let Some(cleanup) = runtime_owner.release()
+    {
+        run_embedded_shutdown_cleanup(cleanup).await;
+    }
+}
+
 pub(crate) fn run_embedded_server_drop_cleanup(
+    runtime: &tokio::runtime::Handle,
     ctx: &CancellationToken,
     shutdown_handle: &mut Option<ShutdownHandle>,
-    temp_dir: Option<&Path>,
+    temp_dir: &mut Option<PathBuf>,
+    runtime_owner: Option<EmbeddedRuntimeOwner>,
 ) {
     ctx.cancel();
-    if let Some(shutdown_handle) = shutdown_handle.take() {
+    if let Some(shutdown_handle) = shutdown_handle.as_ref() {
         shutdown_handle.signal();
     }
-    if let Some(dir) = temp_dir {
-        let _ = std::fs::remove_dir_all(dir);
+
+    let shutdown_handle = shutdown_handle.take();
+    let temp_dir = temp_dir.take();
+    runtime.spawn(finish_embedded_server_cleanup(
+        shutdown_handle,
+        temp_dir,
+        release_embedded_runtime_after_drain(runtime_owner),
+    ));
+}
+
+async fn finish_embedded_server_cleanup<F>(shutdown_handle: Option<ShutdownHandle>, temp_dir: Option<PathBuf>, process_cleanup: F)
+where
+    F: Future<Output = ()>,
+{
+    if let Some(shutdown_handle) = shutdown_handle {
+        shutdown_handle.shutdown().await;
+    }
+
+    process_cleanup.await;
+
+    if let Some(dir) = temp_dir
+        && let Err(err) = tokio::fs::remove_dir_all(&dir).await
+    {
+        warn!(
+            component = LOG_COMPONENT_EMBEDDED,
+            subsystem = LOG_SUBSYSTEM_EMBEDDED,
+            event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
+            service = "temp_dir",
+            path = %dir.display(),
+            error = %err,
+            "Embedded shutdown cleanup failed"
+        );
     }
 }
 
 pub(crate) async fn run_embedded_server_shutdown(
+    runtime: &tokio::runtime::Handle,
     ctx: &CancellationToken,
     shutdown_handle: &mut Option<ShutdownHandle>,
-    temp_dir: Option<&Path>,
+    temp_dir: &mut Option<PathBuf>,
+    runtime_owner: Option<EmbeddedRuntimeOwner>,
 ) {
     info!(
         target: "rustfs::embedded",
@@ -272,22 +500,22 @@ pub(crate) async fn run_embedded_server_shutdown(
 
     ctx.cancel();
 
-    run_embedded_shutdown_cleanup().await;
-
-    if let Some(shutdown_handle) = shutdown_handle.take() {
-        shutdown_handle.shutdown().await;
+    if let Some(shutdown_handle) = shutdown_handle.as_ref() {
+        shutdown_handle.signal();
     }
 
-    if let Some(dir) = temp_dir
-        && let Err(err) = tokio::fs::remove_dir_all(dir).await
-    {
+    let cleanup_task = runtime.spawn(finish_embedded_server_cleanup(
+        shutdown_handle.take(),
+        temp_dir.take(),
+        release_embedded_runtime_after_drain(runtime_owner),
+    ));
+    if let Err(err) = cleanup_task.await {
         warn!(
             component = LOG_COMPONENT_EMBEDDED,
             subsystem = LOG_SUBSYSTEM_EMBEDDED,
             event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
-            service = "temp_dir",
-            path = %dir.display(),
-            error = %err,
+            service = "server_cleanup",
+            reason = join_failure_reason(&err),
             "Embedded shutdown cleanup failed"
         );
     }
@@ -300,25 +528,16 @@ pub(crate) async fn run_embedded_server_shutdown(
         state = "stopped",
         "Embedded server state changed"
     );
-
-    if let Err(err) = startup_runtime_sources::shutdown_observability_guard() {
-        warn!(
-            component = LOG_COMPONENT_EMBEDDED,
-            subsystem = LOG_SUBSYSTEM_EMBEDDED,
-            event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED,
-            service = "observability",
-            error = %err,
-            "Embedded shutdown cleanup failed"
-        );
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::{
-        BackgroundShutdownStep, background_shutdown_steps, run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown,
+        BackgroundShutdownStep, EmbeddedRuntimeOwners, background_shutdown_steps, finish_embedded_server_cleanup,
+        run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown,
     };
     use crate::server::ShutdownHandle;
+    use std::sync::{Arc, mpsc};
     use std::time::Duration;
     use tokio::sync::broadcast;
     use tokio_util::sync::CancellationToken;
@@ -367,15 +586,152 @@ mod tests {
         let cancel_token = CancellationToken::new();
         let temp_dir = tempfile::tempdir().expect("temp dir should create");
         let temp_path = temp_dir.path().to_path_buf();
+        let mut owned_temp_path = Some(temp_path.clone());
 
-        run_embedded_server_drop_cleanup(&cancel_token, &mut shutdown_handle, Some(temp_dir.path()));
+        run_embedded_server_drop_cleanup(
+            &tokio::runtime::Handle::current(),
+            &cancel_token,
+            &mut shutdown_handle,
+            &mut owned_temp_path,
+            None,
+        );
 
         tokio::time::timeout(Duration::from_secs(1), observed_rx)
             .await
             .expect("drop cleanup should signal shutdown")
             .expect("shutdown signal should be delivered");
+        tokio::time::timeout(Duration::from_secs(1), async {
+            while temp_path.exists() {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("drop cleanup should remove the temporary directory");
         assert!(cancel_token.is_cancelled());
         assert!(shutdown_handle.is_none());
+        assert!(owned_temp_path.is_none());
         assert!(!temp_path.exists());
     }
+
+    #[test]
+    fn only_the_last_embedded_owner_cleans_process_runtime() {
+        let owners = EmbeddedRuntimeOwners::new();
+        assert!(owners.register().is_none());
+        assert!(owners.register().is_none());
+        assert!(owners.release_with(|_| ()).is_none());
+        assert!(owners.release_with(|_| ()).is_some());
+        assert!(owners.release_with(|_| ()).is_none());
+    }
+
+    #[test]
+    fn last_owner_cleanup_intent_is_serialized_before_new_registration() {
+        let owners = Arc::new(EmbeddedRuntimeOwners::new());
+        assert!(owners.register().is_none());
+
+        let (cleanup_entered_tx, cleanup_entered_rx) = mpsc::channel();
+        let (allow_cleanup_tx, allow_cleanup_rx) = mpsc::channel();
+        let release_owners = owners.clone();
+        let release_task = std::thread::spawn(move || {
+            release_owners.release_with(|completion| {
+                cleanup_entered_tx.send(()).expect("cleanup preparation should be observed");
+                allow_cleanup_rx.recv().expect("cleanup preparation should be released");
+                completion.cancel();
+            })
+        });
+        cleanup_entered_rx
+            .recv_timeout(Duration::from_secs(1))
+            .expect("last-owner cleanup should enter preparation");
+
+        let (register_started_tx, register_started_rx) = mpsc::channel();
+        let (register_done_tx, register_done_rx) = mpsc::channel();
+        let register_owners = owners.clone();
+        let register_task = std::thread::spawn(move || {
+            register_started_tx.send(()).expect("registration should start");
+            let pending_cleanup = register_owners.register();
+            register_done_tx
+                .send(pending_cleanup)
+                .expect("registration result should be observed");
+        });
+        register_started_rx
+            .recv_timeout(Duration::from_secs(1))
+            .expect("registration should start");
+        let early_registration = match register_done_rx.try_recv() {
+            Ok(result) => Some(result),
+            Err(mpsc::TryRecvError::Empty) => None,
+            Err(mpsc::TryRecvError::Disconnected) => panic!("registration thread disconnected"),
+        };
+        let registered_before_intent_published = early_registration.is_some();
+
+        allow_cleanup_tx.send(()).expect("cleanup preparation should finish");
+        assert!(release_task.join().expect("release thread should not panic").is_some());
+        let registration = match early_registration {
+            Some(result) => result,
+            None => register_done_rx
+                .recv_timeout(Duration::from_secs(1))
+                .expect("registration should complete"),
+        };
+        register_task.join().expect("register thread should not panic");
+        assert!(!registered_before_intent_published);
+        assert!(registration.is_none());
+        assert_eq!(owners.owner_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn new_owner_waits_for_prior_last_owner_cleanup() {
+        let owners = EmbeddedRuntimeOwners::new();
+        assert!(owners.register().is_none());
+        let cleanup_finished = owners
+            .release_with(|completion| completion)
+            .expect("last owner should publish a cleanup barrier");
+        let pending_cleanup = owners.register().expect("new owner should observe unfinished cleanup");
+
+        let wait_task = tokio::spawn(async move {
+            pending_cleanup.cancelled().await;
+        });
+        tokio::task::yield_now().await;
+        assert!(!wait_task.is_finished());
+
+        cleanup_finished.cancel();
+        wait_task.await.expect("cleanup waiter should not panic");
+    }
+
+    #[tokio::test]
+    async fn server_drain_and_process_runtime_cleanup_finish_before_temp_dir_removal() {
+        let temp_dir = tempfile::tempdir().expect("temp dir should create");
+        let temp_path = temp_dir.path().to_path_buf();
+        let (shutdown_tx, mut shutdown_rx) = broadcast::channel(1);
+        let (shutdown_entered_tx, shutdown_entered_rx) = tokio::sync::oneshot::channel();
+        let (allow_shutdown_tx, allow_shutdown_rx) = tokio::sync::oneshot::channel();
+        let shutdown_task = tokio::spawn(async move {
+            let _ = shutdown_rx.recv().await;
+            shutdown_entered_tx.send(()).expect("shutdown should be observed");
+            allow_shutdown_rx.await.expect("shutdown should be released");
+        });
+        let shutdown_handle = ShutdownHandle::new(shutdown_tx, shutdown_task);
+        let (cleanup_entered_tx, mut cleanup_entered_rx) = tokio::sync::oneshot::channel();
+        let (allow_cleanup_tx, allow_cleanup_rx) = tokio::sync::oneshot::channel();
+
+        let cleanup_task = tokio::spawn(finish_embedded_server_cleanup(
+            Some(shutdown_handle),
+            Some(temp_path.clone()),
+            async move {
+                cleanup_entered_tx.send(()).expect("cleanup should be observed");
+                allow_cleanup_rx.await.expect("cleanup should be released");
+            },
+        ));
+        shutdown_entered_rx.await.expect("shutdown should start");
+        assert!(
+            matches!(cleanup_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
+            "process runtime cleanup must wait for the server to drain"
+        );
+        assert!(temp_path.exists(), "temporary data must remain available while the server drains");
+
+        allow_shutdown_tx.send(()).expect("shutdown should finish");
+        cleanup_entered_rx.await.expect("cleanup should start");
+        assert!(temp_path.exists(), "temporary data must remain available during runtime cleanup");
+
+        allow_cleanup_tx.send(()).expect("cleanup should finish");
+        cleanup_task.await.expect("cleanup task should not panic");
+        assert!(!temp_path.exists(), "temporary data should be removed only after runtime cleanup");
+    }
 }

rustfs/src/storage/mod.rs

@@ -70,9 +70,9 @@ pub(crate) use storage_api::{
     init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients,
     is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class,
     load_bucket_metadata, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy,
-    rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize, set_bucket_metadata,
-    table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata,
-    try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
+    rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, set_bucket_metadata, table_catalog_path_hash,
+    to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
+    try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader,
 };
 
 #[cfg(test)]

rustfs/src/storage/rpc/node_service.rs

@@ -16,15 +16,15 @@ use crate::admin::service::{
     config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
     site_replication::reload_site_replication_runtime_state,
 };
+use crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM;
 use crate::storage::storage_api::ecstore_tier::tier_mutation_peer::{self, TierMutationPeerState as EcTierMutationPeerState};
-#[cfg(test)]
 use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS;
 #[cfg(test)]
 use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
 use crate::storage::storage_api::rpc_consumer::node_service::{
-    DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG,
-    SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
-    reload_transition_tier_config,
+    DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+    SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path,
+    find_local_disk_by_ref, reload_transition_tier_config,
 };
 use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
 use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest};
@@ -32,6 +32,9 @@ use bytes::Bytes;
 use futures::Stream;
 use futures_util::future::join_all;
 use rmp_serde::Deserializer;
+use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
+use rustfs_config::notify::NOTIFY_SUB_SYSTEMS;
+use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
 use rustfs_filemeta::MetacacheReader;
 use rustfs_iam::store::UserType;
 use rustfs_lock::LockClient;
@@ -75,6 +78,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1;
 const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2;
 const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3;
 
+fn supports_dynamic_config_rpc(sub_system: &str) -> bool {
+    NOTIFY_SUB_SYSTEMS.contains(&sub_system)
+        || matches!(
+            sub_system,
+            STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS
+        )
+}
+
 #[derive(Debug)]
 struct HealControlReplayEntry {
     command_digest: [u8; 32],
@@ -1517,6 +1528,18 @@ impl Node for NodeService {
         let raw_signal = vars.get(PEER_RESTSIGNAL).map(String::as_str);
         let signal = raw_signal.and_then(|value| value.parse::<u64>().ok());
         let sub_system = vars.get(PEER_RESTSUB_SYS).map(String::as_str).unwrap_or_default();
+        let dry_run = match vars.get(PEER_RESTDRY_RUN).map(String::as_str) {
+            None => false,
+            Some(value) => match value.parse::<bool>() {
+                Ok(value) => value,
+                Err(_) => {
+                    return Ok(Response::new(SignalServiceResponse {
+                        success: false,
+                        error_info: Some(format!("invalid dry-run value: {value}")),
+                    }));
+                }
+            },
+        };
 
         match signal {
             Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await {
@@ -1524,21 +1547,36 @@ impl Node for NodeService {
                     success: true,
                     error_info: None,
                 })),
-                Err(err) => Ok(Response::new(SignalServiceResponse {
+                Err(_) => Ok(Response::new(SignalServiceResponse {
                     success: false,
-                    error_info: Some(err.to_string()),
-                })),
-            },
-            Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => match reload_dynamic_config_runtime_state(sub_system).await {
-                Ok(()) => Ok(Response::new(SignalServiceResponse {
-                    success: true,
-                    error_info: None,
-                })),
-                Err(err) => Ok(Response::new(SignalServiceResponse {
-                    success: false,
-                    error_info: Some(err.to_string()),
+                    error_info: Some("runtime config snapshot reload failed".to_string()),
                 })),
             },
+            Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => {
+                let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system);
+                if !supported {
+                    return Ok(Response::new(SignalServiceResponse {
+                        success: false,
+                        error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")),
+                    }));
+                }
+                if dry_run {
+                    return Ok(Response::new(SignalServiceResponse {
+                        success: true,
+                        error_info: None,
+                    }));
+                }
+                match reload_dynamic_config_runtime_state(sub_system).await {
+                    Ok(()) => Ok(Response::new(SignalServiceResponse {
+                        success: true,
+                        error_info: None,
+                    })),
+                    Err(_) => Ok(Response::new(SignalServiceResponse {
+                        success: false,
+                        error_info: Some(format!("dynamic config reload failed for {sub_system}")),
+                    })),
+                }
+            }
             Some(other) => Ok(Response::new(SignalServiceResponse {
                 success: false,
                 error_info: Some(format!("unsupported service signal: {other}")),
@@ -1817,12 +1855,12 @@ impl Node for NodeService {
 #[allow(unused_imports)]
 mod tests {
     use super::{
-        CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService, PEER_RESTSIGNAL,
-        PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS,
-        admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager,
-        initialize_heal_topology_fingerprint, make_heal_control_server, make_heal_control_server_with_cache, make_server,
-        make_tier_mutation_control_server_for_context, remove_heal_control_replay, scanner_activity_response,
-        stop_rebalance_response,
+        CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService,
+        PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
+        STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
+        execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server,
+        make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server_for_context,
+        remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
     };
     use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
     use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -4185,6 +4223,44 @@ mod tests {
         assert!(error_info.contains("unsupported dynamic config subsystem: identity_openid"));
     }
 
+    #[test]
+    fn dynamic_config_rpc_allowlist_matches_supported_subsystems() {
+        for sub_system in rustfs_config::notify::NOTIFY_SUB_SYSTEMS {
+            assert!(super::supports_dynamic_config_rpc(sub_system));
+        }
+        for sub_system in [
+            STORAGE_CLASS_SUB_SYS,
+            rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
+            rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
+            rustfs_config::SCANNER_SUB_SYS,
+            rustfs_config::HEAL_SUB_SYS,
+        ] {
+            assert!(super::supports_dynamic_config_rpc(sub_system));
+        }
+        assert!(!super::supports_dynamic_config_rpc("identity_openid"));
+    }
+
+    #[tokio::test]
+    async fn test_signal_service_dry_run_accepts_notify_without_runtime_mutation() {
+        let service = create_test_node_service();
+
+        let mut vars = HashMap::new();
+        vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
+        vars.insert(PEER_RESTSUB_SYS.to_string(), rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_string());
+        vars.insert(PEER_RESTDRY_RUN.to_string(), true.to_string());
+
+        let response = service
+            .signal_service(Request::new(SignalServiceRequest {
+                vars: Some(Mss { value: vars }),
+            }))
+            .await
+            .expect("notify capability probe should return a response")
+            .into_inner();
+
+        assert!(response.success, "new nodes must advertise notify lifecycle reload support");
+        assert!(response.error_info.is_none());
+    }
+
     #[tokio::test]
     #[ignore = "requires isolated global object layer state"]
     #[serial_test::serial]
@@ -4204,7 +4280,7 @@ mod tests {
         let signal_response = response.unwrap().into_inner();
         assert!(!signal_response.success);
         let error_info = signal_response.error_info.expect("expected error info");
-        assert!(error_info.contains("storage layer not initialized"));
+        assert_eq!(error_info, "runtime config snapshot reload failed");
     }
 
     #[tokio::test]
@@ -4227,7 +4303,7 @@ mod tests {
         let signal_response = response.unwrap().into_inner();
         assert!(!signal_response.success);
         let error_info = signal_response.error_info.expect("expected error info");
-        assert!(error_info.contains("storage layer not initialized"));
+        assert_eq!(error_info, format!("dynamic config reload failed for {STORAGE_CLASS_SUB_SYS}"));
     }
 
     fn assert_unimplemented_status<T>(response: Result<Response<T>, Status>, method: &str) {

rustfs/src/storage/storage_api.rs

@@ -220,11 +220,11 @@ pub(crate) mod rpc_consumer {
     pub(crate) mod node_service {
         pub(crate) use super::super::{
             BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore,
-            ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq,
-            ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt,
-            StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref,
-            get_local_server_property, load_bucket_metadata, reload_transition_tier_config, remove_bucket_metadata,
-            set_bucket_metadata, validate_batch_read_version_item_count,
+            ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
+            ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
+            StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics,
+            find_local_disk_by_ref, get_local_server_property, load_bucket_metadata, reload_transition_tier_config,
+            remove_bucket_metadata, set_bucket_metadata, validate_batch_read_version_item_count,
         };
         pub(crate) type StorageResult<T> = super::super::Result<T>;
 
@@ -233,7 +233,6 @@ pub(crate) mod rpc_consumer {
         #[cfg(test)]
         pub(crate) type HealBucketInfo = super::super::contract::bucket::BucketInfo;
 
-        #[cfg(test)]
         pub(crate) const STORAGE_CLASS_SUB_SYS: &str = super::super::STORAGE_CLASS_SUB_SYS;
 
         pub(crate) mod contract {
@@ -479,9 +478,9 @@ pub(crate) mod ecstore_rio {
 
 pub(crate) mod ecstore_rpc {
     pub(crate) use rustfs_ecstore::api::rpc::{
-        LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
-        SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, sign_tonic_rpc_response_proof,
-        verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
+        LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
+        SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
+        sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature,
     };
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::rpc::{
@@ -533,6 +532,7 @@ pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCK
 pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG;
 pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE;
 pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG;
+pub(crate) const PEER_RESTDRY_RUN: &str = ecstore_rpc::PEER_RESTDRY_RUN;
 pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL;
 pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS;
 pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG;
@@ -554,7 +554,6 @@ pub(crate) use ecstore_rpc::sign_tonic_rpc_response_proof;
 #[cfg(test)]
 pub(crate) use ecstore_rpc::verify_tonic_rpc_response_proof;
 
-#[cfg(test)]
 pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS;
 
 pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata;
@@ -873,6 +872,14 @@ pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>
     ecstore_config::com::read_config(api, file).await
 }
 
+pub(crate) async fn read_config_no_lock(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
+    ecstore_config::com::read_config_no_lock(api, file).await
+}
+
+pub(crate) async fn read_existing_server_config_no_lock(api: Arc<ECStore>) -> Result<rustfs_config::server_config::Config> {
+    ecstore_config::com::read_existing_server_config_no_lock(api).await
+}
+
 pub(crate) async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc<InstanceContext>) {
     ecstore_storage::prewarm_local_disk_id_map_with_instance_ctx(instance_ctx).await;
 }
@@ -881,8 +888,35 @@ pub(crate) fn replication_queue_current_count() -> Option<i64> {
     get_global_replication_stats().and_then(|stats| stats.queue_current_count())
 }
 
-pub(crate) async fn save_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
-    ecstore_config::com::save_config(api, file, data).await
+pub(crate) async fn save_config_no_lock(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
+    ecstore_config::com::save_config_no_lock(api, file, data).await
+}
+
+pub(crate) async fn with_config_object_write_lock<F, Fut, T>(api: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    ecstore_config::com::with_config_object_write_lock(api, object, operation).await
+}
+
+pub(crate) async fn with_config_object_read_lock<F, Fut, T>(api: Arc<ECStore>, object: String, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    ecstore_config::com::with_config_object_read_lock(api, object, operation).await
+}
+
+pub(crate) async fn with_server_config_read_lock<F, Fut, T>(api: Arc<ECStore>, operation: F) -> Result<T>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    ecstore_config::com::with_server_config_read_lock(api, operation).await
 }
 
 pub(crate) fn shutdown_background_services() {

rustfs/src/storage_api.rs

@@ -89,7 +89,10 @@ pub(crate) mod server {
             }
         }
 
-        pub(crate) use crate::storage::storage_api::{EventArgs, StorageObjectInfo, register_event_dispatch_hook};
+        pub(crate) use crate::storage::storage_api::{
+            EventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
+            with_server_config_read_lock,
+        };
     }
 
     pub(crate) mod http {
@@ -151,7 +154,10 @@ pub(crate) mod server {
     }
 
     pub(crate) mod module_switch {
-        pub(crate) use crate::storage::storage_api::{Error, read_config, save_config};
+        pub(crate) use crate::storage::storage_api::{
+            Error, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock,
+            with_config_object_write_lock,
+        };
     }
 
     pub(crate) mod readiness {

rustfs/tests/embedded_notification_lifecycle_test.rs

@@ -0,0 +1,102 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use aws_sdk_s3::config::{Credentials, Region};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::{Client, Config};
+use rustfs::embedded::{RustFSServerBuilder, find_available_port};
+use rustfs_notify::{NotificationRuntimeState, notification_system};
+
+fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
+    let credentials = Credentials::new(access_key, secret_key, None, None, "test");
+    let config = Config::builder()
+        .credentials_provider(credentials)
+        .region(Region::new("us-east-1"))
+        .endpoint_url(endpoint)
+        .force_path_style(true)
+        .behavior_version_latest()
+        .build();
+    Client::from_conf(config)
+}
+
+#[tokio::test]
+async fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains() {
+    temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async {
+        let port_a = match find_available_port() {
+            Ok(port) => port,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+            Err(err) => panic!("find free port for server A: {err}"),
+        };
+        let server_a = RustFSServerBuilder::new()
+            .address(format!("127.0.0.1:{port_a}"))
+            .access_key("shared-access")
+            .secret_key("shared-secret")
+            .build()
+            .await
+            .expect("start embedded server A");
+
+        let port_b = match find_available_port() {
+            Ok(port) => port,
+            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
+                server_a.shutdown().await;
+                return;
+            }
+            Err(err) => {
+                server_a.shutdown().await;
+                panic!("find free port for server B: {err}");
+            }
+        };
+        let server_b = RustFSServerBuilder::new()
+            .address(format!("127.0.0.1:{port_b}"))
+            .access_key("shared-access")
+            .secret_key("shared-secret")
+            .build()
+            .await
+            .expect("start embedded server B");
+
+        let notification = notification_system().expect("embedded startup should initialize notification runtime");
+        let active_state = notification.runtime_lifecycle_state();
+        assert!(matches!(active_state, NotificationRuntimeState::TargetsEnabled { .. }));
+        assert!(notification.runtime_lifecycle_is_converged());
+
+        server_b.shutdown().await;
+        assert_eq!(
+            notification.runtime_lifecycle_state(),
+            active_state,
+            "one owner must not suspend the shared notification runtime"
+        );
+        assert!(notification.runtime_lifecycle_is_converged());
+
+        let client_a = s3_client(&server_a.endpoint(), server_a.access_key(), server_a.secret_key());
+        client_a
+            .create_bucket()
+            .bucket("survives-notify-owner-shutdown")
+            .send()
+            .await
+            .expect("server A should remain usable after server B shuts down");
+        client_a
+            .put_object()
+            .bucket("survives-notify-owner-shutdown")
+            .key("marker.txt")
+            .body(ByteStream::from_static(b"still here"))
+            .send()
+            .await
+            .expect("server A should still write after server B shuts down");
+
+        server_a.shutdown().await;
+        assert_eq!(notification.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
+        assert!(notification.runtime_lifecycle_is_converged());
+    })
+    .await;
+}