fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination * fix(notify): repair lifecycle convergence checks * fix(admin): expose effective notify state (#5097)
2026-08-13 08:36:54 +00:00 · 2026-07-22 13:01:15 +08:00
parent 0adb3c5ea1
commit 1655f3192e
66 changed files with 8091 additions and 1370 deletions
@@ -13,89 +13,83 @@
 // limitations under the License.

 use crate::{
-    Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine,
+    NotificationError,
+    lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator},
+    registry::TargetRegistry,
+    resolve_notify_object_store_handle,
+    rule_engine::NotifyRuleEngine,
    runtime_facade::NotifyRuntimeFacade,
+    with_notify_server_config_read_lock, with_notify_server_config_write_lock,
 };
 use rustfs_config::notify::{
    NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
    NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::server_config::{Config, KVS};
-use rustfs_targets::{Target, arn::TargetID};
-use std::sync::{Arc, LazyLock};
-use tokio::sync::{Mutex, RwLock};
+use rustfs_targets::arn::TargetID;
+use std::sync::Arc;
+use tokio::sync::RwLock;
 use tracing::{debug, info};

-/// Serializes the read-modify-write sequence over the persisted notify server
-/// config. The persisted config is a single process-global resource (there is
-/// only one backing object store), so without this guard two concurrent updates
-/// can both read the same base config, apply disjoint changes, and race their
-/// full-config writes — the later write silently overwrites the earlier one,
-/// losing updates. Holding this mutex across the whole read→modify→write makes
-/// concurrent updates apply serially so every change is preserved.
-///
-/// The lock is only ever acquired inside `update_server_config`; it never nests
-/// with the per-manager `config` RwLock (the in-memory reload runs after this
-/// guard is released), so it introduces no lock-ordering risk.
-static NOTIFY_CONFIG_RMW_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));
-
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_CONFIG: &str = "config";
-const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update";

 #[derive(Debug)]
 enum NotifyConfigStoreError {
+    Lock(String),
    StorageNotAvailable,
    Read(String),
    Save(String),
 }

-async fn update_server_config<F>(modifier: F) -> Result<Option<Config>, NotifyConfigStoreError>
+async fn update_server_config<F>(
+    modifier: F,
+    lifecycle: NotifyLifecycleCoordinator,
+) -> Result<Option<crate::lifecycle::NotificationLifecycleTransition>, NotifyConfigStoreError>
 where
-    F: FnMut(&mut Config) -> bool,
+    F: FnMut(&mut Config) -> bool + Send + 'static,
 {
    let Some(store) = resolve_notify_object_store_handle() else {
        return Err(NotifyConfigStoreError::StorageNotAvailable);
    };

+    let store_for_read = store.clone();
    let store_for_save = store.clone();
-    serialized_read_modify_write(
-        modifier,
-        move || async move {
-            crate::read_notify_server_config_without_migrate(store)
-                .await
-                .map_err(NotifyConfigStoreError::Read)
-        },
-        move |config| async move {
-            crate::save_notify_server_config(store_for_save, &config)
-                .await
-                .map_err(NotifyConfigStoreError::Save)
-        },
-    )
+    with_notify_server_config_write_lock(store, move || {
+        read_modify_write(
+            modifier,
+            move || async move {
+                crate::read_notify_server_config_without_migrate_no_lock(store_for_read)
+                    .await
+                    .map_err(NotifyConfigStoreError::Read)
+            },
+            move |config| async move {
+                crate::save_notify_server_config_no_lock(store_for_save, &config)
+                    .await
+                    .map_err(NotifyConfigStoreError::Save)
+            },
+            move |config| lifecycle.update_config(config),
+        )
+    })
    .await
+    .map_err(NotifyConfigStoreError::Lock)?
 }

-/// Runs a `read → modify → write` over the persisted notify config while holding
-/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber
-/// each other's changes (backlog#968). `read`/`save` are injected so the exact
-/// production serialization path can be exercised in tests without a live store.
-async fn serialized_read_modify_write<F, R, RFut, S, SFut>(
+async fn read_modify_write<F, R, RFut, S, SFut, P, T>(
    mut modifier: F,
    read: R,
    save: S,
-) -> Result<Option<Config>, NotifyConfigStoreError>
+    publish: P,
+) -> Result<Option<T>, NotifyConfigStoreError>
 where
    F: FnMut(&mut Config) -> bool,
    R: FnOnce() -> RFut,
    RFut: std::future::Future<Output = Result<Config, NotifyConfigStoreError>>,
    S: FnOnce(Config) -> SFut,
    SFut: std::future::Future<Output = Result<(), NotifyConfigStoreError>>,
+    P: FnOnce(Config) -> T,
 {
-    // Hold the RMW lock across the entire read→modify→write so concurrent
-    // updates serialize and cannot clobber each other's changes (backlog#968).
-    let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await;
-
    let mut new_config = read().await?;

    if !modifier(&mut new_config) {
@@ -104,7 +98,7 @@ where

    save(new_config.clone()).await?;

-    Ok(Some(new_config))
+    Ok(Some(publish(new_config)))
 }

 pub(crate) fn notify_configuration_hint() -> String {
@@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) ->
 #[derive(Clone)]
 pub struct NotifyConfigManager {
    config: Arc<RwLock<Config>>,
-    registry: Arc<TargetRegistry>,
+    lifecycle: NotifyLifecycleCoordinator,
    rule_engine: NotifyRuleEngine,
-    runtime_facade: NotifyRuntimeFacade,
 }

 impl NotifyConfigManager {
@@ -154,64 +147,21 @@ impl NotifyConfigManager {
        rule_engine: NotifyRuleEngine,
        runtime_facade: NotifyRuntimeFacade,
    ) -> Self {
+        let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade);
        Self {
            config,
-            registry,
+            lifecycle,
            rule_engine,
-            runtime_facade,
        }
    }

+    pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator {
+        self.lifecycle.clone()
+    }
+
    pub async fn init(&self) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initializing",
-            "notify runtime lifecycle"
-        );
-
-        let config = {
-            let guard = self.config.read().await;
-            debug!(
-                subsystem_count = guard.0.len(),
-                "Initializing notification system with configuration summary"
-            );
-            guard.clone()
-        };
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self.registry.create_targets_from_config(&config).await?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
-        }
-
-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "initialized",
-            "notify runtime lifecycle"
-        );
-        Ok(())
+        let config = self.config.read().await.clone();
+        self.lifecycle.set_mode(true, Some(config)).wait().await
    }

    pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
@@ -227,6 +177,7 @@ impl NotifyConfigManager {

        let ttype = target_type.to_lowercase();
        let tname = target_id.id.to_lowercase();
+        let log_target_id = target_id.clone();

        // Guard against orphaning bucket notification rules (backlog#979). Removing a
        // target while a bucket rule still references it would leave a dangling
@@ -240,7 +191,7 @@ impl NotifyConfigManager {
            )));
        }

-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            let mut changed = false;
            if let Some(targets_of_type) = config.0.get_mut(&ttype) {
                if targets_of_type.remove(&tname).is_some() {
@@ -249,7 +200,7 @@ impl NotifyConfigManager {
                            component = LOG_COMPONENT_NOTIFY,
                        subsystem = LOG_SUBSYSTEM_CONFIG,
                        action = "remove_target",
-                        target_id = %target_id,
+                        target_id = %log_target_id,
                        result = "removed",
                        "notify config update"
                    );
@@ -265,7 +216,7 @@ impl NotifyConfigManager {
                    component = LOG_COMPONENT_NOTIFY,
                    subsystem = LOG_SUBSYSTEM_CONFIG,
                    action = "remove_target",
-                    target_id = %target_id,
+                    target_id = %log_target_id,
                    result = "not_found",
                    "notify config update"
                );
@@ -287,7 +238,7 @@ impl NotifyConfigManager {
        );
        let ttype = target_type.to_lowercase();
        let tname = target_name.to_lowercase();
-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone());
            true
        })
@@ -316,7 +267,7 @@ impl NotifyConfigManager {
            )));
        }

-        self.update_config_and_reload(|config| {
+        self.update_config_and_reload(move |config| {
            let mut changed = false;
            if let Some(targets) = config.0.get_mut(&ttype) {
                if targets.remove(&tname).is_some() {
@@ -332,8 +283,8 @@ impl NotifyConfigManager {
                    component = LOG_COMPONENT_NOTIFY,
                    subsystem = LOG_SUBSYSTEM_CONFIG,
                    action = "remove_target_config",
-                    target_type = %target_type,
-                    target_name = %target_name,
+                    target_type = %ttype,
+                    target_name = %tname,
                    result = "not_found",
                    "notify config update"
                );
@@ -348,81 +299,62 @@ impl NotifyConfigManager {
    }

    pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> {
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloading",
-            "notify runtime lifecycle"
-        );
+        self.lifecycle.set_mode(true, Some(new_config)).wait().await
+    }

-        self.update_config(new_config.clone()).await;
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        let Some(store) = resolve_notify_object_store_handle() else {
+            return Err(NotificationError::StorageNotAvailable(
+                "Failed to load target configuration: server storage not initialized".to_string(),
+            ));
+        };
+        self.reload_persisted_config_from_store(store).await
+    }

-        // Stop the currently running replay workers *before* activating the new ones
-        // (backlog#970). Each replay worker drains a per-target persisted store; if the
-        // new workers start while the old ones are still running against the same
-        // stores, both drain the same queues and re-deliver events. `replace_targets`
-        // below also stops workers, but only after `activate_targets_with_replay` has
-        // already spawned the new ones — so without this explicit stop-before-start
-        // there is a window where old and new workers overlap. (The full "signal +
-        // join" shutdown lives in the targets crate and is tracked under the same
-        // issue; this reorders the notify-side lifecycle.)
-        self.runtime_facade.stop_replay_workers().await;
-
-        let targets: Vec<Box<dyn Target<Event> + Send + Sync>> = self
-            .registry
-            .create_targets_from_config(&new_config)
-            .await
-            .map_err(NotificationError::Target)?;
-
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "targets_created",
-            target_count = targets.len(),
-            "notify runtime lifecycle"
-        );
-        if targets.is_empty() {
-            debug!(
-                event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-                component = LOG_COMPONENT_NOTIFY,
-                subsystem = LOG_SUBSYSTEM_CONFIG,
-                state = "idle",
-                reason = "no_targets_configured",
-                hint = %notify_configuration_hint(),
-                "notify runtime lifecycle"
-            );
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
        }

-        let activation = self.runtime_facade.activate_targets_with_replay(targets).await;
-        self.runtime_facade.replace_targets(activation).await?;
-        info!(
-            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
-            component = LOG_COMPONENT_NOTIFY,
-            subsystem = LOG_SUBSYSTEM_CONFIG,
-            state = "reloaded",
-            "notify runtime lifecycle"
-        );
+        let read_store = store.clone();
+        let config_cache = self.config.clone();
+        let lifecycle = self.lifecycle.clone();
+        let transition = with_notify_server_config_read_lock(store, move || async move {
+            let config = crate::read_existing_notify_server_config_no_lock(read_store)
+                .await
+                .map_err(NotificationError::ReadConfig)?;
+            Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() {
+                None
+            } else {
+                Some(lifecycle.update_config(config))
+            })
+        })
+        .await
+        .map_err(NotificationError::StorageNotAvailable)??;
+
+        if let Some(transition) = transition {
+            transition.wait().await?;
+        }
        Ok(())
    }

-    async fn update_config(&self, new_config: Config) {
-        let mut config = self.config.write().await;
-        *config = new_config;
-    }
-
-    async fn update_config_and_reload<F>(&self, mut modifier: F) -> Result<(), NotificationError>
+    async fn update_config_and_reload<F>(&self, modifier: F) -> Result<(), NotificationError>
    where
-        F: FnMut(&mut Config) -> bool,
+        F: FnMut(&mut Config) -> bool + Send + 'static,
    {
-        let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err {
-            NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
-                "Failed to save target configuration: server storage not initialized".to_string(),
-            ),
-            NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
-            NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
-        })?
+        if self.lifecycle.state() == NotificationRuntimeState::Terminated {
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        let Some(transition) = update_server_config(modifier, self.lifecycle.clone())
+            .await
+            .map_err(|err| match err {
+                NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err),
+                NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable(
+                    "Failed to save target configuration: server storage not initialized".to_string(),
+                ),
+                NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err),
+                NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err),
+            })?
        else {
            debug!(
                event = EVENT_NOTIFY_CONFIG_UPDATE,
@@ -443,15 +375,15 @@ impl NotifyConfigManager {
            result = "updated",
            "notify config update"
        );
-        self.reload_config(new_config).await
+        transition.wait().await
    }
 }

 #[cfg(test)]
 mod tests {
-    use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write};
-    use crate::NotificationError;
+    use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem};
    use crate::rules::RulesMap;
+    use crate::{NotificationError, NotificationRuntimeState};
    use crate::{
        integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
        runtime_facade::NotifyRuntimeFacade,
@@ -464,7 +396,10 @@ mod tests {
    use rustfs_s3_types::EventName;
    use rustfs_targets::ReplayWorkerManager;
    use rustfs_targets::arn::TargetID;
-    use std::sync::Arc;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicBool, Ordering},
+    };
    use tokio::sync::{RwLock, Semaphore};

    fn build_manager() -> NotifyConfigManager {
@@ -474,9 +409,10 @@ mod tests {
        let rule_engine = NotifyRuleEngine::new();
        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
        let target_list = notifier.target_list();
-        let runtime_facade = NotifyRuntimeFacade::new(
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
            target_list,
            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
            Arc::new(Semaphore::new(4)),
            metrics,
        );
@@ -524,78 +460,39 @@ mod tests {
            .reload_config(Config::default())
            .await
            .expect("reload_config should succeed for empty targets");
+        assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. }));
    }

-    // Regression test for backlog#968: the read-modify-write over the persisted
-    // notify config must be serialized. Many tasks concurrently add a distinct
-    // target through the same production RMW path (`serialized_read_modify_write`,
-    // which holds the global RMW lock across read→modify→write) against a shared
-    // in-memory backend. Every update must survive — no lost updates. Without the
-    // lock, concurrent tasks would read the same base config and clobber each
-    // other's writes, leaving only a subset of targets.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_config_updates_preserve_all_targets() {
-        // Shared in-memory stand-in for the persisted config backend.
-        let backend = Arc::new(RwLock::new(Config::default()));
-        const TASKS: usize = 32;
+    #[tokio::test]
+    async fn read_modify_write_publishes_only_after_save() {
+        let saved = Arc::new(AtomicBool::new(false));
+        let saved_by_writer = saved.clone();
+        let observed_by_publisher = saved.clone();

-        let mut handles = Vec::with_capacity(TASKS);
-        for idx in 0..TASKS {
-            let backend = backend.clone();
-            handles.push(tokio::spawn(async move {
-                let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase();
-                let tname = format!("target-{idx}");
-
-                let read_backend = backend.clone();
-                let save_backend = backend.clone();
-
-                let result = serialized_read_modify_write(
-                    |config: &mut Config| {
-                        config
-                            .0
-                            .entry(ttype.clone())
-                            .or_default()
-                            .insert(tname.clone(), KVS::default());
-                        true
-                    },
-                    move || async move {
-                        let snapshot = read_backend.read().await.clone();
-                        // Yield inside the critical section to widen the race window:
-                        // if the RMW were not serialized, other tasks would read this
-                        // same base snapshot and their writes would clobber ours.
-                        tokio::task::yield_now().await;
-                        Ok::<_, NotifyConfigStoreError>(snapshot)
-                    },
-                    move |config: Config| async move {
-                        *save_backend.write().await = config;
-                        Ok::<_, NotifyConfigStoreError>(())
-                    },
-                )
-                .await
-                .expect("serialized RMW should succeed");
-                assert!(result.is_some(), "modifier reported a change, expected Some(config)");
-            }));
-        }
-
-        for handle in handles {
-            handle.await.expect("update task should not panic");
-        }
-
-        let final_config = backend.read().await;
-        let webhook_targets = final_config
-            .0
-            .get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase())
-            .expect("webhook subsystem should exist after updates");
-
-        assert_eq!(
-            webhook_targets.len(),
-            TASKS,
-            "all concurrent target additions must be preserved (no lost updates)"
-        );
-        for idx in 0..TASKS {
-            let tname = format!("target-{idx}");
-            assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost");
-        }
+        read_modify_write(
+            |config| {
+                config
+                    .0
+                    .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+                    .or_default()
+                    .insert("primary".to_string(), KVS::default());
+                true
+            },
+            || async { Ok::<_, NotifyConfigStoreError>(Config::default()) },
+            move |_config| async move {
+                saved_by_writer.store(true, Ordering::Release);
+                Ok::<_, NotifyConfigStoreError>(())
+            },
+            move |_config| {
+                assert!(
+                    observed_by_publisher.load(Ordering::Acquire),
+                    "publication must observe the completed save"
+                );
+            },
+        )
+        .await
+        .expect("read-modify-write should succeed")
+        .expect("changed config should publish");
    }

    #[test]
@@ -15,6 +15,7 @@
 use rustfs_targets::{TargetError, arn::TargetID};
 use std::io;
 use thiserror::Error;
+use tokio::task::JoinError;

 /// Errors related to the notification system's lifecycle.
 #[derive(Debug, Error)]
@@ -70,3 +71,24 @@ pub enum NotificationError {
    #[error("Storage not available: {0}")]
    StorageNotAvailable(String),
 }
+
+pub(crate) fn transition_join_error(error: JoinError) -> NotificationError {
+    let reason = if error.is_cancelled() { "cancelled" } else { "panicked" };
+    NotificationError::Initialization(format!("Notification lifecycle transition task {reason}"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::transition_join_error;
+
+    #[tokio::test]
+    async fn transition_join_error_does_not_expose_panic_payload() {
+        let join_error = tokio::spawn(async { panic!("do-not-expose-payload") })
+            .await
+            .expect_err("test task should panic");
+
+        let rendered = transition_join_error(join_error).to_string();
+        assert!(rendered.contains("panicked"));
+        assert!(!rendered.contains("do-not-expose-payload"));
+    }
+}
@@ -14,38 +14,86 @@

 use crate::{
    BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot,
-    NotificationSystem, NotificationTargetMetricSnapshot,
+    NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error,
 };
 use rustfs_config::server_config::Config;
 use rustfs_s3_types::EventName;
 use rustfs_targets::arn::TargetID;
-use std::sync::{Arc, OnceLock};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak};
 use tracing::error;

 static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
+static LEGACY_INITIALIZATION: LazyLock<Mutex<Option<LegacyInitialization>>> = LazyLock::new(|| Mutex::new(None));
+
+enum LegacyInitialization {
+    Initializing(Weak<NotificationSystem>),
+    Retryable(Weak<NotificationSystem>),
+    Initialized,
+}
 const LOG_COMPONENT_NOTIFY: &str = "notify";
 const LOG_SUBSYSTEM_GLOBAL: &str = "global";
 const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state";

-/// Initialize the global notification system with the given configuration.
-/// This function should only be called once throughout the application life cycle.
-pub async fn initialize(config: Config) -> Result<(), NotificationError> {
-    // `new` is synchronous and responsible for creating instances
-    let system = NotificationSystem::new(config);
-    // `init` is asynchronous and responsible for performing I/O-intensive initialization
-    system.init().await?;
+fn notification_system_or_init(config: Config) -> Arc<NotificationSystem> {
+    NOTIFICATION_SYSTEM
+        .get_or_init(|| Arc::new(NotificationSystem::new(config)))
+        .clone()
+}

-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(losing_system) => {
-            // Another initializer won the race. `init()` above already started this
-            // system's targets and replay workers, so simply dropping it would leak
-            // those background tasks. Shut the losing instance down cleanly before
-            // reporting the conflict (backlog#984).
-            losing_system.shutdown().await;
-            Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized))
+/// Initialize the global notification system with the given configuration.
+///
+/// This preserves the historical one-shot API contract. Server lifecycle code
+/// that needs idempotent reconciliation should use [`reconcile`] instead.
+pub async fn initialize(config: Config) -> Result<(), NotificationError> {
+    let system = {
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        match legacy.as_ref() {
+            Some(LegacyInitialization::Retryable(system)) => {
+                let Some(system) = system.upgrade() else {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                };
+                if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
+            Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => {
+                return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+            }
+            None => {
+                if NOTIFICATION_SYSTEM.get().is_some() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                let system = Arc::new(NotificationSystem::new(config.clone()));
+                if NOTIFICATION_SYSTEM.set(system.clone()).is_err() {
+                    return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
+                }
+                *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system)));
+                system
+            }
        }
-    }
+    };
+
+    let task_system = system.clone();
+    tokio::spawn(async move {
+        let result = task_system.set_targets_enabled(true, Some(config)).await;
+        let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
+        if matches!(
+            legacy.as_ref(),
+            Some(LegacyInitialization::Initializing(current))
+                if current.upgrade().is_some_and(|current| Arc::ptr_eq(&current, &task_system))
+        ) {
+            *legacy = Some(if result.is_ok() {
+                LegacyInitialization::Initialized
+            } else {
+                LegacyInitialization::Retryable(Arc::downgrade(&task_system))
+            });
+        }
+        result
+    })
+    .await
+    .map_err(transition_join_error)?
 }

 /// Initialize the global notification system only for live in-process consumers.
@@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
 /// ListenBucketNotification clients can receive live events even when external
 /// notification targets are disabled.
 pub fn initialize_live_events() -> Result<(), NotificationError> {
-    let system = NotificationSystem::new(Config::new());
-
-    match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
-        Ok(_) => Ok(()),
-        Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)),
+    if NOTIFICATION_SYSTEM
+        .set(Arc::new(NotificationSystem::new(Config::new())))
+        .is_err()
+    {
+        return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized));
    }
+    Ok(())
+}
+
+/// Ensures the stable process-wide live-event container exists.
+pub fn ensure_live_events() -> Arc<NotificationSystem> {
+    notification_system_or_init(Config::new())
+}
+
+/// Ensures the stable singleton exists and reconciles its target runtime.
+pub async fn reconcile(config: Config) -> Result<(), NotificationError> {
+    let system = notification_system_or_init(config.clone());
+    system.set_targets_enabled(true, Some(config)).await
 }

 /// Returns a handle to the global NotificationSystem instance.
@@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView;
 use crate::notifier::{EventNotifier, TargetList};
 use crate::services::NotifyServices;
 use crate::{
-    Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine,
+    Event,
+    error::NotificationError,
+    lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState},
+    pipeline::LiveEventHistory,
+    registry::TargetRegistry,
+    rule_engine::NotifyRuleEngine,
    rules::BucketNotificationConfig,
 };
 use hashbrown::HashMap;
@@ -160,11 +165,12 @@ impl NotificationMetrics {

 /// The notification system that integrates all components
 pub struct NotificationSystem {
-    /// The event notifier
+    /// Event dispatcher. Runtime target mutation remains lifecycle-owned.
    pub notifier: Arc<EventNotifier>,
-    /// The target registry
+    /// Target factory registry. Creating a target does not publish it.
    pub registry: Arc<TargetRegistry>,
-    /// The current configuration
+    /// The current cached configuration. Runtime publication must still go
+    /// through the lifecycle methods on this type.
    pub config: Arc<RwLock<Config>>,
    services: NotifyServices,
 }
@@ -220,10 +226,12 @@ impl NotificationSystem {
        self.services.runtime_view.get_active_targets().await
    }

-    /// Gets the complete Target list, including both active and inactive Targets.
-    ///
-    /// # Return
-    /// An `Arc<RwLock<TargetList>>` containing all Targets.
+    pub async fn config_snapshot(&self) -> Config {
+        self.config.read().await.clone()
+    }
+
+    /// Gets a read-only runtime container handle. Public mutation methods on
+    /// `TargetList` are intentionally unavailable outside this crate.
    pub async fn get_all_targets(&self) -> Arc<RwLock<TargetList>> {
        self.services.runtime_view.get_all_targets()
    }
@@ -327,6 +335,43 @@ impl NotificationSystem {
        self.services.config_manager.reload_config(new_config).await
    }

+    /// Synchronously publishes a config generation without changing target mode.
+    pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().update_config(new_config)
+    }
+
+    /// Reconciles the cached and active configuration with the persisted
+    /// server config without changing the target-runtime mode.
+    pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config().await
+    }
+
+    /// Reconciles from an explicitly selected storage context.
+    pub async fn reload_persisted_config_from_store(&self, store: Arc<crate::NotifyStore>) -> Result<(), NotificationError> {
+        self.services.config_manager.reload_persisted_config_from_store(store).await
+    }
+
+    /// Enables or suspends configured notification targets without replacing
+    /// the process-wide live-event container.
+    pub async fn set_targets_enabled(&self, enabled: bool, config: Option<Config>) -> Result<(), NotificationError> {
+        self.publish_targets_enabled(enabled, config).wait().await
+    }
+
+    /// Synchronously accepts a target-runtime mode transition. The returned
+    /// receipt can be awaited after the caller releases its persistence lock.
+    pub fn publish_targets_enabled(&self, enabled: bool, config: Option<Config>) -> NotificationLifecycleTransition {
+        self.services.config_manager.lifecycle().set_mode(enabled, config)
+    }
+
+    pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState {
+        self.services.config_manager.lifecycle().state()
+    }
+
+    /// Returns whether the latest accepted lifecycle generation is active.
+    pub fn runtime_lifecycle_is_converged(&self) -> bool {
+        self.services.config_manager.lifecycle().is_converged()
+    }
+
    /// Loads the bucket notification configuration
    pub async fn load_bucket_notification_config(
        &self,
@@ -365,9 +410,13 @@ impl NotificationSystem {
        self.services.runtime_view.runtime_status_snapshot().await
    }

-    // Add a method to shut down the system
    pub async fn shutdown(&self) {
-        self.services.runtime_facade.shutdown().await;
+        let _ = self.shutdown_checked().await;
+    }
+
+    /// Irreversibly terminates the notification target runtime for this process.
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        self.services.config_manager.lifecycle().terminate().wait().await
    }
 }

@@ -25,6 +25,7 @@ mod event;
 pub mod factory;
 mod global;
 pub mod integration;
+mod lifecycle;
 mod notification_system_subscriber;
 pub mod notifier;
 mod pipeline;
@@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
 pub use error::{LifecycleError, NotificationError};
 pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
 pub use global::{
-    initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
-    notification_target_metrics, notifier_global,
+    ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot,
+    notification_system, notification_target_metrics, notifier_global, reconcile,
 };
 pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot};
+pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState};
 pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline};
 pub use rule_engine::NotifyRuleEngine;
 pub use rules::BucketNotificationConfig;
@@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade;
 pub use runtime_view::NotifyRuntimeView;
 pub use services::NotifyServices;
 pub use status_view::NotifyStatusView;
+pub use storage_api::NotifyStore;
 pub(crate) use storage_api::crate_boundary::{
-    read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+    read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+    resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+    with_notify_server_config_write_lock,
 };
@@ -12,13 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
+use crate::error::NotificationError;
+use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine};
 use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY};
+use rustfs_targets::Target;
 use rustfs_targets::arn::TargetID;
 use rustfs_targets::target::EntityTarget;
-use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager};
-use std::sync::Arc;
-use tokio::sync::{RwLock, Semaphore};
+use rustfs_targets::{SharedTarget, TargetRuntimeManager};
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak};
+use tokio::sync::{RwLock, Semaphore, watch};
+use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, instrument, warn};

 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started";
 const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";

+async fn wait_for_dispatch_tasks(handles: Vec<JoinHandle<()>>) {
+    for handle in handles {
+        if let Err(e) = handle.await {
+            let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" };
+            error!(
+                event = EVENT_NOTIFY_DISPATCH_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_DISPATCH,
+                reason,
+                "Notify dispatch task failed"
+            );
+        }
+    }
+}
+
 pub type SharedNotifyTargetList = Arc<RwLock<TargetList>>;

+static TARGET_LIST_DISPATCH_GATES: LazyLock<StdMutex<HashMap<usize, Weak<RwLock<()>>>>> =
+    LazyLock::new(|| StdMutex::new(HashMap::new()));
+
+pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option<Arc<RwLock<()>>>) -> Arc<RwLock<()>> {
+    let key = Arc::as_ptr(target_list) as usize;
+    let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner());
+    gates.retain(|_, gate| gate.strong_count() != 0);
+    if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) {
+        return gate;
+    }
+    let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(())));
+    gates.insert(key, Arc::downgrade(&gate));
+    gate
+}
+
+pub(crate) struct DirectDispatchTracker {
+    cancellation: CancellationToken,
+    inflight: watch::Sender<usize>,
+}
+
+impl DirectDispatchTracker {
+    fn new() -> Self {
+        let (inflight, _) = watch::channel(0);
+        Self {
+            cancellation: CancellationToken::new(),
+            inflight,
+        }
+    }
+
+    fn acquire(self: &Arc<Self>) -> DirectDispatchLease {
+        self.inflight.send_modify(|count| *count += 1);
+        DirectDispatchLease {
+            tracker: Arc::clone(self),
+        }
+    }
+
+    pub(crate) async fn wait_idle(&self) {
+        let mut inflight = self.inflight.subscribe();
+        while *inflight.borrow_and_update() != 0 {
+            if inflight.changed().await.is_err() {
+                return;
+            }
+        }
+    }
+
+    fn cancel_pending(&self) {
+        self.cancellation.cancel();
+    }
+}
+
+struct DirectDispatchLease {
+    tracker: Arc<DirectDispatchTracker>,
+}
+
+impl DirectDispatchLease {
+    fn cancellation(&self) -> CancellationToken {
+        self.tracker.cancellation.clone()
+    }
+}
+
+impl Drop for DirectDispatchLease {
+    fn drop(&mut self) {
+        self.tracker.inflight.send_modify(|count| {
+            if let Some(next) = count.checked_sub(1) {
+                *count = next;
+            } else {
+                error!(
+                    event = EVENT_NOTIFY_DISPATCH_FAILED,
+                    component = LOG_COMPONENT_NOTIFY,
+                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                    reason = "direct_lease_underflow",
+                    "Notify direct dispatch lease accounting underflowed"
+                );
+            }
+        });
+    }
+}
+
 /// Resolves the effective send concurrency (semaphore permit count).
 ///
 /// A value of `0` would build a zero-permit semaphore, so `acquire` never
@@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize {

 /// Manages event notification to targets based on rules
 pub struct EventNotifier {
+    dispatch_gate: Arc<RwLock<()>>,
+    enqueue_limiter: Arc<Semaphore>,
    metrics: Arc<NotificationMetrics>,
    rule_engine: NotifyRuleEngine,
    target_list: SharedNotifyTargetList,
@@ -82,10 +182,14 @@ impl EventNotifier {
    /// Returns a new instance of EventNotifier.
    pub fn new(metrics: Arc<NotificationMetrics>, rule_engine: NotifyRuleEngine) -> Self {
        let max_inflight = resolve_send_concurrency();
+        let target_list = Arc::new(RwLock::new(TargetList::new()));
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
        EventNotifier {
+            dispatch_gate,
+            enqueue_limiter: Arc::new(Semaphore::new(max_inflight)),
            metrics,
            rule_engine,
-            target_list: Arc::new(RwLock::new(TargetList::new())),
+            target_list,
            send_limiter: Arc::new(Semaphore::new(max_inflight)),
        }
    }
@@ -99,6 +203,10 @@ impl EventNotifier {
        Arc::clone(&self.target_list)
    }

+    pub(crate) fn dispatch_gate(&self) -> Arc<RwLock<()>> {
+        self.dispatch_gate.clone()
+    }
+
    /// Returns a list of ARNs for the registered targets
    ///
    /// # Arguments
@@ -115,12 +223,9 @@ impl EventNotifier {
            .collect()
    }

-    /// Removes all targets
    pub async fn remove_all_bucket_targets(&self) {
        let mut target_list_guard = self.target_list.write().await;
-        // The logic for sending cancel signals via stream_cancel_senders would be removed.
-        // TargetList::clear_targets_only already handles calling target.close().
-        target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel
+        target_list_guard.clear_targets_only().await;
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -155,7 +260,14 @@ impl EventNotifier {
            return;
        }
        let target_ids_len = target_ids.len();
-        let mut handles = vec![];
+        let mut deferred_handles = Vec::new();
+        let mut direct_handles = Vec::new();
+
+        // A lifecycle writer holds this gate only while handing queue-store
+        // ownership from one runtime generation to the next. Taking the read
+        // guard before cloning targets means the writer both blocks new sends
+        // and drains every save already using the old generation.
+        let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await);

        // Use scope to limit the borrow scope of target_list
        let target_list_guard = self.target_list.read().await;
@@ -185,10 +297,17 @@ impl EventNotifier {
                    );
                    continue;
                }
-                let limiter = self.send_limiter.clone();
+                let is_deferred = target_for_task.store().is_some();
+                let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease());
+                let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation);
+                let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard));
+                let limiter = if is_deferred {
+                    self.enqueue_limiter.clone()
+                } else {
+                    self.send_limiter.clone()
+                };
                let metrics = self.metrics.clone();
                let event_clone = event.clone();
-                let is_deferred = target_for_task.store().is_some();
                let target_name_for_task = target_for_task.name(); // Get the name before generating the task
                debug!(
                    event = EVENT_NOTIFY_DISPATCH_STARTED,
@@ -207,9 +326,32 @@ impl EventNotifier {
                    data: event_clone.as_ref().clone(),
                });
                let handle = tokio::spawn(async move {
+                    let _direct_dispatch_lease = direct_dispatch_lease;
+                    let _deferred_dispatch_guard = deferred_dispatch_guard;
                    metrics.increment_processing();
-                    let _permit = match limiter.acquire_owned().await {
-                        Ok(p) => p,
+                    let permit = if let Some(cancellation) = direct_cancellation {
+                        tokio::select! {
+                            biased;
+                            _ = cancellation.cancelled() => {
+                                metrics.decrement_processing();
+                                metrics.increment_skipped();
+                                debug!(
+                                    event = EVENT_NOTIFY_DISPATCH_SKIPPED,
+                                    component = LOG_COMPONENT_NOTIFY,
+                                    subsystem = LOG_SUBSYSTEM_DISPATCH,
+                                    target_id = %target_name_for_task,
+                                    reason = "runtime_generation_replaced",
+                                    "Skipped pending direct notify dispatch"
+                                );
+                                return;
+                            }
+                            permit = limiter.acquire_owned() => permit,
+                        }
+                    } else {
+                        limiter.acquire_owned().await
+                    };
+                    let _permit = match permit {
+                        Ok(permit) => permit,
                        Err(e) => {
                            error!(
                                event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -218,7 +360,7 @@ impl EventNotifier {
                                target_id = %target_name_for_task,
                                error = %e,
                                reason = "permit_acquire_failed",
-                                "Failed to acquire notify send permit"
+                                "Failed to acquire notify dispatch permit"
                            );
                            metrics.increment_failed();
                            return;
@@ -260,7 +402,11 @@ impl EventNotifier {
                        );
                    }
                });
-                handles.push(handle);
+                if is_deferred {
+                    deferred_handles.push(handle);
+                } else {
+                    direct_handles.push(handle);
+                }
            } else {
                warn!(
                    event = EVENT_NOTIFY_DISPATCH_FAILED,
@@ -276,19 +422,13 @@ impl EventNotifier {
        // target_list is automatically released here
        drop(target_list_guard);

-        // Wait for all tasks to be completed
-        for handle in handles {
-            if let Err(e) = handle.await {
-                error!(
-                    event = EVENT_NOTIFY_DISPATCH_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_DISPATCH,
-                    error = %e,
-                    reason = "join_failed",
-                    "Notify dispatch task failed"
-                );
-            }
-        }
+        // Every store-backed save owns a share of the generation guard, so
+        // caller cancellation cannot race lifecycle handoff with an enqueue.
+        // Direct targets own no queue store and may finish against the detached
+        // target while lifecycle progresses.
+        drop(dispatch_guard);
+        wait_for_dispatch_tasks(deferred_handles).await;
+        wait_for_dispatch_tasks(direct_handles).await;
        debug!(
            event = EVENT_NOTIFY_DISPATCH_COMPLETED,
            component = LOG_COMPONENT_NOTIFY,
@@ -320,7 +460,7 @@ impl EventNotifier {
            target_list_guard.add(target)?;
        }

-        info!(
+        tracing::info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
            subsystem = LOG_SUBSYSTEM_DISPATCH,
@@ -334,6 +474,7 @@ impl EventNotifier {

 /// A thread-safe list of targets
 pub struct TargetList {
+    direct_dispatches: Arc<DirectDispatchTracker>,
    /// Map of TargetID to Target
    runtime: TargetRuntimeManager<Event>,
 }
@@ -348,6 +489,7 @@ impl TargetList {
    /// Creates a new TargetList
    pub fn new() -> Self {
        TargetList {
+            direct_dispatches: Arc::new(DirectDispatchTracker::new()),
            runtime: TargetRuntimeManager::new(),
        }
    }
@@ -435,6 +577,20 @@ impl TargetList {
        self.runtime.status_snapshot(replay_workers)
    }

+    fn direct_dispatch_lease(&self) -> DirectDispatchLease {
+        self.direct_dispatches.acquire()
+    }
+
+    pub(crate) fn replace_runtime(
+        &mut self,
+        replacement: TargetRuntimeManager<Event>,
+    ) -> (TargetRuntimeManager<Event>, Arc<DirectDispatchTracker>) {
+        let runtime = std::mem::replace(&mut self.runtime, replacement);
+        let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new()));
+        direct_dispatches.cancel_pending();
+        (runtime, direct_dispatches)
+    }
+
    pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager<Event> {
        &mut self.runtime
    }
@@ -458,7 +614,7 @@ mod tests {
    use rustfs_s3_types::EventName;
    use rustfs_targets::StoreError;
    use rustfs_targets::{
-        TargetError,
+        ReplayWorkerManager, TargetError,
        store::{Key, QueueStore, Store},
        target::{EntityTarget, QueuedPayload, QueuedPayloadMeta},
    };
@@ -466,6 +622,7 @@ mod tests {
        Arc,
        atomic::{AtomicUsize, Ordering},
    };
+    use tokio::sync::Notify;

    #[tokio::test]
    async fn encoded_event_key_matches_raw_prefix_suffix_filter() {
@@ -525,19 +682,44 @@ mod tests {

    #[derive(Clone)]
    struct TestTarget {
+        block_first_save: Option<(Arc<Notify>, Arc<Notify>)>,
+        close_calls: Arc<AtomicUsize>,
+        close_entered: Option<Arc<Notify>>,
        id: TargetID,
        enabled: bool,
        save_calls: Arc<AtomicUsize>,
+        selected_calls: Arc<AtomicUsize>,
+        store: Option<QueueStore<QueuedPayload>>,
    }

    impl TestTarget {
        fn new(id: &str, name: &str, enabled: bool) -> Self {
            Self {
+                block_first_save: None,
+                close_calls: Arc::new(AtomicUsize::new(0)),
+                close_entered: None,
                id: TargetID::new(id.to_string(), name.to_string()),
                enabled,
                save_calls: Arc::new(AtomicUsize::new(0)),
+                selected_calls: Arc::new(AtomicUsize::new(0)),
+                store: None,
            }
        }
+
+        fn with_blocked_first_save(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.block_first_save = Some((entered, release));
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
+
+        fn with_close_observer(mut self, close_entered: Arc<Notify>) -> Self {
+            self.close_entered = Some(close_entered);
+            self
+        }
    }

    #[async_trait]
@@ -554,7 +736,13 @@ mod tests {
        }

        async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
-            self.save_calls.fetch_add(1, Ordering::SeqCst);
+            let call = self.save_calls.fetch_add(1, Ordering::SeqCst);
+            if call == 0
+                && let Some((entered, release)) = &self.block_first_save
+            {
+                entered.notify_one();
+                release.notified().await;
+            }
            Ok(())
        }

@@ -563,11 +751,17 @@ mod tests {
        }

        async fn close(&self) -> Result<(), TargetError> {
+            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(close_entered) = &self.close_entered {
+                close_entered.notify_one();
+            }
            Ok(())
        }

        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
        }

        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -580,10 +774,425 @@ mod tests {
        }

        fn is_enabled(&self) -> bool {
+            self.selected_calls.fetch_add(1, Ordering::SeqCst);
            self.enabled
        }
    }

+    #[tokio::test]
+    async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("gated-target", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let first_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"),
+            _ = std::future::ready(()) => {}
+        }
+
+        save_release.notify_one();
+        first_dispatch.await.expect("first dispatch task should finish");
+        let pause_guard = pause.await;
+
+        let replacement = TestTarget::new("gated-target", "webhook", true);
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.clear();
+            target_list
+                .add(Arc::new(replacement.clone()))
+                .expect("replacement target install should succeed");
+        }
+
+        let mut second_dispatch =
+            Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))));
+        tokio::select! {
+            biased;
+            _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(
+            replacement.selected_calls.load(Ordering::SeqCst),
+            0,
+            "a paused dispatch must not select a target from the replacement generation early"
+        );
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0);
+
+        drop(pause_guard);
+        second_dispatch.await;
+        assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let target =
+            TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone());
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target install should succeed");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"),
+        };
+        drop(pause_guard);
+        save_release.notify_one();
+        dispatch.await.expect("direct dispatch should finish after release");
+    }
+
+    #[tokio::test]
+    async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let first_entered = Arc::new(Notify::new());
+        let first_release = Arc::new(Notify::new());
+        let close_entered = Arc::new(Notify::new());
+        let target = TestTarget::new("direct-target", "webhook", true)
+            .with_blocked_first_save(first_entered.clone(), first_release.clone())
+            .with_close_observer(close_entered);
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target.clone()))
+            .expect("target should install");
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics.clone(),
+        );
+        let first = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        first_entered.notified().await;
+
+        // This task selects the old generation and acquires its lease before
+        // waiting for the saturated direct-send permit.
+        let second = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while target.selected_calls.load(Ordering::SeqCst) != 2 {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("both direct sends should select the old generation");
+
+        let activation = facade.activate_targets_with_replay(Vec::new()).await;
+        let mut replace = Box::pin(facade.replace_targets(activation));
+        tokio::select! {
+            biased;
+            result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"),
+            _ = std::future::ready(()) => {}
+        }
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 0);
+
+        first_release.notify_one();
+        first.await.expect("first direct dispatch should finish");
+        second.await.expect("permit-waiting direct dispatch should be cancelled");
+        replace.await.expect("replacement should close after direct leases drain");
+        assert_eq!(target.save_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(target.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(metrics.processing_count(), 0);
+        assert_eq!(metrics.processed_count(), 1);
+        assert_eq!(metrics.skipped_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn caller_abort_does_not_release_deferred_generation_lease() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone()));
+        let save_entered = Arc::new(Notify::new());
+        let save_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let target = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(save_entered.clone(), save_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut rules_map = RulesMap::new();
+        rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+        notifier
+            .target_list()
+            .write()
+            .await
+            .add(Arc::new(target))
+            .expect("target should install");
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        save_entered.notified().await;
+        dispatch.abort();
+        let _ = dispatch.await;
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("caller abort released the deferred generation lease"),
+            _ = std::future::ready(()) => {}
+        }
+        save_release.notify_one();
+        let pause_guard = pause.await;
+        drop(pause_guard);
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_concurrency_is_bounded() {
+        const LIMIT: usize = 2;
+        const TARGETS: usize = 3;
+
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            enqueue_limiter: Arc::new(Semaphore::new(LIMIT)),
+            ..EventNotifier::new(metrics, rule_engine.clone())
+        });
+        let entered = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let mut targets = Vec::new();
+        let mut rules_map = RulesMap::new();
+        for index in 0..TARGETS {
+            let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true)
+                .with_blocked_first_save(entered.clone(), release.clone())
+                .with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event"));
+            rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone());
+            notifier
+                .target_list()
+                .write()
+                .await
+                .add(Arc::new(target.clone()))
+                .expect("target should install");
+            targets.push(target);
+        }
+        rule_engine.set_bucket_rules("bucket", rules_map).await;
+
+        let dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        let total_calls = || {
+            targets
+                .iter()
+                .map(|target| target.save_calls.load(Ordering::SeqCst))
+                .sum::<usize>()
+        };
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != LIMIT {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the configured number of enqueues should enter");
+        for _ in 0..10 {
+            tokio::task::yield_now().await;
+        }
+        assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity");
+
+        release.notify_waiters();
+        tokio::time::timeout(std::time::Duration::from_secs(1), async {
+            while total_calls() != TARGETS {
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .expect("the waiting enqueue should enter after a permit is released");
+        release.notify_waiters();
+        dispatch.await.expect("all bounded enqueues should finish");
+    }
+
+    #[tokio::test]
+    async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() {
+        let metrics = Arc::new(NotificationMetrics::new());
+        let rule_engine = NotifyRuleEngine::new();
+        let notifier = Arc::new(EventNotifier {
+            send_limiter: Arc::new(Semaphore::new(1)),
+            ..EventNotifier::new(metrics.clone(), rule_engine.clone())
+        });
+        let direct_entered = Arc::new(Notify::new());
+        let direct_release = Arc::new(Notify::new());
+        let direct =
+            TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone());
+        let deferred_entered = Arc::new(Notify::new());
+        let deferred_release = Arc::new(Notify::new());
+        let queue_dir = tempfile::tempdir().expect("queue tempdir should be created");
+        let deferred = TestTarget::new("deferred", "webhook", true)
+            .with_blocked_first_save(deferred_entered.clone(), deferred_release.clone())
+            .with_store(QueueStore::new(queue_dir.path(), 16, ".event"));
+
+        let mut direct_rules = RulesMap::new();
+        direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone());
+        rule_engine.set_bucket_rules("direct-bucket", direct_rules).await;
+        let mut deferred_rules = RulesMap::new();
+        deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone());
+        rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await;
+        {
+            let target_list = notifier.target_list();
+            let mut target_list = target_list.write().await;
+            target_list.add(Arc::new(direct)).expect("direct target should install");
+            target_list.add(Arc::new(deferred)).expect("deferred target should install");
+        }
+
+        let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate(
+            notifier.target_list(),
+            Arc::new(RwLock::new(ReplayWorkerManager::new())),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(1)),
+            metrics,
+        );
+        let direct_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        direct_entered.notified().await;
+        let deferred_dispatch = tokio::spawn({
+            let notifier = notifier.clone();
+            async move {
+                notifier
+                    .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut)))
+                    .await;
+            }
+        });
+        tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified())
+            .await
+            .expect("queue persistence must not wait behind a direct network send permit");
+
+        let mut pause = Box::pin(facade.pause_dispatch());
+        tokio::select! {
+            biased;
+            _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"),
+            _ = std::future::ready(()) => {}
+        }
+        deferred_release.notify_one();
+        deferred_dispatch
+            .await
+            .expect("deferred dispatch should finish after release");
+        let pause_guard = tokio::select! {
+            biased;
+            guard = &mut pause => guard,
+            _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"),
+        };
+        drop(pause_guard);
+
+        direct_release.notify_one();
+        direct_dispatch.await.expect("direct dispatch should finish after release");
+    }
+
    #[tokio::test]
    async fn test_send_event_skips_disabled_target() {
        let rule_engine = NotifyRuleEngine::new();
@@ -38,6 +38,11 @@ impl TargetRegistry {
        TargetRegistry { plugins }
    }

+    #[cfg(test)]
+    pub(crate) fn with_plugins(plugins: TargetPluginRegistry<Event>) -> Self {
+        Self { plugins }
+    }
+
    pub fn supports_target_type(&self, target_type: &str) -> bool {
        self.plugins.supports_target_type(target_type)
    }
@@ -67,6 +72,15 @@ impl TargetRegistry {
    ) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
        self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
    }
+
+    pub(crate) async fn create_dormant_targets_from_config(
+        &self,
+        config: &Config,
+    ) -> Result<(Vec<Box<dyn Target<Event> + Send + Sync>>, Vec<String>), TargetError> {
+        self.plugins
+            .create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX)
+            .await
+    }
 }

 #[cfg(test)]
@@ -12,13 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList};
-use rustfs_targets::{
-    BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target,
+use crate::{
+    Event, NotificationError,
+    error::transition_join_error,
+    integration::NotificationMetrics,
+    notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
+};
+use rustfs_targets::{
+    BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
+    RuntimeActivation, Target, TargetRuntimeManager,
+};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
 };
-use std::sync::Arc;
 use std::time::Duration;
-use tokio::sync::{RwLock, Semaphore};
+use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
+use tokio_util::sync::CancellationToken;
 use tracing::{debug, info, warn};

 const LOG_COMPONENT_NOTIFY: &str = "notify";
@@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
 const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
 const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
 const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
+const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);

+pub(crate) struct DetachedNotifyRuntime {
+    direct_dispatches: Arc<DirectDispatchTracker>,
+    runtime: TargetRuntimeManager<Event>,
+    replay_workers: ReplayWorkerManager,
+}
+
+// Multi-lock publication order: replay_workers -> target_list ->
+// publication_gate. The lifecycle may already hold dispatch_gate while
+// entering this facade; no path that needs these three locks may acquire
+// publication_gate before either runtime lock.
 #[derive(Clone)]
 pub struct NotifyRuntimeFacade {
+    dispatch_gate: Arc<RwLock<()>>,
+    legacy_terminated: Arc<AtomicBool>,
    target_list: SharedNotifyTargetList,
    replay_workers: Arc<RwLock<ReplayWorkerManager>>,
-    runtime_adapter: Arc<dyn PluginRuntimeAdapter<Event>>,
+    runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
 }

 impl NotifyRuntimeFacade {
@@ -41,6 +64,18 @@ impl NotifyRuntimeFacade {
        concurrency_limiter: Arc<Semaphore>,
        metrics: Arc<NotificationMetrics>,
    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, None);
+        Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
+    }
+
+    pub(crate) fn new_with_dispatch_gate(
+        target_list: SharedNotifyTargetList,
+        replay_workers: Arc<RwLock<ReplayWorkerManager>>,
+        dispatch_gate: Arc<RwLock<()>>,
+        concurrency_limiter: Arc<Semaphore>,
+        metrics: Arc<NotificationMetrics>,
+    ) -> Self {
+        let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
        let replay_metrics = metrics;
        let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
            Arc::new(move |event: ReplayEvent<Event>| {
@@ -97,36 +132,166 @@ impl NotifyRuntimeFacade {
        );

        Self {
+            dispatch_gate,
+            legacy_terminated: Arc::new(AtomicBool::new(false)),
            target_list,
            replay_workers,
            runtime_adapter: Arc::new(runtime_adapter),
        }
    }

+    pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
+        self.dispatch_gate.clone().write_owned().await
+    }
+
    pub async fn activate_targets_with_replay(
        &self,
        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
    ) -> RuntimeActivation<Event> {
-        self.runtime_adapter.activate_with_replay(targets).await
+        // The compatibility pair must not start replacement replay before
+        // replace_targets has stopped and joined the current generation.
+        self.runtime_adapter.prepare_dormant_compat_activation(targets).await
    }

-    pub async fn replace_targets(&self, activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
-        // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+    pub(crate) async fn prepare_targets(
+        &self,
+        targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<Event> {
+        self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
+    }
+
+    pub(crate) async fn open_prepared_stores(
+        &self,
+        prepared: PreparedActivation<Event>,
+    ) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
+        let runtime_adapter = self.runtime_adapter.clone();
+        tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
+            .await
+            .map_err(transition_join_error)
+    }
+
+    pub(crate) async fn commit_prepared<Committed>(
+        &self,
+        opened: OpenedActivation<Event>,
+        on_committed: Committed,
+    ) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
+    where
+        Committed: FnOnce(bool),
+    {
+        // The lifecycle coordinator validates the generation immediately before
+        // entering the non-cancellable handoff. Once old replay workers have
+        // been joined, this generation must publish before a later accepted
+        // intent can run; abandoning it here would leave the old runtime
+        // visible without replay workers.
        let mut replay_workers = self.replay_workers.write().await;
        let mut target_list = self.target_list.write().await;
-        self.runtime_adapter
-            .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation)
-            .await
-            .map_err(NotificationError::Target)?;
+
+        let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
+        let fully_activated = rejected.failure_summary().is_none();
+        let (runtime, replay_workers, direct_dispatches) =
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation);
+        on_committed(fully_activated);
+        (
+            DetachedNotifyRuntime {
+                direct_dispatches,
+                runtime,
+                replay_workers,
+            },
+            rejected,
+        )
+    }
+
+    pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
+    where
+        Committed: FnOnce(),
+    {
+        // Lock order: replay_workers -> target_list. The lifecycle coordinator
+        // crosses its publication barrier before stopping the old workers, so
+        // this non-cancellable commit must publish even if a newer intent was
+        // accepted while the workers joined.
+        let mut replay_workers = self.replay_workers.write().await;
+        let mut target_list = self.target_list.write().await;
+
+        let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+        let detached = DetachedNotifyRuntime {
+            direct_dispatches,
+            runtime,
+            replay_workers: std::mem::take(&mut *replay_workers),
+        };
+        on_committed();
+        detached
+    }
+
+    pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
+        // A caller may supply an activation created outside the compatibility
+        // prepare method. Stop any already-running replacement workers before
+        // entering the ordered handoff; the supported activate→replace pair is
+        // dormant here and therefore never overlaps the old generation.
+        self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
+        let dispatch_guard = self.pause_dispatch().await;
+        if self.legacy_terminated.load(Ordering::Acquire) {
+            drop(dispatch_guard);
+            self.runtime_adapter
+                .close_compat_activation(activation)
+                .await
+                .map_err(NotificationError::Target)?;
+            return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
+        }
+        self.stop_active_replay_workers().await;
+        let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
+        let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        let (old_runtime, old_replay_workers, old_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            Self::swap_activation(&mut target_list, &mut replay_workers, activation)
+        };
+        drop(dispatch_guard);
+        let close_old = self.close_detached_targets(DetachedNotifyRuntime {
+            direct_dispatches: old_direct_dispatches,
+            runtime: old_runtime,
+            replay_workers: old_replay_workers,
+        });
+        let close_open_rejected = self.close_prepared(open_rejected);
+        let close_activation_rejected = self.close_prepared(activation_rejected);
+        let (close_old, close_open_rejected, close_activation_rejected) =
+            tokio::join!(close_old, close_open_rejected, close_activation_rejected);
+        close_old?;
+        close_open_rejected?;
+        close_activation_rejected?;
+        if !activation_failures.is_empty() {
+            return Err(NotificationError::Initialization(format!(
+                "one or more notification targets failed to activate: {}",
+                activation_failures.join("; ")
+            )));
+        }
        Ok(())
    }

    pub async fn stop_replay_workers(&self) {
-        let mut replay_workers = self.replay_workers.write().await;
-        self.runtime_adapter.stop_replay_workers(&mut replay_workers).await;
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.stop_active_replay_workers().await;
+    }
+
+    pub(crate) async fn stop_active_replay_workers(&self) {
+        let mut detached = {
+            let mut replay_workers = self.replay_workers.write().await;
+            std::mem::take(&mut *replay_workers)
+        };
+        self.runtime_adapter.stop_replay_workers(&mut detached).await;
    }

    pub async fn shutdown(&self) {
+        let _ = self.shutdown_checked().await;
+    }
+
+    pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
+        let _dispatch_guard = self.pause_dispatch().await;
+        self.legacy_terminated.store(true, Ordering::Release);
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -135,7 +300,14 @@ impl NotifyRuntimeFacade {
            "notify runtime lifecycle"
        );

-        let active_targets = self.replay_workers.read().await.len();
+        let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
+            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
+            let mut replay_workers = self.replay_workers.write().await;
+            let mut target_list = self.target_list.write().await;
+            let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
+            (runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
+        };
+        let active_targets = detached_replay_workers.len();
        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
            component = LOG_COMPONENT_NOTIFY,
@@ -145,25 +317,22 @@ impl NotifyRuntimeFacade {
            "notify runtime lifecycle"
        );

-        {
-            // Lock order: replay_workers -> target_list (matches notify AGENTS.md).
-            let mut replay_workers = self.replay_workers.write().await;
-            let mut target_list = self.target_list.write().await;
-            if let Err(err) = self
-                .runtime_adapter
-                .shutdown(target_list.runtime_mut(), &mut replay_workers)
-                .await
-            {
-                tracing::error!(
-                    event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
-                    component = LOG_COMPONENT_NOTIFY,
-                    subsystem = LOG_SUBSYSTEM_RUNTIME,
-                    error = %err,
-                    "Failed to shutdown notify runtime cleanly"
-                );
-            }
+        let shutdown_result = self
+            .close_detached(DetachedNotifyRuntime {
+                direct_dispatches: detached_direct_dispatches,
+                runtime: detached_runtime,
+                replay_workers: detached_replay_workers,
+            })
+            .await;
+        if let Err(err) = &shutdown_result {
+            tracing::error!(
+                event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
+                component = LOG_COMPONENT_NOTIFY,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                error = %err,
+                "Failed to shutdown notify runtime cleanly"
+            );
        }
-        tokio::time::sleep(Duration::from_millis(500)).await;

        info!(
            event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
@@ -172,6 +341,64 @@ impl NotifyRuntimeFacade {
            state = "stopped",
            "notify runtime lifecycle"
        );
+        shutdown_result
+    }
+
+    fn swap_activation(
+        target_list: &mut crate::notifier::TargetList,
+        replay_workers: &mut ReplayWorkerManager,
+        activation: RuntimeActivation<Event>,
+    ) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
+        let mut replacement = TargetRuntimeManager::new();
+        for target in activation.targets {
+            replacement.add_arc(target);
+        }
+        let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
+        (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
+    }
+
+    pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
+        // Replay join is intentionally not wrapped in the target-close timeout:
+        // returning before a worker is confirmed stopped could let a
+        // replacement drain the same persistent queue concurrently.
+        self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
+    }
+
+    pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        // Direct sends are allowed to finish after the replacement runtime is
+        // published, but the old targets must remain open until every task that
+        // selected that generation has released its lease.
+        detached.direct_dispatches.wait_idle().await;
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
+            Ok(close_errors) if close_errors.is_empty() => Ok(()),
+            Ok(close_errors) => {
+                let targets = close_errors
+                    .into_iter()
+                    .map(|(target_id, _)| target_id)
+                    .collect::<Vec<_>>()
+                    .join("; ");
+                Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
+                    "Failed to close {targets}"
+                ))))
+            }
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing replaced notification targets".to_string(),
+            ))),
+        }
+    }
+
+    async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
+        self.stop_detached_replay(&mut detached).await;
+        self.close_detached_targets(detached).await
+    }
+
+    pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
+        match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
+            Ok(result) => result.map_err(NotificationError::Target),
+            Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
+                "Timed out closing superseded notification targets".to_string(),
+            ))),
+        }
    }
 }

@@ -184,26 +411,50 @@ mod tests {
    };
    use async_trait::async_trait;
    use rustfs_targets::arn::TargetID;
-    use rustfs_targets::store::{Key, Store};
+    use rustfs_targets::store::{Key, QueueStore, Store};
    use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
    use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};
-    use tokio::sync::{RwLock, Semaphore};
+    use tokio::sync::{Notify, RwLock, Semaphore};

    #[derive(Clone)]
    struct TestTarget {
+        close_entered: Option<Arc<Notify>>,
+        close_error: bool,
+        close_release: Option<Arc<Notify>>,
        close_calls: Arc<AtomicUsize>,
        id: TargetID,
+        store: Option<QueueStore<QueuedPayload>>,
    }

    impl TestTarget {
        fn new(id: &str, name: &str) -> Self {
            Self {
+                close_entered: None,
+                close_error: false,
+                close_release: None,
                close_calls: Arc::new(AtomicUsize::new(0)),
                id: TargetID::new(id.to_string(), name.to_string()),
+                store: None,
            }
        }
+
+        fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
+            self.close_entered = Some(entered);
+            self.close_release = Some(release);
+            self
+        }
+
+        fn with_close_error(mut self) -> Self {
+            self.close_error = true;
+            self
+        }
+
+        fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
+            self.store = Some(store);
+            self
+        }
    }

    #[async_trait]
@@ -229,11 +480,22 @@ mod tests {

        async fn close(&self) -> Result<(), TargetError> {
            self.close_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(entered) = &self.close_entered {
+                entered.notify_one();
+            }
+            if let Some(release) = &self.close_release {
+                release.notified().await;
+            }
+            if self.close_error {
+                return Err(TargetError::Storage("forced close failure".to_string()));
+            }
            Ok(())
        }

        fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            None
+            self.store
+                .as_ref()
+                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
        }

        fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -254,7 +516,13 @@ mod tests {
        let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
        let target_list = notifier.target_list();
        let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
-        let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics);
+        let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            replay_workers.clone(),
+            notifier.dispatch_gate(),
+            Arc::new(Semaphore::new(4)),
+            metrics,
+        );
        (facade, notifier, replay_workers)
    }

@@ -273,6 +541,26 @@ mod tests {
        assert_eq!(activation.replay_workers.len(), 0);
    }

+    #[tokio::test]
+    async fn compatibility_activation_stays_dormant_until_ordered_replace() {
+        let (facade, _, replay_workers) = build_facade();
+        let queue_root = tempfile::tempdir().expect("queue root");
+        let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
+        let target = TestTarget::new("primary", "webhook").with_store(store);
+
+        let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
+        assert_eq!(replay_workers.read().await.len(), 0);
+
+        facade
+            .replace_targets(activation)
+            .await
+            .expect("ordered compatibility replace should succeed");
+        assert_eq!(replay_workers.read().await.len(), 1);
+        facade.shutdown_checked().await.expect("test runtime should shut down");
+    }
+
    #[tokio::test]
    async fn runtime_facade_replace_targets_commits_runtime_state() {
        let (facade, notifier, replay_workers) = build_facade();
@@ -292,4 +580,129 @@ mod tests {
        assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
        assert_eq!(replay_workers.read().await.len(), 0);
    }
+
+    #[tokio::test]
+    async fn runtime_queries_do_not_wait_for_target_close() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let close_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
+        close_release.notify_one();
+        shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect("shutdown should succeed");
+
+        let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test]
+    async fn runtime_locks_are_released_while_replay_worker_joins() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let cancel_received = Arc::new(Notify::new());
+        let release = Arc::new(Notify::new());
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
+        let join = tokio::spawn({
+            let cancel_received = cancel_received.clone();
+            let release = release.clone();
+            async move {
+                let _ = cancel_rx.recv().await;
+                cancel_received.notify_one();
+                release.notified().await;
+            }
+        });
+        replay_workers
+            .write()
+            .await
+            .insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
+
+        let stop = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.stop_replay_workers().await }
+        });
+        cancel_received.notified().await;
+
+        assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
+        let target_list = notifier.target_list();
+        assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
+
+        release.notify_one();
+        stop.await.expect("stop task should finish");
+    }
+
+    #[tokio::test]
+    async fn shutdown_returns_close_error_after_detaching_runtime() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let target = TestTarget::new("primary", "webhook").with_close_error();
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
+
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
+
+    #[tokio::test(start_paused = true)]
+    async fn shutdown_bounds_a_target_that_never_closes() {
+        let (facade, notifier, replay_workers) = build_facade();
+        let close_entered = Arc::new(Notify::new());
+        let never_release = Arc::new(Notify::new());
+        let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
+        facade
+            .replace_targets(rustfs_targets::RuntimeActivation {
+                replay_workers: ReplayWorkerManager::new(),
+                targets: vec![Arc::new(target) as SharedTarget<Event>],
+            })
+            .await
+            .expect("target install should succeed");
+
+        let shutdown = tokio::spawn({
+            let facade = facade.clone();
+            async move { facade.shutdown_checked().await }
+        });
+        close_entered.notified().await;
+        tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
+
+        let err = shutdown
+            .await
+            .expect("shutdown task should not panic")
+            .expect_err("blocked close must time out");
+        assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
+        let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
+            .runtime_status_snapshot()
+            .await;
+        assert_eq!(snapshot.target_count, 0);
+        assert_eq!(snapshot.replay_worker_count, 0);
+    }
 }
@@ -57,7 +57,13 @@ impl NotifyServices {
        live_event_history: Arc<RwLock<LiveEventHistory>>,
    ) -> Self {
        let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone());
-        let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone());
+        let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate(
+            target_list,
+            stream_cancellers,
+            notifier.dispatch_gate(),
+            concurrency_limiter,
+            metrics.clone(),
+        );
        let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone());
        let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view);
        let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history);
@@ -15,35 +15,70 @@
 use std::sync::Arc;

 use rustfs_ecstore::api::config::com::{
-    read_config_without_migrate as read_notify_config_without_migrate_from_backend,
-    save_server_config as save_notify_server_config_to_backend,
+    read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock,
+    read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock,
+    save_server_config_no_lock as save_notify_server_config_to_backend_no_lock,
+    with_server_config_read_lock as with_notify_server_config_read_lock_from_backend,
+    with_server_config_write_lock as with_notify_server_config_write_lock_from_backend,
 };
 use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend;
-pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
+pub use rustfs_ecstore::api::storage::ECStore as NotifyStore;

 pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
    resolve_notify_object_store_handle_from_backend()
 }

-pub(crate) async fn read_notify_server_config_without_migrate(
+pub(crate) async fn read_notify_server_config_without_migrate_no_lock(
    store: Arc<NotifyStore>,
 ) -> Result<rustfs_config::server_config::Config, String> {
-    read_notify_config_without_migrate_from_backend(store)
+    read_notify_config_without_migrate_from_backend_no_lock(store)
        .await
        .map_err(|err| err.to_string())
 }

-pub(crate) async fn save_notify_server_config(
+pub(crate) async fn read_existing_notify_server_config_no_lock(
+    store: Arc<NotifyStore>,
+) -> Result<rustfs_config::server_config::Config, String> {
+    read_existing_notify_config_from_backend_no_lock(store)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn save_notify_server_config_no_lock(
    store: Arc<NotifyStore>,
    config: &rustfs_config::server_config::Config,
 ) -> Result<(), String> {
-    save_notify_server_config_to_backend(store, config)
+    save_notify_server_config_to_backend_no_lock(store, config)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_write_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_write_lock_from_backend(store, operation)
+        .await
+        .map_err(|err| err.to_string())
+}
+
+pub(crate) async fn with_notify_server_config_read_lock<F, Fut, T>(store: Arc<NotifyStore>, operation: F) -> Result<T, String>
+where
+    F: FnOnce() -> Fut + Send + 'static,
+    Fut: std::future::Future<Output = T> + Send + 'static,
+    T: Send + 'static,
+{
+    with_notify_server_config_read_lock_from_backend(store, operation)
        .await
        .map_err(|err| err.to_string())
 }

 pub(crate) mod crate_boundary {
    pub(crate) use super::{
-        read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config,
+        read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock,
+        resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock,
+        with_notify_server_config_write_lock,
    };
 }