fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

crates/targets/src/runtime/adapter.rs

@@ -13,21 +13,53 @@
 // limitations under the License.
 
 use super::{
-    ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot,
-    TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker,
+    OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation,
+    RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target,
+    start_replay_worker,
 };
 use crate::plugin::PluginEvent;
-use crate::{Target, TargetError};
+use crate::{SharedTarget, Target, TargetError};
 use async_trait::async_trait;
+use rayon::prelude::*;
 use std::future::Future;
+use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::pin::Pin;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::Semaphore;
+use tokio_util::sync::CancellationToken;
 
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 type ReplayStartObserver = Arc<dyn Fn(&str, bool) + Send + Sync>;
 
+const MAX_PARALLEL_STORE_OPENS: usize = 4;
+
+static STORE_OPEN_POOL: LazyLock<Result<rayon::ThreadPool, rayon::ThreadPoolBuildError>> = LazyLock::new(|| {
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(MAX_PARALLEL_STORE_OPENS)
+        .thread_name(|index| format!("rustfs-target-store-open-{index}"))
+        .build()
+});
+
+enum StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    Accepted(SharedTarget<E>),
+    Rejected { panicked: bool, target: SharedTarget<E> },
+}
+
+fn open_target_store<E>(target: SharedTarget<E>) -> StoreOpenOutcome<E>
+where
+    E: PluginEvent,
+{
+    match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) {
+        Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target),
+        Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target },
+        Err(_) => StoreOpenOutcome::Rejected { panicked: true, target },
+    }
+}
+
 /// Shared runtime contract for target plugins.
 #[async_trait]
 pub trait PluginRuntimeAdapter<E>: Send + Sync
@@ -96,6 +128,234 @@ where
             stop_log_prefix: stop_log_prefix.into(),
         }
     }
+
+    pub async fn prepare_targets(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, None).await
+    }
+
+    pub async fn prepare_targets_cancellable(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: &CancellationToken,
+    ) -> PreparedActivation<E> {
+        self.prepare_targets_inner(targets, Some(cancellation)).await
+    }
+
+    async fn prepare_targets_inner(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+        cancellation: Option<&CancellationToken>,
+    ) -> PreparedActivation<E> {
+        let mut prepared = Vec::with_capacity(targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        let mut targets = targets.into_iter();
+        while let Some(target) = targets.next() {
+            match prepare_target(target, cancellation).await {
+                PrepareTargetResult::Ready(target) => prepared.push(target),
+                PrepareTargetResult::Degraded { error, target } => {
+                    drop(error);
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: initialization failed", target.id()),
+                    });
+                    prepared.push(target);
+                }
+                PrepareTargetResult::Failed { error, target } => {
+                    drop(error);
+                    let target_id = target.id().to_string();
+                    tracing::warn!(
+                        target_id,
+                        reason = "initialization_failed",
+                        "Target initialization failed during lifecycle preparation"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: initialization failed"),
+                    });
+                    rejected_targets.push(Arc::from(target));
+                }
+                PrepareTargetResult::Cancelled(target) => {
+                    prepared.push(Arc::from(target));
+                    prepared.extend(targets.map(Arc::from));
+                    break;
+                }
+            }
+        }
+        PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: prepared,
+        }
+    }
+
+    /// Opens queue stores only after the previous runtime generation has been
+    /// quiesced. Targets whose stores cannot be opened retain the established
+    /// fault-isolation behavior and are returned for lock-free shutdown.
+    pub fn open_prepared_stores(&self, prepared: PreparedActivation<E>) -> (OpenedActivation<E>, PreparedActivation<E>) {
+        let mut accepted = Vec::with_capacity(prepared.targets.len());
+        let mut failures = prepared.failures;
+        let mut rejected = prepared.rejected_targets;
+        let outcomes = if prepared.targets.len() < 2 {
+            prepared.targets.into_iter().map(open_target_store).collect()
+        } else {
+            match STORE_OPEN_POOL.as_ref() {
+                // Vec's indexed parallel iterator preserves configuration
+                // order in collect, keeping failure summaries deterministic.
+                Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::<Vec<_>>()),
+                Err(err) => {
+                    tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially");
+                    prepared.targets.into_iter().map(open_target_store).collect()
+                }
+            }
+        };
+        for outcome in outcomes {
+            match outcome {
+                StoreOpenOutcome::Accepted(target) => accepted.push(target),
+                StoreOpenOutcome::Rejected { panicked, target } => {
+                    if panicked {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_panicked",
+                            "Target queue store panicked while opening during runtime handoff"
+                        );
+                    } else {
+                        tracing::error!(
+                            target_id = %target.id(),
+                            reason = "store_open_failed",
+                            "Failed to open target queue store during runtime handoff"
+                        );
+                    }
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{}: queue store open failed", target.id()),
+                    });
+                    rejected.push(target);
+                }
+            }
+        }
+        (
+            OpenedActivation { targets: accepted },
+            PreparedActivation {
+                failures,
+                rejected_targets: rejected,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    pub fn try_activate_prepared(&self, opened: OpenedActivation<E>) -> (RuntimeActivation<E>, PreparedActivation<E>) {
+        let mut replay_workers = ReplayWorkerManager::new();
+        let mut accepted = Vec::with_capacity(opened.targets.len());
+        let mut failures = Vec::new();
+        let mut rejected_targets = Vec::new();
+        for target in opened.targets {
+            let target_id = target.id().to_string();
+            let replay = catch_unwind(AssertUnwindSafe(|| {
+                target.store().filter(|_| target.is_enabled()).map(|store| {
+                    start_replay_worker(
+                        store.boxed_clone(),
+                        Arc::clone(&target),
+                        Arc::clone(&self.replay_hook),
+                        self.replay_semaphore.clone(),
+                        self.batch_timeout,
+                        self.idle_sleep,
+                    )
+                })
+            }));
+            let replay = match replay {
+                Ok(replay) => replay,
+                Err(_) => {
+                    tracing::error!(
+                        target_id,
+                        reason = "replay_activation_panicked",
+                        "Target replay activation panicked during runtime handoff"
+                    );
+                    failures.push(TargetActivationFailure {
+                        detail: format!("{target_id}: replay activation failed"),
+                    });
+                    rejected_targets.push(target);
+                    continue;
+                }
+            };
+            (self.replay_start_observer)(&target_id, replay.is_some());
+            if let Some((cancel_tx, join)) = replay {
+                replay_workers.insert_with_handle(target_id, cancel_tx, join);
+            }
+            accepted.push(target);
+        }
+
+        (
+            RuntimeActivation {
+                replay_workers,
+                targets: accepted,
+            },
+            PreparedActivation {
+                failures,
+                rejected_targets,
+                targets: Vec::new(),
+            },
+        )
+    }
+
+    #[doc(hidden)]
+    pub async fn prepare_dormant_compat_activation(
+        &self,
+        targets: Vec<Box<dyn Target<E> + Send + Sync>>,
+    ) -> RuntimeActivation<E> {
+        let PreparedActivation {
+            failures,
+            rejected_targets,
+            targets,
+        } = self.prepare_targets(targets).await;
+        let rejected = PreparedActivation {
+            failures,
+            rejected_targets,
+            targets: Vec::new(),
+        };
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation");
+        }
+        RuntimeActivation {
+            replay_workers: ReplayWorkerManager::new(),
+            targets,
+        }
+    }
+
+    #[doc(hidden)]
+    pub fn start_dormant_compat_activation(
+        &self,
+        activation: RuntimeActivation<E>,
+    ) -> (RuntimeActivation<E>, PreparedActivation<E>, PreparedActivation<E>) {
+        let prepared = PreparedActivation {
+            failures: Vec::new(),
+            rejected_targets: Vec::new(),
+            targets: activation.targets,
+        };
+        let (opened, open_rejected) = self.open_prepared_stores(prepared);
+        let (activation, activation_rejected) = self.try_activate_prepared(opened);
+        (activation, open_rejected, activation_rejected)
+    }
+
+    #[doc(hidden)]
+    pub async fn close_compat_activation(&self, mut activation: RuntimeActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in activation.targets {
+            runtime.add_arc(target);
+        }
+        self.shutdown(&mut runtime, &mut activation.replay_workers).await
+    }
+
+    pub async fn close_prepared(&self, prepared: PreparedActivation<E>) -> Result<(), TargetError> {
+        let mut runtime = TargetRuntimeManager::new();
+        for target in prepared.targets.into_iter().chain(prepared.rejected_targets) {
+            runtime.add_arc(target);
+        }
+        let mut replay_workers = ReplayWorkerManager::new();
+        self.shutdown(&mut runtime, &mut replay_workers).await
+    }
 }
 
 #[async_trait]
@@ -104,36 +364,16 @@ where
     E: PluginEvent,
 {
     async fn activate_with_replay(&self, targets: Vec<Box<dyn Target<E> + Send + Sync>>) -> RuntimeActivation<E> {
-        let replay_hook = Arc::clone(&self.replay_hook);
-        let replay_start_observer = Arc::clone(&self.replay_start_observer);
-        let replay_semaphore = self.replay_semaphore.clone();
-        let batch_timeout = self.batch_timeout;
-        let idle_sleep = self.idle_sleep;
-
-        activate_targets_with_replay(targets, move |target| {
-            let replay_hook = Arc::clone(&replay_hook);
-            let replay_start_observer = Arc::clone(&replay_start_observer);
-            let replay_semaphore = replay_semaphore.clone();
-
-            async move {
-                init_target_and_optionally_start_replay(
-                    target,
-                    move |target_id, has_replay| replay_start_observer(target_id, has_replay),
-                    move |store, target| {
-                        start_replay_worker(
-                            store,
-                            target,
-                            Arc::clone(&replay_hook),
-                            replay_semaphore.clone(),
-                            batch_timeout,
-                            idle_sleep,
-                        )
-                    },
-                )
-                .await
-            }
-        })
-        .await
+        let prepared = self.prepare_targets(targets).await;
+        let (opened, rejected) = self.open_prepared_stores(prepared);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened");
+        }
+        let (activation, rejected) = self.try_activate_prepared(opened);
+        if let Err(err) = self.close_prepared(rejected).await {
+            tracing::warn!(error = %err, "Failed to close targets rejected during replay activation");
+        }
+        activation
     }
 
     async fn replace_runtime_targets(
@@ -189,7 +429,7 @@ where
         if !close_errors.is_empty() {
             let detail = close_errors
                 .into_iter()
-                .map(|(target_id, err)| format!("{target_id}: {err}"))
+                .map(|(target_id, _)| target_id)
                 .collect::<Vec<_>>()
                 .join("; ");
             return Err(TargetError::Storage(format!("Failed to close {detail}")));
@@ -200,24 +440,143 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter};
+    use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter};
     use crate::PluginEvent;
     use crate::arn::TargetID;
     use crate::store::{Key, QueueStore, Store};
     use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
     use crate::{StoreError, Target, TargetError};
     use async_trait::async_trait;
-    use std::sync::Arc;
     use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::{Arc, Condvar, Mutex};
     use std::time::Duration;
     use tempfile::tempdir;
+    use tokio::sync::Notify;
+    use tokio_util::sync::CancellationToken;
+
+    type TestStore = dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync;
+
+    type BeforeOpen = Arc<dyn Fn() + Send + Sync>;
+
+    #[derive(Clone)]
+    struct TestOpenStore {
+        before_clone: BeforeOpen,
+        before_open: BeforeOpen,
+        store: QueueStore<QueuedPayload>,
+    }
+
+    impl Store<QueuedPayload> for TestOpenStore {
+        type Error = StoreError;
+        type Key = Key;
+
+        fn open(&self) -> Result<(), Self::Error> {
+            (self.before_open)();
+            self.store.open()
+        }
+
+        fn put(&self, item: Arc<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put(item)
+        }
+
+        fn put_multiple(&self, items: Vec<QueuedPayload>) -> Result<Self::Key, Self::Error> {
+            self.store.put_multiple(items)
+        }
+
+        fn put_raw(&self, data: &[u8]) -> Result<Self::Key, Self::Error> {
+            self.store.put_raw(data)
+        }
+
+        fn get(&self, key: &Self::Key) -> Result<QueuedPayload, Self::Error> {
+            self.store.get(key)
+        }
+
+        fn get_multiple(&self, key: &Self::Key) -> Result<Vec<QueuedPayload>, Self::Error> {
+            self.store.get_multiple(key)
+        }
+
+        fn get_raw(&self, key: &Self::Key) -> Result<Vec<u8>, Self::Error> {
+            self.store.get_raw(key)
+        }
+
+        fn del(&self, key: &Self::Key) -> Result<(), Self::Error> {
+            self.store.del(key)
+        }
+
+        fn delete(&self) -> Result<(), Self::Error> {
+            self.store.delete()
+        }
+
+        fn list(&self) -> Vec<Self::Key> {
+            self.store.list()
+        }
+
+        fn len(&self) -> usize {
+            self.store.len()
+        }
+
+        fn is_empty(&self) -> bool {
+            self.store.is_empty()
+        }
+
+        fn boxed_clone(&self) -> Box<dyn Store<QueuedPayload, Error = Self::Error, Key = Self::Key> + Send + Sync> {
+            (self.before_clone)();
+            Box::new(self.clone())
+        }
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGate {
+        changed: Condvar,
+        state: Mutex<StoreOpenGateState>,
+    }
+
+    #[derive(Default)]
+    struct StoreOpenGateState {
+        active: usize,
+        max_active: usize,
+        released: bool,
+    }
+
+    impl StoreOpenGate {
+        fn enter(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.active += 1;
+            state.max_active = state.max_active.max(state.active);
+            self.changed.notify_all();
+            while !state.released {
+                state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner());
+            }
+            state.active -= 1;
+        }
+
+        fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool {
+            let state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            let (state, _) = self
+                .changed
+                .wait_timeout_while(state, timeout, |state| state.max_active < expected)
+                .unwrap_or_else(|err| err.into_inner());
+            state.max_active >= expected
+        }
+
+        fn release(&self) {
+            let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner());
+            state.released = true;
+            self.changed.notify_all();
+        }
+
+        fn max_active(&self) -> usize {
+            self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active
+        }
+    }
 
     #[derive(Clone)]
     struct TestTarget {
         close_calls: Arc<AtomicUsize>,
         id: TargetID,
+        init_calls: Arc<AtomicUsize>,
+        init_entered: Option<Arc<Notify>>,
         init_fails: bool,
-        store: Option<QueueStore<QueuedPayload>>,
+        store: Option<Arc<TestStore>>,
     }
 
     impl TestTarget {
@@ -225,6 +584,8 @@ mod tests {
             Self {
                 close_calls: Arc::new(AtomicUsize::new(0)),
                 id: TargetID::new(id.to_string(), name.to_string()),
+                init_calls: Arc::new(AtomicUsize::new(0)),
+                init_entered: None,
                 init_fails: false,
                 store: None,
             }
@@ -235,11 +596,16 @@ mod tests {
             self
         }
 
+        fn with_pending_init(mut self, init_entered: Arc<Notify>) -> Self {
+            self.init_entered = Some(init_entered);
+            self
+        }
+
         fn with_store(mut self) -> Self {
             let dir = tempdir().expect("tempdir should be created for queue store tests");
             let store = QueueStore::<QueuedPayload>::new(dir.path(), 16, ".queue");
             store.open().expect("queue store should open");
-            self.store = Some(store);
+            self.store = Some(Arc::new(store));
             self
         }
     }
@@ -271,9 +637,7 @@ mod tests {
         }
 
         fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
-            self.store
-                .as_ref()
-                .map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
+            self.store.as_deref()
         }
 
         fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
@@ -281,6 +645,11 @@ mod tests {
         }
 
         async fn init(&self) -> Result<(), TargetError> {
+            self.init_calls.fetch_add(1, Ordering::SeqCst);
+            if let Some(init_entered) = &self.init_entered {
+                init_entered.notify_one();
+                return std::future::pending().await;
+            }
             if self.init_fails {
                 return Err(TargetError::Configuration("forced init failure".to_string()));
             }
@@ -334,6 +703,212 @@ mod tests {
         assert_eq!(activation.replay_workers.len(), 1);
     }
 
+    #[tokio::test]
+    async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() {
+        let adapter = builtin_adapter();
+        let target = TestTarget::new("primary", "webhook").with_failed_init().with_store();
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert_eq!(prepared.targets.len(), 1);
+        assert!(
+            prepared
+                .failure_summary()
+                .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure"))
+        );
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.failure_summary().is_some());
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.targets.len(), 1);
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation.replay_workers.stop_all("stop degraded target replay worker").await;
+    }
+
+    #[tokio::test]
+    async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() {
+        let adapter = builtin_adapter();
+        let init_entered = Arc::new(Notify::new());
+        let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone());
+        let first_close_calls = first.close_calls.clone();
+        let second = TestTarget::new("second", "webhook");
+        let second_close_calls = second.close_calls.clone();
+        let second_init_calls = second.init_calls.clone();
+        let cancellation = CancellationToken::new();
+        let prepare_adapter = adapter.clone();
+        let prepare_cancellation = cancellation.clone();
+        let prepare = tokio::spawn(async move {
+            prepare_adapter
+                .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation)
+                .await
+        });
+
+        init_entered.notified().await;
+        cancellation.cancel();
+        let prepared = tokio::time::timeout(Duration::from_secs(1), prepare)
+            .await
+            .expect("cancellation should interrupt target initialization")
+            .expect("preparation task should finish");
+        assert_eq!(prepared.targets.len(), 2);
+
+        adapter
+            .close_prepared(prepared)
+            .await
+            .expect("cancelled targets should close");
+        assert_eq!(first_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(second_init_calls.load(Ordering::SeqCst), 0);
+    }
+
+    #[tokio::test]
+    async fn prepared_activation_opens_store_before_starting_replay() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let queue_path = dir.path().join("queue");
+        let mut target = TestTarget::new("primary", "webhook");
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&queue_path, 16, ".queue")));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        assert!(!queue_path.exists(), "dormant preparation must not open the queue store");
+
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        assert!(rejected.targets.is_empty());
+        assert!(queue_path.is_dir(), "handoff must open the queue store before activation");
+        let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened);
+        assert!(activation_rejected.failure_summary().is_none());
+        assert_eq!(activation.replay_workers.len(), 1);
+        activation
+            .replay_workers
+            .stop_all("stop prepared activation test worker")
+            .await;
+    }
+
+    #[tokio::test]
+    async fn activation_closes_a_target_when_its_queue_store_cannot_open() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let invalid_base = dir.path().join("not-a-directory");
+        std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created");
+        let mut target = TestTarget::new("primary", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(QueueStore::<QueuedPayload>::new(&invalid_base, 16, ".queue")));
+
+        let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
+
+        assert!(activation.targets.is_empty());
+        assert!(activation.replay_workers.is_empty());
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() {
+        const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2;
+
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let gate = Arc::new(StoreOpenGate::default());
+        let mut targets: Vec<Box<dyn Target<String> + Send + Sync>> = Vec::with_capacity(TARGETS);
+        let mut expected_ids = Vec::with_capacity(TARGETS);
+        for index in 0..TARGETS {
+            let mut target = TestTarget::new(&format!("target-{index}"), "webhook");
+            expected_ids.push(target.id.to_string());
+            let open_gate = gate.clone();
+            target.store = Some(Arc::new(TestOpenStore {
+                before_clone: Arc::new(|| {}),
+                before_open: Arc::new(move || open_gate.enter()),
+                store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"),
+            }));
+            targets.push(Box::new(target));
+        }
+
+        let prepared = adapter.prepare_targets(targets).await;
+        let open_adapter = adapter.clone();
+        let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared));
+        let wait_gate = gate.clone();
+        let reached_bound =
+            tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30)))
+                .await
+                .expect("store-open observer should not panic");
+        gate.release();
+        let (opened, rejected) = opening.await.expect("bounded store opens should not panic");
+        let opened_ids = opened
+            .targets
+            .iter()
+            .map(|target| target.id().to_string())
+            .collect::<Vec<_>>();
+
+        assert!(reached_bound, "store opens did not use the configured parallelism");
+        assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS);
+        assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order");
+        assert!(rejected.targets.is_empty());
+        assert!(rejected.failure_summary().is_none());
+    }
+
+    #[tokio::test]
+    async fn panicking_store_open_rejects_and_closes_only_that_target() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| {}),
+            before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, rejected) = adapter.open_prepared_stores(prepared);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store should report a generic activation failure");
+        let (activation, activation_rejected) = adapter.try_activate_prepared(opened);
+
+        assert!(activation.targets.is_empty(), "a target without an open store must not become visible");
+        assert!(
+            activation.replay_workers.is_empty(),
+            "a rejected target must not publish without a replay worker"
+        );
+        assert!(activation_rejected.failure_summary().is_none());
+        assert!(summary.contains("queue store open failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected after a store panic should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
+    #[tokio::test]
+    async fn panicking_store_clone_cannot_publish_target_without_replay_worker() {
+        let adapter = builtin_adapter();
+        let dir = tempdir().expect("tempdir should be created");
+        let mut target = TestTarget::new("panicking-clone", "webhook");
+        let close_calls = target.close_calls.clone();
+        target.store = Some(Arc::new(TestOpenStore {
+            before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")),
+            before_open: Arc::new(|| {}),
+            store: QueueStore::new(dir.path(), 16, ".queue"),
+        }));
+
+        let prepared = adapter.prepare_targets(vec![Box::new(target)]).await;
+        let (opened, open_rejected) = adapter.open_prepared_stores(prepared);
+        assert!(open_rejected.failure_summary().is_none());
+        let (activation, rejected) = adapter.try_activate_prepared(opened);
+        let summary = rejected
+            .failure_summary()
+            .expect("panicking store clone should report a generic activation failure");
+
+        assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible");
+        assert!(activation.replay_workers.is_empty());
+        assert!(summary.contains("replay activation failed"));
+        assert!(!summary.contains("do-not-expose-payload"));
+        adapter
+            .close_prepared(rejected)
+            .await
+            .expect("a target rejected during replay activation should close");
+        assert_eq!(close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[tokio::test]
     async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() {
         let adapter = builtin_adapter();

crates/targets/src/runtime/mod.rs

@@ -27,11 +27,21 @@ use crate::store::{Key, Store, ensure_store_entry_raw_readable};
 use crate::target::QueuedPayload;
 use crate::target::TargetDeliverySnapshot;
 use crate::{StoreError, TargetError};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use std::sync::Arc;
 use std::{collections::HashMap, fmt::Debug};
 use std::{future::Future, pin::Pin, time::Duration};
 use tokio::sync::{Semaphore, mpsc};
 use tokio::task::JoinHandle;
+use tokio_util::sync::CancellationToken;
+
+fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str {
+    if error.is_cancelled() {
+        "join_cancelled"
+    } else {
+        "join_panicked"
+    }
+}
 
 /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full
 /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before
@@ -68,11 +78,21 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration {
 pub type SharedTarget<E> = Arc<dyn Target<E> + Send + Sync>;
 type ReplayHook<E> = Arc<dyn Fn(ReplayEvent<E>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
 
-/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single
-/// replay worker to observe its cancel signal and exit before it is forcibly
-/// aborted. Workers observe cancellation promptly (including during retry
-/// backoff), so this only guards against a wedged task.
-const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5);
+pub(crate) enum PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    Ready(SharedTarget<E>),
+    Degraded {
+        error: TargetError,
+        target: SharedTarget<E>,
+    },
+    Failed {
+        error: TargetError,
+        target: Box<dyn Target<E> + Send + Sync>,
+    },
+    Cancelled(Box<dyn Target<E> + Send + Sync>),
+}
 
 /// Tracks a running replay worker: its cancel channel and, when the worker was
 /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown.
@@ -113,7 +133,7 @@ impl ReplayWorkerManager {
     }
 
     /// Registers a cancel channel together with the worker's join handle so
-    /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]).
+    /// `stop_all` can await the worker's exit.
     pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) {
         self.cancellers.insert(
             target_id,
@@ -140,37 +160,36 @@ impl ReplayWorkerManager {
     }
 
     /// Stops every replay worker: it first signals cancellation to all of them,
-    /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after
-    /// which the task is aborted). Signalling before joining lets all workers
-    /// wind down concurrently, and joining guarantees no worker keeps draining
-    /// the shared store after this returns — preventing duplicate delivery and
-    /// orphaned tasks across reloads and shutdown.
+    /// then strictly awaits each worker's exit. A worker already awaiting a
+    /// delivery acknowledgement is allowed to finish; aborting it at an
+    /// arbitrary deadline could leave an acknowledged queue entry undeleted and
+    /// make the replacement worker deliver it again. Signalling before joining
+    /// lets all workers wind down concurrently. Legacy joinless registrations
+    /// can only be signalled.
     pub async fn stop_all(&mut self, log_prefix: &str) {
-        let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect();
+        let mut joins = std::collections::VecDeque::new();
 
         // Phase 1: signal cancellation to all workers.
-        for (target_id, handle) in &handles {
+        for (target_id, handle) in handles {
             tracing::info!(target_id = %target_id, "{log_prefix}");
-            let _ = handle.cancel_tx.send(()).await;
+            let _ = handle.cancel_tx.try_send(());
+            if let Some(join) = handle.join {
+                joins.push_back((target_id, join));
+            } else {
+                tracing::warn!(
+                    target_id = %target_id,
+                    "Replay worker has no join handle; cancellation was signalled but exit cannot be verified"
+                );
+            }
         }
 
-        // Phase 2: await each worker's exit, forcibly aborting any that overrun.
-        for (target_id, handle) in handles.drain(..) {
-            let Some(mut join) = handle.join else {
-                continue;
-            };
-            match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await {
-                Ok(Ok(())) => {}
-                Ok(Err(err)) => {
-                    tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally");
-                }
-                Err(_) => {
-                    join.abort();
-                    tracing::warn!(
-                        target_id = %target_id,
-                        "Timed out awaiting replay worker exit; task aborted"
-                    );
-                }
+        // Phase 2: strict join. Delivery operations own their own protocol
+        // deadlines; lifecycle must not invent a shorter deadline that turns an
+        // acknowledgement race into duplicate delivery.
+        while let Some((target_id, join)) = joins.pop_front() {
+            if let Err(err) = join.await {
+                tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally");
             }
         }
     }
@@ -184,6 +203,55 @@ where
     pub targets: Vec<SharedTarget<E>>,
 }
 
+/// Targets whose persistent queue stores are open and are ready to start
+/// replay. This distinct stage prevents activation from skipping store open.
+pub struct OpenedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+struct TargetActivationFailure {
+    detail: String,
+}
+
+/// Targets that have completed initialization but have not started replay
+/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop
+/// the previous workers before the replacement workers are spawned.
+pub struct PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    failures: Vec<TargetActivationFailure>,
+    rejected_targets: Vec<SharedTarget<E>>,
+    pub(crate) targets: Vec<SharedTarget<E>>,
+}
+
+impl<E> PreparedActivation<E>
+where
+    E: PluginEvent,
+{
+    pub fn failure_summary(&self) -> Option<String> {
+        if self.failures.is_empty() {
+            return None;
+        }
+
+        Some(
+            self.failures
+                .iter()
+                .map(|failure| failure.detail.clone())
+                .collect::<Vec<_>>()
+                .join("; "),
+        )
+    }
+
+    pub fn extend_creation_failures(&mut self, failures: impl IntoIterator<Item = String>) {
+        self.failures
+            .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail }));
+    }
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Eq)]
 pub struct RuntimeStatusSnapshot {
     pub replay_worker_count: usize,
@@ -339,17 +407,19 @@ where
     /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the
     /// runtime.
     pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> {
-        let target_ids: Vec<String> = self.targets.keys().cloned().collect();
+        let targets = std::mem::take(&mut self.targets);
+        let mut closes = FuturesUnordered::new();
+        for (target_id, target) in targets {
+            closes.push(async move { (target_id, target.close().await) });
+        }
+
         let mut errors = Vec::new();
-        for target_id in target_ids {
-            if let Some(target) = self.targets.remove(&target_id)
-                && let Err(err) = target.close().await
-            {
+        while let Some((target_id, result)) = closes.next().await {
+            if let Err(err) = result {
                 tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown");
                 errors.push((target_id, err));
             }
         }
-        self.targets.clear();
         errors
     }
 
@@ -440,21 +510,16 @@ where
         SharedTarget<E>,
     ) -> (mpsc::Sender<()>, JoinHandle<()>),
 {
-    let target_id = target.id().to_string();
-    let has_store = target.store().is_some();
-
-    if let Err(err) = target.init().await {
-        tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target");
-        if !has_store {
+    let shared = match prepare_target(target, None).await {
+        PrepareTargetResult::Ready(target) => target,
+        PrepareTargetResult::Degraded { target, .. } => target,
+        PrepareTargetResult::Failed { target, .. } => {
+            let _ = target.close().await;
             return None;
         }
-        tracing::warn!(
-            target_id = %target_id,
-            "Proceeding with store-backed target despite init failure"
-        );
-    }
-
-    let shared: SharedTarget<E> = Arc::from(target);
+        PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
+    };
+    let target_id = shared.id().to_string();
     if !shared.is_enabled() {
         on_replay_start(&target_id, false);
         return Some((shared, None));
@@ -467,6 +532,45 @@ where
     Some((shared, cancel))
 }
 
+pub(crate) async fn prepare_target<E>(
+    target: Box<dyn Target<E> + Send + Sync>,
+    cancellation: Option<&CancellationToken>,
+) -> PrepareTargetResult<E>
+where
+    E: PluginEvent,
+{
+    let target_id = target.id().to_string();
+    let has_store = target.store().is_some();
+
+    let init_result = match cancellation {
+        Some(cancellation) => {
+            tokio::select! {
+                biased;
+                _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target),
+                result = target.init() => result,
+            }
+        }
+        None => target.init().await,
+    };
+
+    if let Err(err) = init_result {
+        tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
+        if !has_store {
+            return PrepareTargetResult::Failed { error: err, target };
+        }
+        tracing::warn!(
+            target_id = %target_id,
+            "Proceeding with store-backed target despite init failure"
+        );
+        return PrepareTargetResult::Degraded {
+            error: err,
+            target: Arc::from(target),
+        };
+    }
+
+    PrepareTargetResult::Ready(Arc::from(target))
+}
+
 type ActivatedTarget<E> = (SharedTarget<E>, Option<(mpsc::Sender<()>, JoinHandle<()>)>);
 
 pub async fn activate_targets_with_replay<E, F, Fut>(
@@ -584,7 +688,11 @@ async fn stream_replay_worker<E>(
                 }
                 Ok(Ok(_)) => {}
                 Err(join_err) => {
-                    tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join");
+                    tracing::warn!(
+                        target_id = %target.id(),
+                        reason = join_failure_reason(&join_err),
+                        "The failed-events maintenance task failed to join"
+                    );
                 }
             }
             last_prune = tokio::time::Instant::now();
@@ -835,7 +943,8 @@ mod tests {
     use crate::{Target, TargetError};
     use async_trait::async_trait;
     use std::sync::Arc;
-    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use tokio::sync::{Notify, Semaphore};
 
     #[tokio::test(start_paused = true)]
     async fn seed_interval_start_backdates_by_one_interval() {
@@ -900,14 +1009,20 @@ mod tests {
     #[derive(Clone)]
     struct TestTarget {
         id: TargetID,
+        block_on_close: Arc<AtomicBool>,
+        close_gate: Arc<Semaphore>,
         close_calls: Arc<AtomicUsize>,
+        close_started: Arc<Notify>,
     }
 
     impl TestTarget {
         fn new(id: &str, name: &str) -> Self {
             Self {
                 id: TargetID::new(id.to_string(), name.to_string()),
+                block_on_close: Arc::new(AtomicBool::new(false)),
+                close_gate: Arc::new(Semaphore::new(0)),
                 close_calls: Arc::new(AtomicUsize::new(0)),
+                close_started: Arc::new(Notify::new()),
             }
         }
     }
@@ -935,6 +1050,10 @@ mod tests {
 
         async fn close(&self) -> Result<(), TargetError> {
             self.close_calls.fetch_add(1, Ordering::SeqCst);
+            self.close_started.notify_one();
+            if self.block_on_close.load(Ordering::SeqCst) {
+                let _permit = self.close_gate.acquire().await.expect("close gate should remain open");
+            }
             Ok(())
         }
 
@@ -966,6 +1085,45 @@ mod tests {
         assert_eq!(close_calls.load(Ordering::SeqCst), 1);
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() {
+        let mut manager = TargetRuntimeManager::<String>::new();
+        let first = TestTarget::new("first", "webhook");
+        let second = TestTarget::new("second", "webhook");
+        let first_observer = first.clone();
+        let second_observer = second.clone();
+        manager.add_boxed(Box::new(first));
+        manager.add_boxed(Box::new(second));
+
+        let first_close_key = manager
+            .keys()
+            .into_iter()
+            .next()
+            .expect("two targets should have a first close key");
+        let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() {
+            (first_observer, second_observer)
+        } else {
+            (second_observer, first_observer)
+        };
+        blocked.block_on_close.store(true, Ordering::SeqCst);
+
+        let close_task = tokio::spawn(async move { manager.clear_and_close().await });
+
+        tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified())
+            .await
+            .expect("the first target close should start");
+        tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified())
+            .await
+            .expect("a blocked first close must not prevent the second close from starting");
+        assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target");
+
+        blocked.close_gate.add_permits(1);
+        let errors = close_task.await.expect("clear_and_close task should join");
+        assert!(errors.is_empty());
+        assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1);
+        assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1);
+    }
+
     #[test]
     fn runtime_manager_snapshots_targets() {
         let mut manager = TargetRuntimeManager::<String>::new();
@@ -1030,6 +1188,51 @@ mod tests {
         assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion");
     }
 
+    #[tokio::test(start_paused = true)]
+    async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() {
+        use super::ReplayWorkerManager;
+        use std::sync::atomic::{AtomicBool, Ordering};
+        use tokio::sync::Notify;
+
+        let mut manager = ReplayWorkerManager::new();
+        let acknowledgement = Arc::new(Notify::new());
+        let worker_started = Arc::new(Notify::new());
+        let exited = Arc::new(AtomicBool::new(false));
+        let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1);
+        let worker_acknowledgement = acknowledgement.clone();
+        let worker_started_signal = worker_started.clone();
+        let worker_exited = exited.clone();
+        let join = tokio::spawn(async move {
+            worker_started_signal.notify_one();
+            let _ = cancel_rx.recv().await;
+            // Model a protocol operation that has accepted the request but has
+            // not returned its acknowledgement yet. Lifecycle must not abort
+            // this future or the same durable entry can be sent twice.
+            worker_acknowledgement.notified().await;
+            worker_exited.store(true, Ordering::SeqCst);
+        });
+        manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join);
+        worker_started.notified().await;
+
+        let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker"));
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"),
+            _ = std::future::ready(()) => {}
+        }
+        tokio::time::advance(std::time::Duration::from_secs(60)).await;
+        tokio::select! {
+            biased;
+            _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"),
+            _ = std::future::ready(()) => {}
+        }
+
+        acknowledgement.notify_one();
+        stop.await;
+        assert!(exited.load(Ordering::SeqCst));
+        assert!(manager.is_empty());
+    }
+
     mod classifier {
         use super::super::{ReplayEvent, stream_replay_worker};
         use crate::arn::TargetID;