mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-28 17:18:58 +00:00
cxymds
1655f3192e
* fix(notify): unify runtime lifecycle coordination * fix(notify): repair lifecycle convergence checks * fix(admin): expose effective notify state (#5097)
709 lines
29 KiB
Rust
709 lines
29 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
use crate::{
|
|
Event, NotificationError,
|
|
error::transition_join_error,
|
|
integration::NotificationMetrics,
|
|
notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate},
|
|
};
|
|
use rustfs_targets::{
|
|
BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager,
|
|
RuntimeActivation, Target, TargetRuntimeManager,
|
|
};
|
|
use std::sync::{
|
|
Arc,
|
|
atomic::{AtomicBool, Ordering},
|
|
};
|
|
use std::time::Duration;
|
|
use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore};
|
|
use tokio_util::sync::CancellationToken;
|
|
use tracing::{debug, info, warn};
|
|
|
|
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
|
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
|
|
const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle";
|
|
const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed";
|
|
const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted";
|
|
const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
|
|
|
|
pub(crate) struct DetachedNotifyRuntime {
|
|
direct_dispatches: Arc<DirectDispatchTracker>,
|
|
runtime: TargetRuntimeManager<Event>,
|
|
replay_workers: ReplayWorkerManager,
|
|
}
|
|
|
|
// Multi-lock publication order: replay_workers -> target_list ->
|
|
// publication_gate. The lifecycle may already hold dispatch_gate while
|
|
// entering this facade; no path that needs these three locks may acquire
|
|
// publication_gate before either runtime lock.
|
|
#[derive(Clone)]
|
|
pub struct NotifyRuntimeFacade {
|
|
dispatch_gate: Arc<RwLock<()>>,
|
|
legacy_terminated: Arc<AtomicBool>,
|
|
target_list: SharedNotifyTargetList,
|
|
replay_workers: Arc<RwLock<ReplayWorkerManager>>,
|
|
runtime_adapter: Arc<BuiltinPluginRuntimeAdapter<Event>>,
|
|
}
|
|
|
|
impl NotifyRuntimeFacade {
|
|
pub fn new(
|
|
target_list: SharedNotifyTargetList,
|
|
replay_workers: Arc<RwLock<ReplayWorkerManager>>,
|
|
concurrency_limiter: Arc<Semaphore>,
|
|
metrics: Arc<NotificationMetrics>,
|
|
) -> Self {
|
|
let dispatch_gate = shared_dispatch_gate(&target_list, None);
|
|
Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics)
|
|
}
|
|
|
|
pub(crate) fn new_with_dispatch_gate(
|
|
target_list: SharedNotifyTargetList,
|
|
replay_workers: Arc<RwLock<ReplayWorkerManager>>,
|
|
dispatch_gate: Arc<RwLock<()>>,
|
|
concurrency_limiter: Arc<Semaphore>,
|
|
metrics: Arc<NotificationMetrics>,
|
|
) -> Self {
|
|
let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate));
|
|
let replay_metrics = metrics;
|
|
let runtime_adapter = BuiltinPluginRuntimeAdapter::new(
|
|
Arc::new(move |event: ReplayEvent<Event>| {
|
|
let metrics = replay_metrics.clone();
|
|
Box::pin(async move {
|
|
match event {
|
|
ReplayEvent::Delivered { .. } => metrics.increment_processed(),
|
|
ReplayEvent::RetryableError { .. } => {}
|
|
ReplayEvent::RetryExhausted { detail, key, target } => {
|
|
warn!(
|
|
event = EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
target_id = %target.id(),
|
|
replay_key = %key,
|
|
error = %detail,
|
|
"notify replay retry budget exhausted, entry stays queued and retries"
|
|
);
|
|
}
|
|
ReplayEvent::Dropped { target, .. } | ReplayEvent::PermanentFailure { target, .. } => {
|
|
target.record_final_failure();
|
|
metrics.increment_failed();
|
|
}
|
|
ReplayEvent::UnreadableEntry { .. } => {}
|
|
}
|
|
})
|
|
}),
|
|
Arc::new(|target_id, has_replay| {
|
|
if has_replay {
|
|
info!(
|
|
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
target_id = %target_id,
|
|
state = "replay_started",
|
|
"notify runtime lifecycle"
|
|
);
|
|
} else {
|
|
debug!(
|
|
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
target_id = %target_id,
|
|
state = "replay_skipped",
|
|
reason = "no_store_configured",
|
|
"notify runtime lifecycle"
|
|
);
|
|
}
|
|
}),
|
|
Some(concurrency_limiter),
|
|
Duration::from_secs(5),
|
|
Duration::from_millis(500),
|
|
"Stop event stream processing for target",
|
|
);
|
|
|
|
Self {
|
|
dispatch_gate,
|
|
legacy_terminated: Arc::new(AtomicBool::new(false)),
|
|
target_list,
|
|
replay_workers,
|
|
runtime_adapter: Arc::new(runtime_adapter),
|
|
}
|
|
}
|
|
|
|
pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> {
|
|
self.dispatch_gate.clone().write_owned().await
|
|
}
|
|
|
|
pub async fn activate_targets_with_replay(
|
|
&self,
|
|
targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
|
|
) -> RuntimeActivation<Event> {
|
|
// The compatibility pair must not start replacement replay before
|
|
// replace_targets has stopped and joined the current generation.
|
|
self.runtime_adapter.prepare_dormant_compat_activation(targets).await
|
|
}
|
|
|
|
pub(crate) async fn prepare_targets(
|
|
&self,
|
|
targets: Vec<Box<dyn Target<Event> + Send + Sync>>,
|
|
cancellation: &CancellationToken,
|
|
) -> PreparedActivation<Event> {
|
|
self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await
|
|
}
|
|
|
|
pub(crate) async fn open_prepared_stores(
|
|
&self,
|
|
prepared: PreparedActivation<Event>,
|
|
) -> Result<(OpenedActivation<Event>, PreparedActivation<Event>), NotificationError> {
|
|
let runtime_adapter = self.runtime_adapter.clone();
|
|
tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared))
|
|
.await
|
|
.map_err(transition_join_error)
|
|
}
|
|
|
|
pub(crate) async fn commit_prepared<Committed>(
|
|
&self,
|
|
opened: OpenedActivation<Event>,
|
|
on_committed: Committed,
|
|
) -> (DetachedNotifyRuntime, PreparedActivation<Event>)
|
|
where
|
|
Committed: FnOnce(bool),
|
|
{
|
|
// The lifecycle coordinator validates the generation immediately before
|
|
// entering the non-cancellable handoff. Once old replay workers have
|
|
// been joined, this generation must publish before a later accepted
|
|
// intent can run; abandoning it here would leave the old runtime
|
|
// visible without replay workers.
|
|
let mut replay_workers = self.replay_workers.write().await;
|
|
let mut target_list = self.target_list.write().await;
|
|
|
|
let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened);
|
|
let fully_activated = rejected.failure_summary().is_none();
|
|
let (runtime, replay_workers, direct_dispatches) =
|
|
Self::swap_activation(&mut target_list, &mut replay_workers, activation);
|
|
on_committed(fully_activated);
|
|
(
|
|
DetachedNotifyRuntime {
|
|
direct_dispatches,
|
|
runtime,
|
|
replay_workers,
|
|
},
|
|
rejected,
|
|
)
|
|
}
|
|
|
|
pub(crate) async fn commit_disabled<Committed>(&self, on_committed: Committed) -> DetachedNotifyRuntime
|
|
where
|
|
Committed: FnOnce(),
|
|
{
|
|
// Lock order: replay_workers -> target_list. The lifecycle coordinator
|
|
// crosses its publication barrier before stopping the old workers, so
|
|
// this non-cancellable commit must publish even if a newer intent was
|
|
// accepted while the workers joined.
|
|
let mut replay_workers = self.replay_workers.write().await;
|
|
let mut target_list = self.target_list.write().await;
|
|
|
|
let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
|
|
let detached = DetachedNotifyRuntime {
|
|
direct_dispatches,
|
|
runtime,
|
|
replay_workers: std::mem::take(&mut *replay_workers),
|
|
};
|
|
on_committed();
|
|
detached
|
|
}
|
|
|
|
pub async fn replace_targets(&self, mut activation: RuntimeActivation<Event>) -> Result<(), NotificationError> {
|
|
// A caller may supply an activation created outside the compatibility
|
|
// prepare method. Stop any already-running replacement workers before
|
|
// entering the ordered handoff; the supported activate→replace pair is
|
|
// dormant here and therefore never overlaps the old generation.
|
|
self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await;
|
|
let dispatch_guard = self.pause_dispatch().await;
|
|
if self.legacy_terminated.load(Ordering::Acquire) {
|
|
drop(dispatch_guard);
|
|
self.runtime_adapter
|
|
.close_compat_activation(activation)
|
|
.await
|
|
.map_err(NotificationError::Target)?;
|
|
return Err(NotificationError::Initialization("Notification runtime has terminated".to_string()));
|
|
}
|
|
self.stop_active_replay_workers().await;
|
|
let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation);
|
|
let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()]
|
|
.into_iter()
|
|
.flatten()
|
|
.collect::<Vec<_>>();
|
|
let (old_runtime, old_replay_workers, old_direct_dispatches) = {
|
|
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
|
let mut replay_workers = self.replay_workers.write().await;
|
|
let mut target_list = self.target_list.write().await;
|
|
Self::swap_activation(&mut target_list, &mut replay_workers, activation)
|
|
};
|
|
drop(dispatch_guard);
|
|
let close_old = self.close_detached_targets(DetachedNotifyRuntime {
|
|
direct_dispatches: old_direct_dispatches,
|
|
runtime: old_runtime,
|
|
replay_workers: old_replay_workers,
|
|
});
|
|
let close_open_rejected = self.close_prepared(open_rejected);
|
|
let close_activation_rejected = self.close_prepared(activation_rejected);
|
|
let (close_old, close_open_rejected, close_activation_rejected) =
|
|
tokio::join!(close_old, close_open_rejected, close_activation_rejected);
|
|
close_old?;
|
|
close_open_rejected?;
|
|
close_activation_rejected?;
|
|
if !activation_failures.is_empty() {
|
|
return Err(NotificationError::Initialization(format!(
|
|
"one or more notification targets failed to activate: {}",
|
|
activation_failures.join("; ")
|
|
)));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub async fn stop_replay_workers(&self) {
|
|
let _dispatch_guard = self.pause_dispatch().await;
|
|
self.stop_active_replay_workers().await;
|
|
}
|
|
|
|
pub(crate) async fn stop_active_replay_workers(&self) {
|
|
let mut detached = {
|
|
let mut replay_workers = self.replay_workers.write().await;
|
|
std::mem::take(&mut *replay_workers)
|
|
};
|
|
self.runtime_adapter.stop_replay_workers(&mut detached).await;
|
|
}
|
|
|
|
pub async fn shutdown(&self) {
|
|
let _ = self.shutdown_checked().await;
|
|
}
|
|
|
|
pub async fn shutdown_checked(&self) -> Result<(), NotificationError> {
|
|
let _dispatch_guard = self.pause_dispatch().await;
|
|
self.legacy_terminated.store(true, Ordering::Release);
|
|
info!(
|
|
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "stopping",
|
|
"notify runtime lifecycle"
|
|
);
|
|
|
|
let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = {
|
|
// Lock order: replay_workers -> target_list (matches notify AGENTS.md).
|
|
let mut replay_workers = self.replay_workers.write().await;
|
|
let mut target_list = self.target_list.write().await;
|
|
let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new());
|
|
(runtime, std::mem::take(&mut *replay_workers), direct_dispatches)
|
|
};
|
|
let active_targets = detached_replay_workers.len();
|
|
info!(
|
|
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "replay_stopping",
|
|
active_targets,
|
|
"notify runtime lifecycle"
|
|
);
|
|
|
|
let shutdown_result = self
|
|
.close_detached(DetachedNotifyRuntime {
|
|
direct_dispatches: detached_direct_dispatches,
|
|
runtime: detached_runtime,
|
|
replay_workers: detached_replay_workers,
|
|
})
|
|
.await;
|
|
if let Err(err) = &shutdown_result {
|
|
tracing::error!(
|
|
event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
error = %err,
|
|
"Failed to shutdown notify runtime cleanly"
|
|
);
|
|
}
|
|
|
|
info!(
|
|
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
|
component = LOG_COMPONENT_NOTIFY,
|
|
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
|
state = "stopped",
|
|
"notify runtime lifecycle"
|
|
);
|
|
shutdown_result
|
|
}
|
|
|
|
fn swap_activation(
|
|
target_list: &mut crate::notifier::TargetList,
|
|
replay_workers: &mut ReplayWorkerManager,
|
|
activation: RuntimeActivation<Event>,
|
|
) -> (TargetRuntimeManager<Event>, ReplayWorkerManager, Arc<DirectDispatchTracker>) {
|
|
let mut replacement = TargetRuntimeManager::new();
|
|
for target in activation.targets {
|
|
replacement.add_arc(target);
|
|
}
|
|
let (runtime, direct_dispatches) = target_list.replace_runtime(replacement);
|
|
(runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches)
|
|
}
|
|
|
|
pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) {
|
|
// Replay join is intentionally not wrapped in the target-close timeout:
|
|
// returning before a worker is confirmed stopped could let a
|
|
// replacement drain the same persistent queue concurrently.
|
|
self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await;
|
|
}
|
|
|
|
pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
|
|
// Direct sends are allowed to finish after the replacement runtime is
|
|
// published, but the old targets must remain open until every task that
|
|
// selected that generation has released its lease.
|
|
detached.direct_dispatches.wait_idle().await;
|
|
match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await {
|
|
Ok(close_errors) if close_errors.is_empty() => Ok(()),
|
|
Ok(close_errors) => {
|
|
let targets = close_errors
|
|
.into_iter()
|
|
.map(|(target_id, _)| target_id)
|
|
.collect::<Vec<_>>()
|
|
.join("; ");
|
|
Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!(
|
|
"Failed to close {targets}"
|
|
))))
|
|
}
|
|
Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
|
|
"Timed out closing replaced notification targets".to_string(),
|
|
))),
|
|
}
|
|
}
|
|
|
|
async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> {
|
|
self.stop_detached_replay(&mut detached).await;
|
|
self.close_detached_targets(detached).await
|
|
}
|
|
|
|
pub(crate) async fn close_prepared(&self, prepared: PreparedActivation<Event>) -> Result<(), NotificationError> {
|
|
match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await {
|
|
Ok(result) => result.map_err(NotificationError::Target),
|
|
Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout(
|
|
"Timed out closing superseded notification targets".to_string(),
|
|
))),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::NotifyRuntimeFacade;
|
|
use crate::{
|
|
Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
|
runtime_view::NotifyRuntimeView,
|
|
};
|
|
use async_trait::async_trait;
|
|
use rustfs_targets::arn::TargetID;
|
|
use rustfs_targets::store::{Key, QueueStore, Store};
|
|
use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta};
|
|
use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError};
|
|
use std::sync::Arc;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use tokio::sync::{Notify, RwLock, Semaphore};
|
|
|
|
#[derive(Clone)]
|
|
struct TestTarget {
|
|
close_entered: Option<Arc<Notify>>,
|
|
close_error: bool,
|
|
close_release: Option<Arc<Notify>>,
|
|
close_calls: Arc<AtomicUsize>,
|
|
id: TargetID,
|
|
store: Option<QueueStore<QueuedPayload>>,
|
|
}
|
|
|
|
impl TestTarget {
|
|
fn new(id: &str, name: &str) -> Self {
|
|
Self {
|
|
close_entered: None,
|
|
close_error: false,
|
|
close_release: None,
|
|
close_calls: Arc::new(AtomicUsize::new(0)),
|
|
id: TargetID::new(id.to_string(), name.to_string()),
|
|
store: None,
|
|
}
|
|
}
|
|
|
|
fn with_blocking_close(mut self, entered: Arc<Notify>, release: Arc<Notify>) -> Self {
|
|
self.close_entered = Some(entered);
|
|
self.close_release = Some(release);
|
|
self
|
|
}
|
|
|
|
fn with_close_error(mut self) -> Self {
|
|
self.close_error = true;
|
|
self
|
|
}
|
|
|
|
fn with_store(mut self, store: QueueStore<QueuedPayload>) -> Self {
|
|
self.store = Some(store);
|
|
self
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl<E> Target<E> for TestTarget
|
|
where
|
|
E: rustfs_targets::PluginEvent,
|
|
{
|
|
fn id(&self) -> TargetID {
|
|
self.id.clone()
|
|
}
|
|
|
|
async fn is_active(&self) -> Result<bool, TargetError> {
|
|
Ok(true)
|
|
}
|
|
|
|
async fn save(&self, _event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
|
|
Ok(())
|
|
}
|
|
|
|
async fn send_raw_from_store(&self, _key: Key, _body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
|
|
Ok(())
|
|
}
|
|
|
|
async fn close(&self) -> Result<(), TargetError> {
|
|
self.close_calls.fetch_add(1, Ordering::SeqCst);
|
|
if let Some(entered) = &self.close_entered {
|
|
entered.notify_one();
|
|
}
|
|
if let Some(release) = &self.close_release {
|
|
release.notified().await;
|
|
}
|
|
if self.close_error {
|
|
return Err(TargetError::Storage("forced close failure".to_string()));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
|
|
self.store
|
|
.as_ref()
|
|
.map(|store| store as &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync))
|
|
}
|
|
|
|
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
|
|
Box::new(self.clone())
|
|
}
|
|
|
|
async fn init(&self) -> Result<(), TargetError> {
|
|
Ok(())
|
|
}
|
|
|
|
fn is_enabled(&self) -> bool {
|
|
true
|
|
}
|
|
}
|
|
|
|
fn build_facade() -> (NotifyRuntimeFacade, Arc<EventNotifier>, Arc<RwLock<ReplayWorkerManager>>) {
|
|
let metrics = Arc::new(NotificationMetrics::new());
|
|
let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new()));
|
|
let target_list = notifier.target_list();
|
|
let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new()));
|
|
let facade = NotifyRuntimeFacade::new_with_dispatch_gate(
|
|
target_list,
|
|
replay_workers.clone(),
|
|
notifier.dispatch_gate(),
|
|
Arc::new(Semaphore::new(4)),
|
|
metrics,
|
|
);
|
|
(facade, notifier, replay_workers)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_facade_stops_empty_replay_workers() {
|
|
let (facade, _, _) = build_facade();
|
|
facade.stop_replay_workers().await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_facade_activates_empty_target_list() {
|
|
let (facade, _, _) = build_facade();
|
|
let activation = facade.activate_targets_with_replay(Vec::new()).await;
|
|
|
|
assert!(activation.targets.is_empty());
|
|
assert_eq!(activation.replay_workers.len(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn compatibility_activation_stays_dormant_until_ordered_replace() {
|
|
let (facade, _, replay_workers) = build_facade();
|
|
let queue_root = tempfile::tempdir().expect("queue root");
|
|
let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false);
|
|
let target = TestTarget::new("primary", "webhook").with_store(store);
|
|
|
|
let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await;
|
|
assert_eq!(activation.targets.len(), 1);
|
|
assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early");
|
|
assert_eq!(replay_workers.read().await.len(), 0);
|
|
|
|
facade
|
|
.replace_targets(activation)
|
|
.await
|
|
.expect("ordered compatibility replace should succeed");
|
|
assert_eq!(replay_workers.read().await.len(), 1);
|
|
facade.shutdown_checked().await.expect("test runtime should shut down");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_facade_replace_targets_commits_runtime_state() {
|
|
let (facade, notifier, replay_workers) = build_facade();
|
|
let target = TestTarget::new("primary", "webhook");
|
|
let activation = rustfs_targets::RuntimeActivation {
|
|
replay_workers: ReplayWorkerManager::new(),
|
|
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
|
};
|
|
|
|
facade
|
|
.replace_targets(activation)
|
|
.await
|
|
.expect("replace_targets should succeed");
|
|
|
|
let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone());
|
|
let active_targets = runtime_view.get_active_targets().await;
|
|
assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]);
|
|
assert_eq!(replay_workers.read().await.len(), 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_queries_do_not_wait_for_target_close() {
|
|
let (facade, notifier, replay_workers) = build_facade();
|
|
let close_entered = Arc::new(Notify::new());
|
|
let close_release = Arc::new(Notify::new());
|
|
let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone());
|
|
facade
|
|
.replace_targets(rustfs_targets::RuntimeActivation {
|
|
replay_workers: ReplayWorkerManager::new(),
|
|
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
|
})
|
|
.await
|
|
.expect("target install should succeed");
|
|
|
|
let shutdown = tokio::spawn({
|
|
let facade = facade.clone();
|
|
async move { facade.shutdown_checked().await }
|
|
});
|
|
close_entered.notified().await;
|
|
|
|
let target_list = notifier.target_list();
|
|
assert!(target_list.try_read().is_ok(), "target list lock must not be held during close");
|
|
assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close");
|
|
close_release.notify_one();
|
|
shutdown
|
|
.await
|
|
.expect("shutdown task should not panic")
|
|
.expect("shutdown should succeed");
|
|
|
|
let snapshot = NotifyRuntimeView::new(target_list, replay_workers)
|
|
.runtime_status_snapshot()
|
|
.await;
|
|
assert_eq!(snapshot.target_count, 0);
|
|
assert_eq!(snapshot.replay_worker_count, 0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn runtime_locks_are_released_while_replay_worker_joins() {
|
|
let (facade, notifier, replay_workers) = build_facade();
|
|
let cancel_received = Arc::new(Notify::new());
|
|
let release = Arc::new(Notify::new());
|
|
let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1);
|
|
let join = tokio::spawn({
|
|
let cancel_received = cancel_received.clone();
|
|
let release = release.clone();
|
|
async move {
|
|
let _ = cancel_rx.recv().await;
|
|
cancel_received.notify_one();
|
|
release.notified().await;
|
|
}
|
|
});
|
|
replay_workers
|
|
.write()
|
|
.await
|
|
.insert_with_handle("blocked-worker".to_string(), cancel_tx, join);
|
|
|
|
let stop = tokio::spawn({
|
|
let facade = facade.clone();
|
|
async move { facade.stop_replay_workers().await }
|
|
});
|
|
cancel_received.notified().await;
|
|
|
|
assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join");
|
|
let target_list = notifier.target_list();
|
|
assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join");
|
|
|
|
release.notify_one();
|
|
stop.await.expect("stop task should finish");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn shutdown_returns_close_error_after_detaching_runtime() {
|
|
let (facade, notifier, replay_workers) = build_facade();
|
|
let target = TestTarget::new("primary", "webhook").with_close_error();
|
|
facade
|
|
.replace_targets(rustfs_targets::RuntimeActivation {
|
|
replay_workers: ReplayWorkerManager::new(),
|
|
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
|
})
|
|
.await
|
|
.expect("target install should succeed");
|
|
|
|
let err = facade.shutdown_checked().await.expect_err("close failure should propagate");
|
|
assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_))));
|
|
|
|
let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
|
|
.runtime_status_snapshot()
|
|
.await;
|
|
assert_eq!(snapshot.target_count, 0);
|
|
assert_eq!(snapshot.replay_worker_count, 0);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn shutdown_bounds_a_target_that_never_closes() {
|
|
let (facade, notifier, replay_workers) = build_facade();
|
|
let close_entered = Arc::new(Notify::new());
|
|
let never_release = Arc::new(Notify::new());
|
|
let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release);
|
|
facade
|
|
.replace_targets(rustfs_targets::RuntimeActivation {
|
|
replay_workers: ReplayWorkerManager::new(),
|
|
targets: vec![Arc::new(target) as SharedTarget<Event>],
|
|
})
|
|
.await
|
|
.expect("target install should succeed");
|
|
|
|
let shutdown = tokio::spawn({
|
|
let facade = facade.clone();
|
|
async move { facade.shutdown_checked().await }
|
|
});
|
|
close_entered.notified().await;
|
|
tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await;
|
|
|
|
let err = shutdown
|
|
.await
|
|
.expect("shutdown task should not panic")
|
|
.expect_err("blocked close must time out");
|
|
assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_))));
|
|
let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers)
|
|
.runtime_status_snapshot()
|
|
.await;
|
|
assert_eq!(snapshot.target_count, 0);
|
|
assert_eq!(snapshot.replay_worker_count, 0);
|
|
}
|
|
}
|