fix(notify): unify runtime lifecycle coordination (#5088)

* fix(notify): unify runtime lifecycle coordination

* fix(notify): repair lifecycle convergence checks

* fix(admin): expose effective notify state (#5097)

crates/notify/tests/global_lifecycle.rs

@@ -0,0 +1,80 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::server_config::Config;
+use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile};
+use std::sync::Arc;
+
+fn assert_terminated(error: NotificationError) {
+    assert!(matches!(
+        error,
+        NotificationError::Initialization(detail) if detail == "Notification runtime has terminated"
+    ));
+}
+
+#[tokio::test]
+async fn global_singleton_survives_suspend_but_not_process_termination() {
+    let system = ensure_live_events();
+    let same_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &same_system));
+    assert!(Arc::ptr_eq(
+        &system,
+        &notification_system().expect("global notification system should exist")
+    ));
+
+    system
+        .set_targets_enabled(true, Some(Config::new()))
+        .await
+        .expect("empty target runtime should enable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .set_targets_enabled(false, None)
+        .await
+        .expect("disable should suspend targets without terminating the singleton");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly);
+
+    system
+        .set_targets_enabled(true, None)
+        .await
+        .expect("a suspended target runtime should be restartable");
+    assert!(matches!(
+        system.runtime_lifecycle_state(),
+        NotificationRuntimeState::TargetsEnabled { .. }
+    ));
+
+    system
+        .shutdown_checked()
+        .await
+        .expect("process shutdown should terminate the target runtime");
+    assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+
+    let lazy_system = ensure_live_events();
+    assert!(Arc::ptr_eq(&system, &lazy_system));
+    assert_terminated(
+        reconcile(Config::new())
+            .await
+            .expect_err("lazy reconciliation must not restart a terminated process runtime"),
+    );
+    assert_terminated(
+        lazy_system
+            .reload_config(Config::new())
+            .await
+            .expect_err("config reload must not restart a terminated process runtime"),
+    );
+    assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated);
+}

crates/notify/tests/legacy_initialize_retry.rs

@@ -0,0 +1,44 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
+use rustfs_config::server_config::{Config, KVS};
+use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT};
+use rustfs_notify::{initialize, notification_system};
+
+#[tokio::test]
+async fn failed_legacy_initialize_can_retry_the_stable_singleton() {
+    let mut invalid_target = KVS::new();
+    invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string());
+    invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string());
+    let mut invalid_config = Config::new();
+    invalid_config
+        .0
+        .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string())
+        .or_default()
+        .insert("primary".to_string(), invalid_target);
+
+    initialize(invalid_config)
+        .await
+        .expect_err("invalid first target activation should fail");
+    initialize(Config::new())
+        .await
+        .expect("legacy initialize should retry after the configuration is fixed");
+
+    notification_system()
+        .expect("stable singleton should remain available")
+        .shutdown_checked()
+        .await
+        .expect("test runtime should terminate");
+}