diff --git a/Cargo.lock b/Cargo.lock index 3bf42650e..c7e109c42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9537,8 +9537,10 @@ dependencies = [ "serde", "serde_json", "starshard", + "tempfile", "thiserror 2.0.19", "tokio", + "tokio-util", "tracing", "tracing-subscriber", "url", @@ -9955,6 +9957,7 @@ dependencies = [ "chrono", "criterion", "deadpool-postgres", + "futures-util", "hashbrown 0.17.1", "hyper", "hyper-rustls", @@ -9964,6 +9967,7 @@ dependencies = [ "mysql_async", "parking_lot", "pulsar", + "rayon", "rcgen", "redis", "regex", @@ -9988,6 +9992,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", + "tokio-util", "tracing", "tracing-subscriber", "url", diff --git a/crates/e2e_test/src/notification_webhook_test.rs b/crates/e2e_test/src/notification_webhook_test.rs index 9d07182f9..74c6fc093 100644 --- a/crates/e2e_test/src/notification_webhook_test.rs +++ b/crates/e2e_test/src/notification_webhook_test.rs @@ -156,7 +156,43 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver Result<(String, Arc, thread::JoinHandle<()>), BoxError> { +struct HttpsEventCollector { + endpoint: String, + running: Arc, + handle: Option>, + events: mpsc::UnboundedReceiver, +} + +impl HttpsEventCollector { + fn endpoint(&self) -> &str { + &self.endpoint + } + + fn events_mut(&mut self) -> &mut mpsc::UnboundedReceiver { + &mut self.events + } + + fn shutdown(&mut self) -> TestResult { + self.running.store(false, Ordering::Relaxed); + if let Ok(parsed) = self.endpoint.parse::() + && let Some(port) = parsed.port() + { + let _ = std::net::TcpStream::connect(("127.0.0.1", port)); + } + if let Some(handle) = self.handle.take() { + handle.join().map_err(|_| "https event collector thread panicked")?; + } + Ok(()) + } +} + +impl Drop for HttpsEventCollector { + fn drop(&mut self) { + let _ = self.shutdown(); + } +} + +fn spawn_https_event_collector(ca_path: &Path) -> Result { use rustls::{ ServerConfig, pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}, @@ -188,48 +224,103 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<(String, Arc { let config = Arc::clone(&server_config); - handle_https_probe(stream, config); + let tx = tx.clone(); + connections.push(thread::spawn(move || { + let _ = handle_https_request(stream, config, tx); + })); } Err(err) if err.kind() == ErrorKind::WouldBlock => thread::sleep(Duration::from_millis(20)), Err(_) => break, } } + for connection in connections { + let _ = connection.join(); + } }); - Ok((format!("https://{endpoint_host}:{}/events", addr.port()), running, handle)) + Ok(HttpsEventCollector { + endpoint: format!("https://{endpoint_host}:{}/events", addr.port()), + running, + handle: Some(handle), + events, + }) } -fn handle_https_probe(stream: std::net::TcpStream, server_config: Arc) { - use std::io::{Read, Write}; - - let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); - let _ = stream.set_write_timeout(Some(Duration::from_secs(5))); - let Ok(connection) = rustls::ServerConnection::new(server_config) else { - return; +fn read_sync_http_message(stream: &mut R) -> Result<(String, Vec), BoxError> { + let mut buffer = Vec::new(); + let mut chunk = [0_u8; 4096]; + let header_end = loop { + let read = stream.read(&mut chunk)?; + if read == 0 { + return Err("connection closed before request headers were complete".into()); + } + buffer.extend_from_slice(&chunk[..read]); + if let Some(pos) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + break pos; + } }; - let mut tls_stream = rustls::StreamOwned::new(connection, stream); - let mut buf = [0u8; 1024]; - if tls_stream.read(&mut buf).is_err() { - return; + + let header_text = std::str::from_utf8(&buffer[..header_end])?; + let mut lines = header_text.split("\r\n"); + let method = lines + .next() + .and_then(|line| line.split_whitespace().next()) + .ok_or("missing request method")? + .to_string(); + let mut content_length = 0usize; + for line in lines { + if let Some((name, value)) = line.split_once(':') + && name.trim().eq_ignore_ascii_case("content-length") + { + content_length = value.trim().parse()?; + } } - let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n"; - let _ = tls_stream.write_all(response.as_bytes()); - let _ = tls_stream.flush(); + + let body_offset = header_end + 4; + while buffer.len().saturating_sub(body_offset) < content_length { + let read = stream.read(&mut chunk)?; + if read == 0 { + return Err("connection closed before request body was complete".into()); + } + buffer.extend_from_slice(&chunk[..read]); + } + Ok((method, buffer[body_offset..body_offset + content_length].to_vec())) } -fn stop_https_event_collector(endpoint: &str, running: Arc, handle: thread::JoinHandle<()>) -> TestResult { - running.store(false, Ordering::Relaxed); - if let Ok(parsed) = endpoint.parse::() - && let Some(port) = parsed.port() - { - let _ = std::net::TcpStream::connect(("127.0.0.1", port)); +fn handle_https_request( + stream: std::net::TcpStream, + server_config: Arc, + tx: mpsc::UnboundedSender, +) -> Result<(), BoxError> { + use std::io::Write; + + stream.set_nonblocking(false)?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + let connection = rustls::ServerConnection::new(server_config)?; + let mut tls_stream = rustls::StreamOwned::new(connection, stream); + let (method, body) = read_sync_http_message(&mut tls_stream)?; + let response = "HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n"; + tls_stream.write_all(response.as_bytes())?; + tls_stream.flush()?; + tls_stream.conn.send_close_notify(); + while tls_stream.conn.wants_write() { + tls_stream.conn.write_tls(&mut tls_stream.sock)?; + } + let _ = tls_stream.sock.shutdown(std::net::Shutdown::Write); + if method == "POST" + && !body.is_empty() + && let Ok(event) = serde_json::from_slice(&body) + { + let _ = tx.send(event); } - handle.join().map_err(|_| "https event collector thread panicked")?; Ok(()) } @@ -415,17 +506,20 @@ async fn wait_for_target_registered(env: &RustFSTestEnvironment, target_name: &s Err(format!("target {target_name} was not registered in admin ARNs").into()) } -async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) -> TestResult { +async fn wait_for_target_online(env: &RustFSTestEnvironment, target_name: &str) -> TestResult { let url = format!("{}/rustfs/admin/v3/target/list", env.url); for _ in 0..40 { let response = signed_admin_request(env, http::Method::GET, &url, None).await?; if response.status() == StatusCode::OK { let body: Value = serde_json::from_slice(&response.bytes().await?)?; + if body["notify_enabled"].as_bool() != Some(true) { + return Err(format!("admin target list did not report notify_enabled=true: {body}").into()); + } let listed = body["notification_endpoints"].as_array().is_some_and(|endpoints| { endpoints.iter().any(|endpoint| { endpoint["account_id"].as_str() == Some(target_name) && endpoint["service"].as_str() == Some("webhook") - && endpoint["status"].as_str().is_some() + && endpoint["status"].as_str() == Some("online") }) }); if listed { @@ -434,7 +528,7 @@ async fn wait_for_target_listed(env: &RustFSTestEnvironment, target_name: &str) } tokio::time::sleep(Duration::from_millis(250)).await; } - Err(format!("target {target_name} was not listed in admin targets").into()) + Err(format!("target {target_name} did not become online in admin targets").into()) } /// Binds a bucket to a webhook target for ObjectCreated:*/ObjectRemoved:* events, @@ -484,11 +578,11 @@ fn trimmed_etag(value: Option<&str>) -> Option { // --------------------------------------------------------------------------- /// Regression for rustfs#5052: with the notify module enabled through -/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must be accepted -/// and remain visible in the admin target list. +/// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become +/// online and receive a real S3 event POST. #[tokio::test] #[serial] -async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult { +async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -496,22 +590,40 @@ async fn test_https_webhook_target_lists_with_notify_env_enabled() -> TestResult .await?; let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem"); - let (endpoint, running, handle) = spawn_https_event_collector(&ca_path)?; + let mut collector = spawn_https_event_collector(&ca_path)?; let target = "peri1https"; + let bucket = "peri1-https-events"; + let key = "uploads/https.dat"; + let client = env.create_s3_client(); + client.create_bucket().bucket(bucket).send().await?; configure_webhook_target_with_key_values( &env, target, vec![ - ("endpoint", endpoint.clone()), + ("endpoint", collector.endpoint().to_string()), ("client_ca", ca_path.to_string_lossy().into_owned()), ], ) .await?; - wait_for_target_listed(&env, target).await?; + wait_for_target_online(&env, target).await?; + wait_for_target_registered(&env, target).await?; + put_notification_config(&client, bucket, target, "uploads/", ".dat").await?; + + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"https webhook event body")) + .send() + .await?; + let event = wait_for_event(collector.events_mut(), key, "s3:ObjectCreated:", Duration::from_secs(20)).await?; + assert_eq!(event["EventName"].as_str(), Some("s3:ObjectCreated:Put")); + assert_eq!(event["Records"][0]["s3"]["bucket"]["name"].as_str(), Some(bucket)); + assert_eq!(event_key(&event).as_deref(), Some(key)); env.stop_server(); - stop_https_event_collector(&endpoint, running, handle)?; + collector.shutdown()?; Ok(()) } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index da875822e..c7f186c91 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -241,8 +241,10 @@ pub mod config { pub use crate::config::com::{ COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS, ServerConfigCorruptError, delete_config, is_server_config_corrupt_error, lookup_configs, read_config, - read_config_no_lock, read_config_with_metadata, read_config_without_migrate, save_config, save_config_with_opts, - save_server_config, try_migrate_server_config, + read_config_no_lock, read_config_with_metadata, read_config_without_migrate, read_config_without_migrate_no_lock, + read_existing_server_config_no_lock, save_config, save_config_no_lock, save_config_with_opts, save_server_config, + save_server_config_no_lock, try_migrate_server_config, with_config_object_read_lock, with_config_object_write_lock, + with_server_config_read_lock, with_server_config_write_lock, }; } @@ -383,9 +385,9 @@ pub mod rio { pub mod rpc { pub use crate::cluster::rpc::{ - LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, - gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, + LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, + gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index 67616dec4..cfa97cd60 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -37,8 +37,8 @@ pub use http_auth::{ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport; pub use internode_data_transport::build_internode_data_transport_from_env; pub use peer_rest_client::{ - PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, - ScannerPeerActivity, + PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG, + SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity, }; pub(crate) use peer_s3_client::heal_bucket_local_on_disks; pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, S3PeerSys}; diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index f8c89871c..d29d56e8d 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -17,12 +17,15 @@ use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET}; use crate::error::{Error, Result}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use crate::runtime::sources as runtime_sources; +use crate::set_disk::get_lock_acquire_timeout; use crate::storage_api_contracts::{ admin::StorageAdminApi, heal::HealOperations, + namespace::NamespaceLocking, object::{DeletedObject, EcstoreObjectIO, ObjectIO, ObjectOperations, ObjectToDelete}, range::HTTPRangeSpec, }; +use crate::store::ECStore; use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_config::audit::{ @@ -45,10 +48,94 @@ use serde_json::{Map, Value}; use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; use std::sync::{Arc, RwLock}; +use tokio::sync::RwLock as AsyncRwLock; use tracing::{debug, error, info, instrument, warn}; pub const CONFIG_PREFIX: &str = "config"; -const CONFIG_FILE: &str = "config.json"; +const SERVER_CONFIG_OBJECT: &str = "config/config.json"; + +// Server-config lock order: SERVER_CONFIG_LOCK -> distributed namespace lock +// for SERVER_CONFIG_OBJECT. Readers and writers must never reverse this order. +static SERVER_CONFIG_LOCK: LazyLock> = LazyLock::new(|| AsyncRwLock::new(())); + +fn config_task_join_error(operation: &'static str, error: tokio::task::JoinError) -> Error { + let outcome = if error.is_cancelled() { "cancelled" } else { "panicked" }; + Error::other(format!("{operation} task {outcome}")) +} + +/// Runs one complete server-config transaction while holding both the local +/// process guard and the distributed namespace write lock for `config.json`. +/// The operation must use the corresponding no-lock read/save functions. +pub async fn with_server_config_write_lock(store: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + tokio::spawn(async move { + // Lock order: SERVER_CONFIG_LOCK -> namespace write lock. + let _local_guard = SERVER_CONFIG_LOCK.write().await; + let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; + let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; + Ok(operation().await) + }) + .await + .map_err(|error| config_task_join_error("server config write", error))? +} + +/// Reads and synchronously publishes one server-config snapshot while holding +/// a shared namespace lock. Concurrent peer reloads may proceed together, but +/// no writer can interleave between the snapshot read and generation accept. +pub async fn with_server_config_read_lock(store: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + tokio::spawn(async move { + // Lock order: SERVER_CONFIG_LOCK -> namespace read lock. + let _local_guard = SERVER_CONFIG_LOCK.read().await; + let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, SERVER_CONFIG_OBJECT).await?; + let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?; + Ok(operation().await) + }) + .await + .map_err(|error| config_task_join_error("server config read", error))? +} + +/// Runs a cancellation-safe transaction under the distributed write lock for +/// one metadata config object. The operation must use no-lock object I/O. +pub async fn with_config_object_write_lock(store: Arc, object: String, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + tokio::spawn(async move { + let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + let _write_guard = namespace_lock.get_write_lock(get_lock_acquire_timeout()).await?; + Ok(operation().await) + }) + .await + .map_err(|error| config_task_join_error("config object write", error))? +} + +/// Runs one read-and-publish transaction under the distributed read lock for +/// a metadata config object. The operation must use no-lock object I/O. +pub async fn with_config_object_read_lock(store: Arc, object: String, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + tokio::spawn(async move { + let namespace_lock = store.new_ns_lock(RUSTFS_META_BUCKET, &object).await?; + let _read_guard = namespace_lock.get_read_lock(get_lock_acquire_timeout()).await?; + Ok(operation().await) + }) + .await + .map_err(|error| config_task_join_error("config object read", error))? +} /// Environment variable gating the startup fallback to the default server /// config when the persisted `config.json` object is corrupt beyond repair @@ -393,6 +480,31 @@ where .await } +pub async fn save_config_no_lock(api: Arc, file: &str, data: Vec) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + save_config_with_opts( + api, + file, + data, + &ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }, + ) + .await +} + #[instrument(skip(api))] pub async fn delete_config(api: Arc, file: &str) -> Result<()> where @@ -462,8 +574,17 @@ where Ok(cfg) } +async fn new_and_save_server_config_no_lock(api: Arc) -> Result +where + S: EcstoreObjectIO + StorageAdminApi, +{ + let cfg = new_server_config(); + save_server_config_no_lock(api, &cfg).await?; + Ok(cfg) +} + fn get_config_file() -> String { - format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}") + SERVER_CONFIG_OBJECT.to_string() } fn storage_class_kvs_mut(cfg: &mut Config) -> &mut KVS { @@ -1267,13 +1388,17 @@ where } /// Handle the situation where the configuration file does not exist, create and save a new configuration -async fn handle_missing_config(api: Arc, context: &str) -> Result +async fn handle_missing_config(api: Arc, context: &str, namespace_lock_held: bool) -> Result where S: EcstoreObjectIO + StorageAdminApi, { warn!("Configuration not found ({}): Start initializing new configuration", context); let cfg = if runtime_sources::first_cluster_node_is_local().await { - new_and_save_server_config(api.clone()).await? + if namespace_lock_held { + new_and_save_server_config_no_lock(api.clone()).await? + } else { + new_and_save_server_config(api.clone()).await? + } } else { new_server_config() }; @@ -1288,6 +1413,46 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result { } pub async fn read_config_without_migrate(api: Arc) -> Result +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + StorageAdminApi, +{ + read_config_without_migrate_inner(api, false).await +} + +/// Reads the server config while an upper layer holds the namespace write lock +/// for [`SERVER_CONFIG_OBJECT`]. Missing-config initialization uses the matching +/// no-lock save path and therefore cannot recursively acquire the same lock. +pub async fn read_config_without_migrate_no_lock(api: Arc) -> Result +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + StorageAdminApi, +{ + read_config_without_migrate_inner(api, true).await +} + +/// Reads an already-initialized server config while a caller owns a namespace +/// read lock. This never initializes or migrates a missing object. +pub async fn read_existing_server_config_no_lock(api: Arc) -> Result { + let data = read_config_no_lock(api, SERVER_CONFIG_OBJECT).await?; + Ok(decode_persisted_server_config(&data)?.merge()) +} + +async fn read_config_without_migrate_inner(api: Arc, namespace_lock_held: bool) -> Result where S: ObjectIO< Error = Error, @@ -1303,13 +1468,13 @@ where // Try to read the configuration file match read_config_no_lock(api.clone(), &config_file).await { - Ok(data) => read_server_config(api, &data).await, - Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await, + Ok(data) => read_server_config(api, &data, namespace_lock_held).await, + Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration", namespace_lock_held).await, Err(err) => handle_config_read_error(err, &config_file), } } -async fn read_server_config(api: Arc, data: &[u8]) -> Result +async fn read_server_config(api: Arc, data: &[u8], namespace_lock_held: bool) -> Result where S: EcstoreObjectIO + StorageAdminApi, { @@ -1324,7 +1489,9 @@ where let cfg = decode_persisted_server_config(&cfg_data)?; return Ok(cfg.merge()); } - Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await, + Err(Error::ConfigNotFound) => { + return handle_missing_config(api, "Read alternate configuration", namespace_lock_held).await; + } Err(err) => return handle_config_read_error(err, &config_file), } } @@ -1534,6 +1701,38 @@ fn fallback_server_config_after_corruption(err: Error, config_file: &str, recove } pub async fn save_server_config(api: Arc, cfg: &Config) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + save_server_config_inner(api, cfg, false).await +} + +/// Saves the server config while an upper layer holds the namespace write +/// lock for [`SERVER_CONFIG_OBJECT`]. +pub async fn save_server_config_no_lock(api: Arc, cfg: &Config) -> Result<()> +where + S: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + >, +{ + save_server_config_inner(api, cfg, true).await +} + +async fn save_server_config_inner(api: Arc, cfg: &Config, no_lock: bool) -> Result<()> where S: ObjectIO< Error = Error, @@ -1546,7 +1745,11 @@ where >, { let config_file = get_config_file(); - let existing = match read_config(api.clone(), &config_file).await { + let existing = match if no_lock { + read_config_no_lock(api.clone(), &config_file).await + } else { + read_config(api.clone(), &config_file).await + } { Ok(v) => Some(v), Err(Error::ConfigNotFound) => None, Err(err) => { @@ -1570,7 +1773,21 @@ where return Ok(()); } - save_config(api, &config_file, data).await + if no_lock { + save_config_with_opts( + api, + &config_file, + data, + &ObjectOptions { + max_parity: true, + no_lock: true, + ..Default::default() + }, + ) + .await + } else { + save_config(api, &config_file, data).await + } } pub async fn lookup_configs(cfg: &mut Config, api: Arc) -> Result<()> @@ -1630,9 +1847,9 @@ where #[cfg(test)] mod tests { use super::{ - apply_dynamic_config_for_sub_sys_with, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob, - is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, read_config_with_metadata, - storage_class_kvs_mut, + apply_dynamic_config_for_sub_sys_with, config_task_join_error, configs_semantically_equal, decode_server_config_blob, + encode_server_config_blob, is_standard_object_server_config, lookup_configs, read_config, read_config_preserve_empty, + read_config_with_metadata, storage_class_kvs_mut, }; use crate::config::{audit, notify, oidc}; use crate::disk::endpoint::Endpoint; @@ -1652,6 +1869,17 @@ mod tests { use rustfs_config::{ DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, }; + + #[tokio::test] + async fn config_task_join_error_does_not_expose_panic_payload() { + let join_error = tokio::spawn(async { panic!("do-not-expose-payload") }) + .await + .expect_err("test task should panic"); + + let rendered = config_task_join_error("server config write", join_error).to_string(); + assert!(rendered.contains("panicked")); + assert!(!rendered.contains("do-not-expose-payload")); + } use rustfs_lock::client::LockClient; use rustfs_lock::client::local::LocalClient; use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats}; diff --git a/crates/ecstore/src/services/notification_sys.rs b/crates/ecstore/src/services/notification_sys.rs index aea1d690e..67316c623 100644 --- a/crates/ecstore/src/services/notification_sys.rs +++ b/crates/ecstore/src/services/notification_sys.rs @@ -253,7 +253,7 @@ impl NotificationSys { join_all(futures).await } - pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec { + async fn signal_dynamic_config(&self, sub_sys: &str, dry_run: bool) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter() { let sub_sys = sub_sys.to_string(); @@ -263,7 +263,7 @@ impl NotificationSys { .signal_service( crate::cluster::rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC, &sub_sys, - false, + dry_run, SystemTime::UNIX_EPOCH, ) .await @@ -288,6 +288,14 @@ impl NotificationSys { join_all(futures).await } + pub async fn preflight_dynamic_config(&self, sub_sys: &str) -> Vec { + self.signal_dynamic_config(sub_sys, true).await + } + + pub async fn reload_dynamic_config(&self, sub_sys: &str) -> Vec { + self.signal_dynamic_config(sub_sys, false).await + } + pub async fn refresh_config_snapshot(&self) -> Vec { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter() { diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 8af36ea9c..41c5ad0c9 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -872,6 +872,8 @@ impl ECStore { let idx = if opts.data_movement && opts.version_id.is_some() { self.select_data_movement_pool_idx(bucket, &object, data.size(), opts, false) .await? + } else if opts.no_lock { + self.get_pool_idx_no_lock(bucket, &object, data.size()).await? } else { self.get_pool_idx(bucket, &object, data.size()).await? }; diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index 9768b9a61..91b434068 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -44,6 +44,7 @@ serde = { workspace = true, features = ["derive"] } starshard = { workspace = true, features = ["rayon", "async", "serde"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] } +tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } metrics = { workspace = true } @@ -62,6 +63,7 @@ axum = { workspace = true } rustfs-utils = { workspace = true, features = ["path"] } serde_json = { workspace = true, features = ["raw_value"] } criterion = { workspace = true, features = ["html_reports"] } +tempfile = { workspace = true } [lints] workspace = true diff --git a/crates/notify/src/config_manager.rs b/crates/notify/src/config_manager.rs index fab815865..91df663d0 100644 --- a/crates/notify/src/config_manager.rs +++ b/crates/notify/src/config_manager.rs @@ -13,89 +13,83 @@ // limitations under the License. use crate::{ - Event, NotificationError, registry::TargetRegistry, resolve_notify_object_store_handle, rule_engine::NotifyRuleEngine, + NotificationError, + lifecycle::{NotificationRuntimeState, NotifyLifecycleCoordinator}, + registry::TargetRegistry, + resolve_notify_object_store_handle, + rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, + with_notify_server_config_read_lock, with_notify_server_config_write_lock, }; use rustfs_config::notify::{ NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::server_config::{Config, KVS}; -use rustfs_targets::{Target, arn::TargetID}; -use std::sync::{Arc, LazyLock}; -use tokio::sync::{Mutex, RwLock}; +use rustfs_targets::arn::TargetID; +use std::sync::Arc; +use tokio::sync::RwLock; use tracing::{debug, info}; -/// Serializes the read-modify-write sequence over the persisted notify server -/// config. The persisted config is a single process-global resource (there is -/// only one backing object store), so without this guard two concurrent updates -/// can both read the same base config, apply disjoint changes, and race their -/// full-config writes — the later write silently overwrites the earlier one, -/// losing updates. Holding this mutex across the whole read→modify→write makes -/// concurrent updates apply serially so every change is preserved. -/// -/// The lock is only ever acquired inside `update_server_config`; it never nests -/// with the per-manager `config` RwLock (the in-memory reload runs after this -/// guard is released), so it introduces no lock-ordering risk. -static NOTIFY_CONFIG_RMW_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - const LOG_COMPONENT_NOTIFY: &str = "notify"; const LOG_SUBSYSTEM_CONFIG: &str = "config"; -const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle"; const EVENT_NOTIFY_CONFIG_UPDATE: &str = "notify_config_update"; #[derive(Debug)] enum NotifyConfigStoreError { + Lock(String), StorageNotAvailable, Read(String), Save(String), } -async fn update_server_config(modifier: F) -> Result, NotifyConfigStoreError> +async fn update_server_config( + modifier: F, + lifecycle: NotifyLifecycleCoordinator, +) -> Result, NotifyConfigStoreError> where - F: FnMut(&mut Config) -> bool, + F: FnMut(&mut Config) -> bool + Send + 'static, { let Some(store) = resolve_notify_object_store_handle() else { return Err(NotifyConfigStoreError::StorageNotAvailable); }; + let store_for_read = store.clone(); let store_for_save = store.clone(); - serialized_read_modify_write( - modifier, - move || async move { - crate::read_notify_server_config_without_migrate(store) - .await - .map_err(NotifyConfigStoreError::Read) - }, - move |config| async move { - crate::save_notify_server_config(store_for_save, &config) - .await - .map_err(NotifyConfigStoreError::Save) - }, - ) + with_notify_server_config_write_lock(store, move || { + read_modify_write( + modifier, + move || async move { + crate::read_notify_server_config_without_migrate_no_lock(store_for_read) + .await + .map_err(NotifyConfigStoreError::Read) + }, + move |config| async move { + crate::save_notify_server_config_no_lock(store_for_save, &config) + .await + .map_err(NotifyConfigStoreError::Save) + }, + move |config| lifecycle.update_config(config), + ) + }) .await + .map_err(NotifyConfigStoreError::Lock)? } -/// Runs a `read → modify → write` over the persisted notify config while holding -/// [`NOTIFY_CONFIG_RMW_LOCK`], so concurrent updates serialize and cannot clobber -/// each other's changes (backlog#968). `read`/`save` are injected so the exact -/// production serialization path can be exercised in tests without a live store. -async fn serialized_read_modify_write( +async fn read_modify_write( mut modifier: F, read: R, save: S, -) -> Result, NotifyConfigStoreError> + publish: P, +) -> Result, NotifyConfigStoreError> where F: FnMut(&mut Config) -> bool, R: FnOnce() -> RFut, RFut: std::future::Future>, S: FnOnce(Config) -> SFut, SFut: std::future::Future>, + P: FnOnce(Config) -> T, { - // Hold the RMW lock across the entire read→modify→write so concurrent - // updates serialize and cannot clobber each other's changes (backlog#968). - let _rmw_guard = NOTIFY_CONFIG_RMW_LOCK.lock().await; - let mut new_config = read().await?; if !modifier(&mut new_config) { @@ -104,7 +98,7 @@ where save(new_config.clone()).await?; - Ok(Some(new_config)) + Ok(Some(publish(new_config))) } pub(crate) fn notify_configuration_hint() -> String { @@ -142,9 +136,8 @@ pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> #[derive(Clone)] pub struct NotifyConfigManager { config: Arc>, - registry: Arc, + lifecycle: NotifyLifecycleCoordinator, rule_engine: NotifyRuleEngine, - runtime_facade: NotifyRuntimeFacade, } impl NotifyConfigManager { @@ -154,64 +147,21 @@ impl NotifyConfigManager { rule_engine: NotifyRuleEngine, runtime_facade: NotifyRuntimeFacade, ) -> Self { + let lifecycle = NotifyLifecycleCoordinator::new(config.clone(), registry, runtime_facade); Self { config, - registry, + lifecycle, rule_engine, - runtime_facade, } } + pub(crate) fn lifecycle(&self) -> NotifyLifecycleCoordinator { + self.lifecycle.clone() + } + pub async fn init(&self) -> Result<(), NotificationError> { - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "initializing", - "notify runtime lifecycle" - ); - - let config = { - let guard = self.config.read().await; - debug!( - subsystem_count = guard.0.len(), - "Initializing notification system with configuration summary" - ); - guard.clone() - }; - - let targets: Vec + Send + Sync>> = self.registry.create_targets_from_config(&config).await?; - - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "targets_created", - target_count = targets.len(), - "notify runtime lifecycle" - ); - if targets.is_empty() { - debug!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "idle", - reason = "no_targets_configured", - hint = %notify_configuration_hint(), - "notify runtime lifecycle" - ); - } - - let activation = self.runtime_facade.activate_targets_with_replay(targets).await; - self.runtime_facade.replace_targets(activation).await?; - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "initialized", - "notify runtime lifecycle" - ); - Ok(()) + let config = self.config.read().await.clone(); + self.lifecycle.set_mode(true, Some(config)).wait().await } pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> { @@ -227,6 +177,7 @@ impl NotifyConfigManager { let ttype = target_type.to_lowercase(); let tname = target_id.id.to_lowercase(); + let log_target_id = target_id.clone(); // Guard against orphaning bucket notification rules (backlog#979). Removing a // target while a bucket rule still references it would leave a dangling @@ -240,7 +191,7 @@ impl NotifyConfigManager { ))); } - self.update_config_and_reload(|config| { + self.update_config_and_reload(move |config| { let mut changed = false; if let Some(targets_of_type) = config.0.get_mut(&ttype) { if targets_of_type.remove(&tname).is_some() { @@ -249,7 +200,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "remove_target", - target_id = %target_id, + target_id = %log_target_id, result = "removed", "notify config update" ); @@ -265,7 +216,7 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "remove_target", - target_id = %target_id, + target_id = %log_target_id, result = "not_found", "notify config update" ); @@ -287,7 +238,7 @@ impl NotifyConfigManager { ); let ttype = target_type.to_lowercase(); let tname = target_name.to_lowercase(); - self.update_config_and_reload(|config| { + self.update_config_and_reload(move |config| { config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone()); true }) @@ -316,7 +267,7 @@ impl NotifyConfigManager { ))); } - self.update_config_and_reload(|config| { + self.update_config_and_reload(move |config| { let mut changed = false; if let Some(targets) = config.0.get_mut(&ttype) { if targets.remove(&tname).is_some() { @@ -332,8 +283,8 @@ impl NotifyConfigManager { component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_CONFIG, action = "remove_target_config", - target_type = %target_type, - target_name = %target_name, + target_type = %ttype, + target_name = %tname, result = "not_found", "notify config update" ); @@ -348,81 +299,62 @@ impl NotifyConfigManager { } pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> { - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "reloading", - "notify runtime lifecycle" - ); + self.lifecycle.set_mode(true, Some(new_config)).wait().await + } - self.update_config(new_config.clone()).await; + pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> { + let Some(store) = resolve_notify_object_store_handle() else { + return Err(NotificationError::StorageNotAvailable( + "Failed to load target configuration: server storage not initialized".to_string(), + )); + }; + self.reload_persisted_config_from_store(store).await + } - // Stop the currently running replay workers *before* activating the new ones - // (backlog#970). Each replay worker drains a per-target persisted store; if the - // new workers start while the old ones are still running against the same - // stores, both drain the same queues and re-deliver events. `replace_targets` - // below also stops workers, but only after `activate_targets_with_replay` has - // already spawned the new ones — so without this explicit stop-before-start - // there is a window where old and new workers overlap. (The full "signal + - // join" shutdown lives in the targets crate and is tracked under the same - // issue; this reorders the notify-side lifecycle.) - self.runtime_facade.stop_replay_workers().await; - - let targets: Vec + Send + Sync>> = self - .registry - .create_targets_from_config(&new_config) - .await - .map_err(NotificationError::Target)?; - - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "targets_created", - target_count = targets.len(), - "notify runtime lifecycle" - ); - if targets.is_empty() { - debug!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "idle", - reason = "no_targets_configured", - hint = %notify_configuration_hint(), - "notify runtime lifecycle" - ); + pub async fn reload_persisted_config_from_store(&self, store: Arc) -> Result<(), NotificationError> { + if self.lifecycle.state() == NotificationRuntimeState::Terminated { + return Err(NotificationError::Initialization("Notification runtime has terminated".to_string())); } - let activation = self.runtime_facade.activate_targets_with_replay(targets).await; - self.runtime_facade.replace_targets(activation).await?; - info!( - event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_CONFIG, - state = "reloaded", - "notify runtime lifecycle" - ); + let read_store = store.clone(); + let config_cache = self.config.clone(); + let lifecycle = self.lifecycle.clone(); + let transition = with_notify_server_config_read_lock(store, move || async move { + let config = crate::read_existing_notify_server_config_no_lock(read_store) + .await + .map_err(NotificationError::ReadConfig)?; + Ok::<_, NotificationError>(if *config_cache.read().await == config && lifecycle.is_converged() { + None + } else { + Some(lifecycle.update_config(config)) + }) + }) + .await + .map_err(NotificationError::StorageNotAvailable)??; + + if let Some(transition) = transition { + transition.wait().await?; + } Ok(()) } - async fn update_config(&self, new_config: Config) { - let mut config = self.config.write().await; - *config = new_config; - } - - async fn update_config_and_reload(&self, mut modifier: F) -> Result<(), NotificationError> + async fn update_config_and_reload(&self, modifier: F) -> Result<(), NotificationError> where - F: FnMut(&mut Config) -> bool, + F: FnMut(&mut Config) -> bool + Send + 'static, { - let Some(new_config) = update_server_config(&mut modifier).await.map_err(|err| match err { - NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( - "Failed to save target configuration: server storage not initialized".to_string(), - ), - NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), - NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), - })? + if self.lifecycle.state() == NotificationRuntimeState::Terminated { + return Err(NotificationError::Initialization("Notification runtime has terminated".to_string())); + } + let Some(transition) = update_server_config(modifier, self.lifecycle.clone()) + .await + .map_err(|err| match err { + NotifyConfigStoreError::Lock(err) => NotificationError::StorageNotAvailable(err), + NotifyConfigStoreError::StorageNotAvailable => NotificationError::StorageNotAvailable( + "Failed to save target configuration: server storage not initialized".to_string(), + ), + NotifyConfigStoreError::Read(err) => NotificationError::ReadConfig(err), + NotifyConfigStoreError::Save(err) => NotificationError::SaveConfig(err), + })? else { debug!( event = EVENT_NOTIFY_CONFIG_UPDATE, @@ -443,15 +375,15 @@ impl NotifyConfigManager { result = "updated", "notify config update" ); - self.reload_config(new_config).await + transition.wait().await } } #[cfg(test)] mod tests { - use super::{NotifyConfigManager, NotifyConfigStoreError, runtime_target_id_for_subsystem, serialized_read_modify_write}; - use crate::NotificationError; + use super::{NotifyConfigManager, NotifyConfigStoreError, read_modify_write, runtime_target_id_for_subsystem}; use crate::rules::RulesMap; + use crate::{NotificationError, NotificationRuntimeState}; use crate::{ integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, @@ -464,7 +396,10 @@ mod tests { use rustfs_s3_types::EventName; use rustfs_targets::ReplayWorkerManager; use rustfs_targets::arn::TargetID; - use std::sync::Arc; + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; use tokio::sync::{RwLock, Semaphore}; fn build_manager() -> NotifyConfigManager { @@ -474,9 +409,10 @@ mod tests { let rule_engine = NotifyRuleEngine::new(); let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); let target_list = notifier.target_list(); - let runtime_facade = NotifyRuntimeFacade::new( + let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate( target_list, Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), Arc::new(Semaphore::new(4)), metrics, ); @@ -524,78 +460,39 @@ mod tests { .reload_config(Config::default()) .await .expect("reload_config should succeed for empty targets"); + assert!(matches!(manager.lifecycle().state(), NotificationRuntimeState::TargetsEnabled { .. })); } - // Regression test for backlog#968: the read-modify-write over the persisted - // notify config must be serialized. Many tasks concurrently add a distinct - // target through the same production RMW path (`serialized_read_modify_write`, - // which holds the global RMW lock across read→modify→write) against a shared - // in-memory backend. Every update must survive — no lost updates. Without the - // lock, concurrent tasks would read the same base config and clobber each - // other's writes, leaving only a subset of targets. - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn concurrent_config_updates_preserve_all_targets() { - // Shared in-memory stand-in for the persisted config backend. - let backend = Arc::new(RwLock::new(Config::default())); - const TASKS: usize = 32; + #[tokio::test] + async fn read_modify_write_publishes_only_after_save() { + let saved = Arc::new(AtomicBool::new(false)); + let saved_by_writer = saved.clone(); + let observed_by_publisher = saved.clone(); - let mut handles = Vec::with_capacity(TASKS); - for idx in 0..TASKS { - let backend = backend.clone(); - handles.push(tokio::spawn(async move { - let ttype = NOTIFY_WEBHOOK_SUB_SYS.to_lowercase(); - let tname = format!("target-{idx}"); - - let read_backend = backend.clone(); - let save_backend = backend.clone(); - - let result = serialized_read_modify_write( - |config: &mut Config| { - config - .0 - .entry(ttype.clone()) - .or_default() - .insert(tname.clone(), KVS::default()); - true - }, - move || async move { - let snapshot = read_backend.read().await.clone(); - // Yield inside the critical section to widen the race window: - // if the RMW were not serialized, other tasks would read this - // same base snapshot and their writes would clobber ours. - tokio::task::yield_now().await; - Ok::<_, NotifyConfigStoreError>(snapshot) - }, - move |config: Config| async move { - *save_backend.write().await = config; - Ok::<_, NotifyConfigStoreError>(()) - }, - ) - .await - .expect("serialized RMW should succeed"); - assert!(result.is_some(), "modifier reported a change, expected Some(config)"); - })); - } - - for handle in handles { - handle.await.expect("update task should not panic"); - } - - let final_config = backend.read().await; - let webhook_targets = final_config - .0 - .get(&NOTIFY_WEBHOOK_SUB_SYS.to_lowercase()) - .expect("webhook subsystem should exist after updates"); - - assert_eq!( - webhook_targets.len(), - TASKS, - "all concurrent target additions must be preserved (no lost updates)" - ); - for idx in 0..TASKS { - let tname = format!("target-{idx}"); - assert!(webhook_targets.contains_key(&tname), "missing target {tname}: concurrent update was lost"); - } + read_modify_write( + |config| { + config + .0 + .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string()) + .or_default() + .insert("primary".to_string(), KVS::default()); + true + }, + || async { Ok::<_, NotifyConfigStoreError>(Config::default()) }, + move |_config| async move { + saved_by_writer.store(true, Ordering::Release); + Ok::<_, NotifyConfigStoreError>(()) + }, + move |_config| { + assert!( + observed_by_publisher.load(Ordering::Acquire), + "publication must observe the completed save" + ); + }, + ) + .await + .expect("read-modify-write should succeed") + .expect("changed config should publish"); } #[test] diff --git a/crates/notify/src/error.rs b/crates/notify/src/error.rs index 317ef7f99..d6c4954ed 100644 --- a/crates/notify/src/error.rs +++ b/crates/notify/src/error.rs @@ -15,6 +15,7 @@ use rustfs_targets::{TargetError, arn::TargetID}; use std::io; use thiserror::Error; +use tokio::task::JoinError; /// Errors related to the notification system's lifecycle. #[derive(Debug, Error)] @@ -70,3 +71,24 @@ pub enum NotificationError { #[error("Storage not available: {0}")] StorageNotAvailable(String), } + +pub(crate) fn transition_join_error(error: JoinError) -> NotificationError { + let reason = if error.is_cancelled() { "cancelled" } else { "panicked" }; + NotificationError::Initialization(format!("Notification lifecycle transition task {reason}")) +} + +#[cfg(test)] +mod tests { + use super::transition_join_error; + + #[tokio::test] + async fn transition_join_error_does_not_expose_panic_payload() { + let join_error = tokio::spawn(async { panic!("do-not-expose-payload") }) + .await + .expect_err("test task should panic"); + + let rendered = transition_join_error(join_error).to_string(); + assert!(rendered.contains("panicked")); + assert!(!rendered.contains("do-not-expose-payload")); + } +} diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 6eab1e73f..c33e47f34 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -14,38 +14,86 @@ use crate::{ BucketNotificationConfig, Event, EventArgs, LifecycleError, NotificationError, NotificationMetricSnapshot, - NotificationSystem, NotificationTargetMetricSnapshot, + NotificationSystem, NotificationTargetMetricSnapshot, error::transition_join_error, }; use rustfs_config::server_config::Config; use rustfs_s3_types::EventName; use rustfs_targets::arn::TargetID; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, LazyLock, Mutex, OnceLock, Weak}; use tracing::error; static NOTIFICATION_SYSTEM: OnceLock> = OnceLock::new(); +static LEGACY_INITIALIZATION: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + +enum LegacyInitialization { + Initializing(Weak), + Retryable(Weak), + Initialized, +} const LOG_COMPONENT_NOTIFY: &str = "notify"; const LOG_SUBSYSTEM_GLOBAL: &str = "global"; const EVENT_NOTIFY_GLOBAL_STATE: &str = "notify_global_state"; -/// Initialize the global notification system with the given configuration. -/// This function should only be called once throughout the application life cycle. -pub async fn initialize(config: Config) -> Result<(), NotificationError> { - // `new` is synchronous and responsible for creating instances - let system = NotificationSystem::new(config); - // `init` is asynchronous and responsible for performing I/O-intensive initialization - system.init().await?; +fn notification_system_or_init(config: Config) -> Arc { + NOTIFICATION_SYSTEM + .get_or_init(|| Arc::new(NotificationSystem::new(config))) + .clone() +} - match NOTIFICATION_SYSTEM.set(Arc::new(system)) { - Ok(_) => Ok(()), - Err(losing_system) => { - // Another initializer won the race. `init()` above already started this - // system's targets and replay workers, so simply dropping it would leak - // those background tasks. Shut the losing instance down cleanly before - // reporting the conflict (backlog#984). - losing_system.shutdown().await; - Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)) +/// Initialize the global notification system with the given configuration. +/// +/// This preserves the historical one-shot API contract. Server lifecycle code +/// that needs idempotent reconciliation should use [`reconcile`] instead. +pub async fn initialize(config: Config) -> Result<(), NotificationError> { + let system = { + let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner()); + match legacy.as_ref() { + Some(LegacyInitialization::Retryable(system)) => { + let Some(system) = system.upgrade() else { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); + }; + if !NOTIFICATION_SYSTEM.get().is_some_and(|global| Arc::ptr_eq(global, &system)) { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); + } + *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system))); + system + } + Some(LegacyInitialization::Initializing(_)) | Some(LegacyInitialization::Initialized) => { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); + } + None => { + if NOTIFICATION_SYSTEM.get().is_some() { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); + } + let system = Arc::new(NotificationSystem::new(config.clone())); + if NOTIFICATION_SYSTEM.set(system.clone()).is_err() { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); + } + *legacy = Some(LegacyInitialization::Initializing(Arc::downgrade(&system))); + system + } } - } + }; + + let task_system = system.clone(); + tokio::spawn(async move { + let result = task_system.set_targets_enabled(true, Some(config)).await; + let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner()); + if matches!( + legacy.as_ref(), + Some(LegacyInitialization::Initializing(current)) + if current.upgrade().is_some_and(|current| Arc::ptr_eq(¤t, &task_system)) + ) { + *legacy = Some(if result.is_ok() { + LegacyInitialization::Initialized + } else { + LegacyInitialization::Retryable(Arc::downgrade(&task_system)) + }); + } + result + }) + .await + .map_err(transition_join_error)? } /// Initialize the global notification system only for live in-process consumers. @@ -54,12 +102,24 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> { /// ListenBucketNotification clients can receive live events even when external /// notification targets are disabled. pub fn initialize_live_events() -> Result<(), NotificationError> { - let system = NotificationSystem::new(Config::new()); - - match NOTIFICATION_SYSTEM.set(Arc::new(system)) { - Ok(_) => Ok(()), - Err(_) => Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)), + if NOTIFICATION_SYSTEM + .set(Arc::new(NotificationSystem::new(Config::new()))) + .is_err() + { + return Err(NotificationError::Lifecycle(LifecycleError::AlreadyInitialized)); } + Ok(()) +} + +/// Ensures the stable process-wide live-event container exists. +pub fn ensure_live_events() -> Arc { + notification_system_or_init(Config::new()) +} + +/// Ensures the stable singleton exists and reconciles its target runtime. +pub async fn reconcile(config: Config) -> Result<(), NotificationError> { + let system = notification_system_or_init(config.clone()); + system.set_targets_enabled(true, Some(config)).await } /// Returns a handle to the global NotificationSystem instance. diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index a97f39cf7..65d1b5f66 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -16,7 +16,12 @@ use crate::notification_system_subscriber::NotificationSystemSubscriberView; use crate::notifier::{EventNotifier, TargetList}; use crate::services::NotifyServices; use crate::{ - Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine, + Event, + error::NotificationError, + lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState}, + pipeline::LiveEventHistory, + registry::TargetRegistry, + rule_engine::NotifyRuleEngine, rules::BucketNotificationConfig, }; use hashbrown::HashMap; @@ -160,11 +165,12 @@ impl NotificationMetrics { /// The notification system that integrates all components pub struct NotificationSystem { - /// The event notifier + /// Event dispatcher. Runtime target mutation remains lifecycle-owned. pub notifier: Arc, - /// The target registry + /// Target factory registry. Creating a target does not publish it. pub registry: Arc, - /// The current configuration + /// The current cached configuration. Runtime publication must still go + /// through the lifecycle methods on this type. pub config: Arc>, services: NotifyServices, } @@ -220,10 +226,12 @@ impl NotificationSystem { self.services.runtime_view.get_active_targets().await } - /// Gets the complete Target list, including both active and inactive Targets. - /// - /// # Return - /// An `Arc>` containing all Targets. + pub async fn config_snapshot(&self) -> Config { + self.config.read().await.clone() + } + + /// Gets a read-only runtime container handle. Public mutation methods on + /// `TargetList` are intentionally unavailable outside this crate. pub async fn get_all_targets(&self) -> Arc> { self.services.runtime_view.get_all_targets() } @@ -327,6 +335,43 @@ impl NotificationSystem { self.services.config_manager.reload_config(new_config).await } + /// Synchronously publishes a config generation without changing target mode. + pub fn publish_config(&self, new_config: Config) -> NotificationLifecycleTransition { + self.services.config_manager.lifecycle().update_config(new_config) + } + + /// Reconciles the cached and active configuration with the persisted + /// server config without changing the target-runtime mode. + pub async fn reload_persisted_config(&self) -> Result<(), NotificationError> { + self.services.config_manager.reload_persisted_config().await + } + + /// Reconciles from an explicitly selected storage context. + pub async fn reload_persisted_config_from_store(&self, store: Arc) -> Result<(), NotificationError> { + self.services.config_manager.reload_persisted_config_from_store(store).await + } + + /// Enables or suspends configured notification targets without replacing + /// the process-wide live-event container. + pub async fn set_targets_enabled(&self, enabled: bool, config: Option) -> Result<(), NotificationError> { + self.publish_targets_enabled(enabled, config).wait().await + } + + /// Synchronously accepts a target-runtime mode transition. The returned + /// receipt can be awaited after the caller releases its persistence lock. + pub fn publish_targets_enabled(&self, enabled: bool, config: Option) -> NotificationLifecycleTransition { + self.services.config_manager.lifecycle().set_mode(enabled, config) + } + + pub fn runtime_lifecycle_state(&self) -> NotificationRuntimeState { + self.services.config_manager.lifecycle().state() + } + + /// Returns whether the latest accepted lifecycle generation is active. + pub fn runtime_lifecycle_is_converged(&self) -> bool { + self.services.config_manager.lifecycle().is_converged() + } + /// Loads the bucket notification configuration pub async fn load_bucket_notification_config( &self, @@ -365,9 +410,13 @@ impl NotificationSystem { self.services.runtime_view.runtime_status_snapshot().await } - // Add a method to shut down the system pub async fn shutdown(&self) { - self.services.runtime_facade.shutdown().await; + let _ = self.shutdown_checked().await; + } + + /// Irreversibly terminates the notification target runtime for this process. + pub async fn shutdown_checked(&self) -> Result<(), NotificationError> { + self.services.config_manager.lifecycle().terminate().wait().await } } diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index be997c017..df93c5a48 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -25,6 +25,7 @@ mod event; pub mod factory; mod global; pub mod integration; +mod lifecycle; mod notification_system_subscriber; pub mod notifier; mod pipeline; @@ -42,10 +43,11 @@ pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem}; pub use error::{LifecycleError, NotificationError}; pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo}; pub use global::{ - initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system, - notification_target_metrics, notifier_global, + ensure_live_events, initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, + notification_system, notification_target_metrics, notifier_global, reconcile, }; pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot}; +pub use lifecycle::{NotificationLifecycleTransition, NotificationRuntimeState}; pub use pipeline::{LiveEventHistory, NotifyEventBridge, NotifyPipeline}; pub use rule_engine::NotifyRuleEngine; pub use rules::BucketNotificationConfig; @@ -53,6 +55,9 @@ pub use runtime_facade::NotifyRuntimeFacade; pub use runtime_view::NotifyRuntimeView; pub use services::NotifyServices; pub use status_view::NotifyStatusView; +pub use storage_api::NotifyStore; pub(crate) use storage_api::crate_boundary::{ - read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config, + read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, + resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, + with_notify_server_config_write_lock, }; diff --git a/crates/notify/src/lifecycle.rs b/crates/notify/src/lifecycle.rs new file mode 100644 index 000000000..b0b37b612 --- /dev/null +++ b/crates/notify/src/lifecycle.rs @@ -0,0 +1,1519 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + Event, NotificationError, config_manager::notify_configuration_hint, error::transition_join_error, registry::TargetRegistry, + runtime_facade::NotifyRuntimeFacade, +}; +use async_trait::async_trait; +use rustfs_config::server_config::Config; +use rustfs_targets::Target; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex}; +use tokio::sync::{Mutex, RwLock}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info}; + +type CurrentGeneration = Arc bool + Send + Sync>; +type CommitGeneration = Box; +type CleanupFuture = Pin> + Send>>; + +const LOG_COMPONENT_NOTIFY: &str = "notify"; +const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; +const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NotificationRuntimeState { + LiveOnly, + TargetsEnabled { generation: u64 }, + Terminated, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DesiredMode { + LiveOnly, + TargetsEnabled, + Terminated, +} + +#[derive(Clone)] +struct DesiredState { + cancellation: CancellationToken, + config: Config, + generation: u64, + mode: DesiredMode, +} + +struct RuntimeApply { + cleanup: Option, +} + +impl RuntimeApply { + fn complete() -> Self { + Self { cleanup: None } + } + + fn with_cleanup(cleanup: CleanupFuture) -> Self { + Self { cleanup: Some(cleanup) } + } + + async fn finish(self) -> Result<(), NotificationError> { + match self.cleanup { + Some(cleanup) => cleanup.await, + None => Ok(()), + } + } +} + +enum TransitionResult { + Ready(Result<(), NotificationError>), + Running(JoinHandle>), +} + +pub struct NotificationLifecycleTransition { + result: TransitionResult, +} + +impl NotificationLifecycleTransition { + fn ready(result: Result<(), NotificationError>) -> Self { + Self { + result: TransitionResult::Ready(result), + } + } + + fn running(handle: JoinHandle>) -> Self { + Self { + result: TransitionResult::Running(handle), + } + } + + pub async fn wait(self) -> Result<(), NotificationError> { + match self.result { + TransitionResult::Ready(result) => result, + TransitionResult::Running(handle) => handle.await.map_err(transition_join_error)?, + } + } +} + +#[async_trait] +trait LifecycleRuntime: Send + Sync { + async fn cache_config(&self, config: Config); + + async fn enable_targets( + &self, + config: &Config, + cancellation: &CancellationToken, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result; + + async fn disable_targets( + &self, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result; +} + +struct DefaultLifecycleRuntime { + config: Arc>, + registry: Arc, + runtime_facade: NotifyRuntimeFacade, + #[cfg(test)] + handoff_observer: Option>, +} + +#[async_trait] +impl LifecycleRuntime for DefaultLifecycleRuntime { + async fn cache_config(&self, config: Config) { + *self.config.write().await = config; + } + + async fn enable_targets( + &self, + config: &Config, + cancellation: &CancellationToken, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result { + info!( + event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + state = "preparing", + "notify runtime lifecycle" + ); + + let (targets, creation_failures): (Vec + Send + Sync>>, Vec) = + self.registry.create_dormant_targets_from_config(config).await?; + if targets.is_empty() { + debug!( + event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + state = "idle", + reason = "no_targets_configured", + hint = %notify_configuration_hint(), + "notify runtime lifecycle" + ); + } + + let mut prepared = self.runtime_facade.prepare_targets(targets, cancellation).await; + prepared.extend_creation_failures(creation_failures); + if !is_current() { + let runtime_facade = self.runtime_facade.clone(); + return Ok(RuntimeApply::with_cleanup(Box::pin(async move { + runtime_facade.close_prepared(prepared).await + }))); + } + + // Drain sends that cloned the old targets, then keep later sends queued + // until queue-store ownership and the target list move atomically. A + // successful current-generation check under publication_gate is the + // non-cancellable commit barrier: later intents serialize after this + // handoff instead of abandoning a workerless old runtime. + let dispatch_guard = self.runtime_facade.pause_dispatch().await; + { + let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner()); + if !is_current() { + drop(_publication_guard); + drop(dispatch_guard); + let runtime_facade = self.runtime_facade.clone(); + return Ok(RuntimeApply::with_cleanup(Box::pin(async move { + runtime_facade.close_prepared(prepared).await + }))); + } + } + + #[cfg(test)] + if let Some(observer) = &self.handoff_observer { + observer(); + } + + self.runtime_facade.stop_active_replay_workers().await; + let (opened, rejected) = self.runtime_facade.open_prepared_stores(prepared).await?; + let open_complete = rejected.failure_summary().is_none(); + // A newer accepted generation remains queued behind apply_lock and + // replaces this one next. + let (detached, activation_rejected) = self + .runtime_facade + .commit_prepared(opened, move |activation_complete| on_committed(open_complete && activation_complete)) + .await; + let activation_failures = [rejected.failure_summary(), activation_rejected.failure_summary()] + .into_iter() + .flatten() + .collect::>(); + drop(dispatch_guard); + let runtime_facade = self.runtime_facade.clone(); + + info!( + event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + state = "enabled", + "notify runtime lifecycle" + ); + Ok(RuntimeApply::with_cleanup(Box::pin(async move { + let (close_old, close_rejected, close_activation_rejected) = tokio::join!( + runtime_facade.close_detached_targets(detached), + runtime_facade.close_prepared(rejected), + runtime_facade.close_prepared(activation_rejected), + ); + close_old?; + close_rejected?; + close_activation_rejected?; + if !activation_failures.is_empty() { + return Err(NotificationError::Initialization(format!( + "one or more notification targets failed to activate: {}", + activation_failures.join("; ") + ))); + } + Ok(()) + }))) + } + + async fn disable_targets( + &self, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result { + let dispatch_guard = self.runtime_facade.pause_dispatch().await; + { + let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner()); + if !is_current() { + drop(_publication_guard); + drop(dispatch_guard); + return Ok(RuntimeApply::complete()); + } + } + + // The generation check above is the non-cancellable publication + // barrier. Later intents serialize behind apply_lock while the old + // replay workers stop, so LiveOnly is not visible until every old + // worker has joined. + self.runtime_facade.stop_active_replay_workers().await; + let detached = self.runtime_facade.commit_disabled(move || on_committed(true)).await; + drop(dispatch_guard); + let runtime_facade = self.runtime_facade.clone(); + Ok(RuntimeApply::with_cleanup(Box::pin(async move { + runtime_facade.close_detached_targets(detached).await + }))) + } +} + +struct LifecycleCoordinatorInner { + applied_generation: AtomicU64, + apply_lock: Mutex<()>, + desired: StdMutex, + publication_gate: Arc>, + runtime: Arc, + state: StdMutex, +} + +#[derive(Clone)] +pub(crate) struct NotifyLifecycleCoordinator { + inner: Arc, +} + +impl NotifyLifecycleCoordinator { + pub(crate) fn new(config: Arc>, registry: Arc, runtime_facade: NotifyRuntimeFacade) -> Self { + let runtime = Arc::new(DefaultLifecycleRuntime { + config, + registry, + runtime_facade, + #[cfg(test)] + handoff_observer: None, + }); + Self::with_runtime(Config::new(), runtime) + } + + fn with_runtime(initial_config: Config, runtime: Arc) -> Self { + Self { + inner: Arc::new(LifecycleCoordinatorInner { + applied_generation: AtomicU64::new(0), + apply_lock: Mutex::new(()), + desired: StdMutex::new(DesiredState { + cancellation: CancellationToken::new(), + config: initial_config, + generation: 0, + mode: DesiredMode::LiveOnly, + }), + publication_gate: Arc::new(StdMutex::new(())), + runtime, + state: StdMutex::new(NotificationRuntimeState::LiveOnly), + }), + } + } + + pub(crate) fn set_mode(&self, enabled: bool, config: Option) -> NotificationLifecycleTransition { + self.accept(false, move |desired| { + desired.mode = if enabled { + DesiredMode::TargetsEnabled + } else { + DesiredMode::LiveOnly + }; + if let Some(config) = config { + desired.config = config; + } + Ok(true) + }) + } + + pub(crate) fn update_config(&self, config: Config) -> NotificationLifecycleTransition { + self.accept(false, move |desired| { + desired.config = config; + Ok(true) + }) + } + + pub(crate) fn terminate(&self) -> NotificationLifecycleTransition { + self.accept(true, |desired| { + if desired.mode == DesiredMode::Terminated { + return Ok(false); + } + desired.mode = DesiredMode::Terminated; + Ok(true) + }) + } + + pub(crate) fn state(&self) -> NotificationRuntimeState { + *self.inner.state.lock().unwrap_or_else(|err| err.into_inner()) + } + + pub(crate) fn is_converged(&self) -> bool { + let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation; + self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation + } + + fn accept(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition + where + F: FnOnce(&mut DesiredState) -> Result, + { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization( + "Notification lifecycle coordinator is unavailable".to_string(), + ))); + }; + let _publication_guard = self.inner.publication_gate.lock().unwrap_or_else(|err| err.into_inner()); + let mut desired = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()); + if desired.mode == DesiredMode::Terminated && !allow_terminated { + return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization( + "Notification runtime has terminated".to_string(), + ))); + } + + let previous = desired.clone(); + let changed = match update(&mut desired) { + Ok(changed) => changed, + Err(err) => return NotificationLifecycleTransition::ready(Err(err)), + }; + if changed { + let Some(next_generation) = desired.generation.checked_add(1) else { + *desired = previous; + return NotificationLifecycleTransition::ready(Err(NotificationError::Initialization( + "Notification lifecycle generation is exhausted".to_string(), + ))); + }; + desired.generation = next_generation; + previous.cancellation.cancel(); + desired.cancellation = CancellationToken::new(); + } + let accepted = desired.clone(); + drop(desired); + drop(_publication_guard); + let inner = Arc::clone(&self.inner); + NotificationLifecycleTransition::running(runtime.spawn(async move { inner.converge(accepted).await })) + } +} + +impl LifecycleCoordinatorInner { + async fn converge(self: &Arc, desired: DesiredState) -> Result<(), NotificationError> { + let _apply_guard = self.apply_lock.lock().await; + if self.applied_generation.load(Ordering::Acquire) >= desired.generation { + return Ok(()); + } + if !self.current_matches(desired.generation, desired.mode) { + return Ok(()); + } + + self.runtime.cache_config(desired.config.clone()).await; + let current_inner = Arc::clone(self); + let generation = desired.generation; + let mode = desired.mode; + let is_current: CurrentGeneration = Arc::new(move || current_inner.current_matches(generation, mode)); + let publication_gate = self.publication_gate.clone(); + + let state = match desired.mode { + DesiredMode::LiveOnly => NotificationRuntimeState::LiveOnly, + DesiredMode::TargetsEnabled => NotificationRuntimeState::TargetsEnabled { + generation: desired.generation, + }, + DesiredMode::Terminated => NotificationRuntimeState::Terminated, + }; + let committed_inner = Arc::clone(self); + let on_committed: CommitGeneration = + Box::new(move |complete| committed_inner.mark_published(generation, state, complete)); + + let applied = match desired.mode { + DesiredMode::LiveOnly | DesiredMode::Terminated => { + self.runtime + .disable_targets(publication_gate, is_current, on_committed) + .await? + } + DesiredMode::TargetsEnabled => { + self.runtime + .enable_targets(&desired.config, &desired.cancellation, publication_gate, is_current, on_committed) + .await? + } + }; + drop(_apply_guard); + applied.finish().await + } + + fn current_matches(&self, generation: u64, mode: DesiredMode) -> bool { + let desired = self.desired.lock().unwrap_or_else(|err| err.into_inner()); + desired.generation == generation && desired.mode == mode + } + + fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) { + *self.state.lock().unwrap_or_else(|err| err.into_inner()) = state; + if complete { + self.applied_generation.store(generation, Ordering::Release); + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + CommitGeneration, CurrentGeneration, DefaultLifecycleRuntime, LifecycleRuntime, NotificationRuntimeState, + NotifyLifecycleCoordinator, RuntimeApply, + }; + use crate::{ + Event, NotificationError, integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, + rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, runtime_view::NotifyRuntimeView, + }; + use async_trait::async_trait; + use rustfs_config::ENABLE_KEY; + use rustfs_config::server_config::{Config, KVS}; + use rustfs_targets::arn::TargetID; + use rustfs_targets::store::{Key, QueueStore, Store}; + use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use rustfs_targets::{ + EventName, ReplayWorkerManager, StoreError, Target, TargetError, TargetPluginDescriptor, TargetPluginRegistry, + }; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex as StdMutex}; + use tokio::sync::{Notify, RwLock, Semaphore, mpsc}; + use tokio_util::sync::CancellationToken; + + const BLOCKING_INIT_TARGET_TYPE: &str = "lifecycle_blocking_init"; + const RETRY_INIT_TARGET_TYPE: &str = "lifecycle_retry_init"; + const REPLAY_TARGET_TYPE: &str = "lifecycle_replay"; + + struct BlockingInitState { + close_calls: AtomicUsize, + init_entered: Notify, + } + + #[derive(Clone)] + struct BlockingInitTarget { + id: TargetID, + state: Arc, + } + + #[async_trait] + impl Target for BlockingInitTarget { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.state.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + self.state.init_entered.notify_one(); + std::future::pending::<()>().await; + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + #[derive(Clone)] + struct RetryInitTarget { + fail_once: Arc, + id: TargetID, + should_fail: bool, + } + + #[async_trait] + impl Target for RetryInitTarget { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + if self.should_fail && self.fail_once.swap(false, Ordering::AcqRel) { + return Err(TargetError::Initialization("forced transient init failure".to_string())); + } + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + struct ReplayState { + active_workers: AtomicUsize, + completed_deliveries: AtomicUsize, + created_targets: AtomicUsize, + delivery_attempts: AtomicUsize, + delivery_release: Notify, + delivered_bodies: StdMutex>>, + blocked_init_generation: usize, + blocked_init_release: Notify, + init_tx: mpsc::UnboundedSender, + list_tx: mpsc::UnboundedSender, + peak_workers: AtomicUsize, + send_tx: mpsc::UnboundedSender, + } + + struct ReplayWorkerLease { + state: Arc, + } + + impl ReplayWorkerLease { + fn acquire(state: Arc) -> Self { + let active = state.active_workers.fetch_add(1, Ordering::SeqCst) + 1; + let mut peak = state.peak_workers.load(Ordering::SeqCst); + while active > peak { + match state + .peak_workers + .compare_exchange_weak(peak, active, Ordering::SeqCst, Ordering::SeqCst) + { + Ok(_) => break, + Err(observed) => peak = observed, + } + } + Self { state } + } + } + + impl Drop for ReplayWorkerLease { + fn drop(&mut self) { + self.state.active_workers.fetch_sub(1, Ordering::SeqCst); + } + } + + #[derive(Clone)] + struct ObservedQueueStore { + generation: usize, + inner: QueueStore, + state: Arc, + _worker_lease: Option>, + } + + impl Store for ObservedQueueStore { + type Error = StoreError; + type Key = Key; + + fn open(&self) -> Result<(), Self::Error> { + self.inner.open() + } + + fn put(&self, item: Arc) -> Result { + self.inner.put(item) + } + + fn put_multiple(&self, items: Vec) -> Result { + self.inner.put_multiple(items) + } + + fn put_raw(&self, data: &[u8]) -> Result { + self.inner.put_raw(data) + } + + fn get(&self, key: &Self::Key) -> Result { + self.inner.get(key) + } + + fn get_multiple(&self, key: &Self::Key) -> Result, Self::Error> { + self.inner.get_multiple(key) + } + + fn get_raw(&self, key: &Self::Key) -> Result, Self::Error> { + self.inner.get_raw(key) + } + + fn del(&self, key: &Self::Key) -> Result<(), Self::Error> { + self.inner.del(key) + } + + fn delete(&self) -> Result<(), Self::Error> { + self.inner.delete() + } + + fn list(&self) -> Vec { + let _ = self.state.list_tx.send(self.generation); + self.inner.list() + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + fn boxed_clone(&self) -> Box + Send + Sync> { + Box::new(Self { + generation: self.generation, + inner: self.inner.clone(), + state: self.state.clone(), + _worker_lease: Some(Arc::new(ReplayWorkerLease::acquire(self.state.clone()))), + }) + } + } + + #[derive(Clone)] + struct ReplayTarget { + id: TargetID, + generation: usize, + state: Arc, + store: ObservedQueueStore, + } + + #[async_trait] + impl Target for ReplayTarget { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + let attempt = self.state.delivery_attempts.fetch_add(1, Ordering::SeqCst) + 1; + let _ = self.state.send_tx.send(attempt); + self.state.delivery_release.notified().await; + self.state + .delivered_bodies + .lock() + .unwrap_or_else(|err| err.into_inner()) + .push(body); + self.state.completed_deliveries.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + Some(&self.store) + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + let _ = self.state.init_tx.send(self.generation); + if self.generation == self.state.blocked_init_generation { + self.state.blocked_init_release.notified().await; + } + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + struct TestRuntime { + block_cleanup: AtomicBool, + block_enable: AtomicBool, + cached_config: StdMutex, + cleanup_entered: Arc, + cleanup_release: Arc, + disable_calls: AtomicUsize, + enable_calls: AtomicUsize, + enable_entered: Notify, + fail_enable: AtomicBool, + enable_release: Notify, + enabled_configs: StdMutex>, + } + + impl TestRuntime { + fn new(initial_config: Config) -> Self { + Self { + block_cleanup: AtomicBool::new(false), + block_enable: AtomicBool::new(false), + cached_config: StdMutex::new(initial_config), + cleanup_entered: Arc::new(Notify::new()), + cleanup_release: Arc::new(Notify::new()), + disable_calls: AtomicUsize::new(0), + enable_calls: AtomicUsize::new(0), + enable_entered: Notify::new(), + fail_enable: AtomicBool::new(false), + enable_release: Notify::new(), + enabled_configs: StdMutex::new(Vec::new()), + } + } + + fn block_next_enable(&self) { + self.block_enable.store(true, Ordering::Release); + } + + fn block_next_cleanup(&self) { + self.block_cleanup.store(true, Ordering::Release); + } + + fn fail_next_enable(&self) { + self.fail_enable.store(true, Ordering::Release); + } + + fn last_enabled_config(&self) -> Option { + self.enabled_configs + .lock() + .unwrap_or_else(|err| err.into_inner()) + .last() + .cloned() + } + } + + #[async_trait] + impl LifecycleRuntime for TestRuntime { + async fn cache_config(&self, config: Config) { + *self.cached_config.lock().unwrap_or_else(|err| err.into_inner()) = config; + } + + async fn enable_targets( + &self, + config: &Config, + _cancellation: &CancellationToken, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result { + self.enable_calls.fetch_add(1, Ordering::SeqCst); + if self.block_enable.swap(false, Ordering::AcqRel) { + self.enable_entered.notify_one(); + self.enable_release.notified().await; + } + if self.fail_enable.swap(false, Ordering::AcqRel) { + return Err(NotificationError::Initialization("forced enable failure".to_string())); + } + let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner()); + if !is_current() { + return Ok(RuntimeApply::complete()); + } + self.enabled_configs + .lock() + .unwrap_or_else(|err| err.into_inner()) + .push(config.clone()); + on_committed(true); + if self.block_cleanup.swap(false, Ordering::AcqRel) { + let cleanup_entered = self.cleanup_entered.clone(); + let cleanup_release = self.cleanup_release.clone(); + return Ok(RuntimeApply::with_cleanup(Box::pin(async move { + cleanup_entered.notify_one(); + cleanup_release.notified().await; + Ok(()) + }))); + } + Ok(RuntimeApply::complete()) + } + + async fn disable_targets( + &self, + publication_gate: Arc>, + is_current: CurrentGeneration, + on_committed: CommitGeneration, + ) -> Result { + let _publication_guard = publication_gate.lock().unwrap_or_else(|err| err.into_inner()); + if !is_current() { + return Ok(RuntimeApply::complete()); + } + self.disable_calls.fetch_add(1, Ordering::SeqCst); + on_committed(true); + Ok(RuntimeApply::complete()) + } + } + + fn config_with_target(name: &str) -> Config { + let mut config = Config::new(); + config + .0 + .entry("notify_webhook".to_string()) + .or_default() + .insert(name.to_string(), KVS::default()); + config + } + + fn config_with_enabled_test_target(target_type: &str, name: &str) -> Config { + let mut target = KVS::new(); + target.insert(ENABLE_KEY.to_string(), "on".to_string()); + let mut config = Config::new(); + config + .0 + .entry(format!("notify_{target_type}")) + .or_default() + .insert(name.to_string(), target); + config + } + + fn real_runtime( + initial_config: Config, + registry: Arc, + handoff_observer: Option>, + ) -> (NotifyLifecycleCoordinator, NotifyRuntimeView, Arc>) { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine)); + let target_list = notifier.target_list(); + let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new())); + let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone()); + let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate( + target_list, + replay_workers.clone(), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(4)), + metrics, + ); + let config = Arc::new(RwLock::new(initial_config.clone())); + let runtime = Arc::new(DefaultLifecycleRuntime { + config, + registry, + runtime_facade, + handoff_observer, + }); + ( + NotifyLifecycleCoordinator::with_runtime(initial_config, runtime), + runtime_view, + replay_workers, + ) + } + + fn blocking_init_runtime() -> ( + Arc, + NotifyLifecycleCoordinator, + NotifyRuntimeView, + Arc, + Config, + ) { + let state = Arc::new(BlockingInitState { + close_calls: AtomicUsize::new(0), + init_entered: Notify::new(), + }); + let mut plugins = TargetPluginRegistry::::new(); + let factory_state = state.clone(); + plugins.register(TargetPluginDescriptor::new( + BLOCKING_INIT_TARGET_TYPE, + &[ENABLE_KEY], + |_config| Ok(()), + move |id, _config| { + Ok(Box::new(BlockingInitTarget { + id: TargetID::new(id, BLOCKING_INIT_TARGET_TYPE.to_string()), + state: factory_state.clone(), + })) + }, + )); + let config = config_with_enabled_test_target(BLOCKING_INIT_TARGET_TYPE, "primary"); + let handoff_count = Arc::new(AtomicUsize::new(0)); + let observer_count = handoff_count.clone(); + let (coordinator, runtime_view, _) = real_runtime( + config.clone(), + Arc::new(TargetRegistry::with_plugins(plugins)), + Some(Arc::new(move || { + observer_count.fetch_add(1, Ordering::SeqCst); + })), + ); + (state, coordinator, runtime_view, handoff_count, config) + } + + async fn recv_until_generation(receiver: &mut mpsc::UnboundedReceiver, expected: usize, context: &str) { + loop { + let generation = receiver.recv().await.expect(context); + if generation == expected { + return; + } + } + } + + fn coordinator(runtime: Arc, initial_config: Config) -> NotifyLifecycleCoordinator { + NotifyLifecycleCoordinator::with_runtime(initial_config, runtime) + } + + #[tokio::test] + async fn slow_enable_then_disable_converges_to_live_only() { + let initial = config_with_target("initial"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + runtime.block_next_enable(); + let coordinator = coordinator(runtime.clone(), initial); + + let enable = coordinator.set_mode(true, None); + runtime.enable_entered.notified().await; + let disable = coordinator.set_mode(false, None); + runtime.enable_release.notify_one(); + + enable.wait().await.expect("enable transition should finish"); + disable.wait().await.expect("disable transition should finish"); + + assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1); + assert_eq!(runtime.disable_calls.load(Ordering::SeqCst), 1); + assert_eq!(runtime.last_enabled_config(), None, "superseded enable must not publish targets"); + } + + #[tokio::test] + async fn real_runtime_disable_supersedes_target_init_and_leaves_no_runtime_state() { + let (state, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime(); + + let enable = coordinator.set_mode(true, Some(config)); + state.init_entered.notified().await; + let disable = coordinator.set_mode(false, None); + + enable.wait().await.expect("superseded real enable should finish"); + disable.wait().await.expect("real disable should converge"); + + let status = runtime_view.runtime_status_snapshot().await; + assert_eq!(status.target_count, 0); + assert_eq!(status.replay_worker_count, 0); + assert_eq!(state.close_calls.load(Ordering::SeqCst), 1); + assert_eq!(handoff_count.load(Ordering::SeqCst), 0); + assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly); + } + + #[tokio::test] + async fn real_runtime_disable_publishes_live_only_after_replay_join() { + let (coordinator, _, replay_workers) = real_runtime(Config::new(), Arc::new(TargetRegistry::new()), None); + coordinator + .set_mode(true, Some(Config::new())) + .wait() + .await + .expect("empty target runtime should enable"); + + let (cancel_tx, mut cancel_rx) = mpsc::channel(1); + let stop_entered = Arc::new(Notify::new()); + let stop_release = Arc::new(Notify::new()); + let worker_entered = stop_entered.clone(); + let worker_release = stop_release.clone(); + let join = tokio::spawn(async move { + cancel_rx.recv().await.expect("worker should receive cancellation"); + worker_entered.notify_one(); + worker_release.notified().await; + }); + replay_workers + .write() + .await + .insert_with_handle("blocking-replay".to_string(), cancel_tx, join); + + let disable = coordinator.set_mode(false, None); + stop_entered.notified().await; + assert_eq!( + coordinator.state(), + NotificationRuntimeState::TargetsEnabled { generation: 1 }, + "LiveOnly must not publish while an old replay worker is still running" + ); + + stop_release.notify_one(); + disable.wait().await.expect("disable should finish after replay joins"); + assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly); + } + + #[tokio::test] + async fn real_runtime_terminate_cancels_target_init_and_is_final() { + let (state, coordinator, runtime_view, handoff_count, config) = blocking_init_runtime(); + + let enable = coordinator.set_mode(true, Some(config)); + state.init_entered.notified().await; + let terminate = coordinator.terminate(); + + enable.wait().await.expect("superseded real enable should finish"); + terminate.wait().await.expect("real terminate should converge"); + + let status = runtime_view.runtime_status_snapshot().await; + assert_eq!(status.target_count, 0); + assert_eq!(status.replay_worker_count, 0); + assert_eq!(state.close_calls.load(Ordering::SeqCst), 1); + assert_eq!(handoff_count.load(Ordering::SeqCst), 0); + assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated); + let err = coordinator + .set_mode(false, None) + .wait() + .await + .expect_err("terminated runtime cannot be restarted"); + assert!(matches!( + err, + NotificationError::Initialization(detail) if detail == "Notification runtime has terminated" + )); + } + + #[tokio::test] + async fn partial_activation_reports_error_and_same_config_can_retry() { + let fail_once = Arc::new(AtomicBool::new(true)); + let mut plugins = TargetPluginRegistry::::new(); + let factory_fail_once = fail_once.clone(); + plugins.register(TargetPluginDescriptor::new( + RETRY_INIT_TARGET_TYPE, + &[ENABLE_KEY], + |_config| Ok(()), + move |id, _config| { + Ok(Box::new(RetryInitTarget { + should_fail: id == "bad", + id: TargetID::new(id, RETRY_INIT_TARGET_TYPE.to_string()), + fail_once: factory_fail_once.clone(), + })) + }, + )); + let mut config = config_with_enabled_test_target(RETRY_INIT_TARGET_TYPE, "good"); + let mut bad = KVS::new(); + bad.insert(ENABLE_KEY.to_string(), "on".to_string()); + config + .0 + .get_mut(&format!("notify_{RETRY_INIT_TARGET_TYPE}")) + .expect("test target subsystem should exist") + .insert("bad".to_string(), bad); + let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None); + + coordinator + .set_mode(true, Some(config.clone())) + .wait() + .await + .expect_err("the originating transition must report the failed target"); + let partial = runtime_view.runtime_status_snapshot().await; + assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active"); + assert!(!coordinator.is_converged(), "a partial generation must remain retryable"); + assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 }); + + coordinator + .update_config(config) + .wait() + .await + .expect("the same persisted config should recover after the transient failure"); + let recovered = runtime_view.runtime_status_snapshot().await; + assert_eq!(recovered.target_count, 2); + assert!(coordinator.is_converged()); + assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 }); + + coordinator + .set_mode(false, None) + .wait() + .await + .expect("test runtime should shut down"); + } + + #[tokio::test(start_paused = true)] + async fn real_runtime_reload_never_overlaps_replay_workers_or_redelivers_queued_payload() { + let queue_root = tempfile::tempdir().expect("queue root should be created"); + let queue_dir = queue_root.path().join("shared-queue"); + let body = br#"{"x":1}"#.to_vec(); + let seed_store = QueueStore::::new_with_compression(&queue_dir, 16, ".event", false); + seed_store.open().expect("seed queue should open"); + let meta = QueuedPayloadMeta::new( + EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "obj.txt".to_string(), + "application/json", + body.len(), + ); + let payload = QueuedPayload::new(meta, body.clone()); + seed_store + .put_raw(&payload.encode().expect("queued payload should encode")) + .expect("queued payload should be seeded"); + assert_eq!(seed_store.len(), 1); + drop(seed_store); + + let (init_tx, mut init_rx) = mpsc::unbounded_channel(); + let (list_tx, mut list_rx) = mpsc::unbounded_channel(); + let (send_tx, mut send_rx) = mpsc::unbounded_channel(); + let state = Arc::new(ReplayState { + active_workers: AtomicUsize::new(0), + blocked_init_generation: 3, + blocked_init_release: Notify::new(), + completed_deliveries: AtomicUsize::new(0), + created_targets: AtomicUsize::new(0), + delivery_attempts: AtomicUsize::new(0), + delivery_release: Notify::new(), + delivered_bodies: StdMutex::new(Vec::new()), + init_tx, + list_tx, + peak_workers: AtomicUsize::new(0), + send_tx, + }); + let mut plugins = TargetPluginRegistry::::new(); + let factory_state = state.clone(); + let factory_queue_dir = queue_dir.clone(); + plugins.register(TargetPluginDescriptor::new( + REPLAY_TARGET_TYPE, + &[ENABLE_KEY], + |_config| Ok(()), + move |id, _config| { + let generation = factory_state.created_targets.fetch_add(1, Ordering::SeqCst) + 1; + Ok(Box::new(ReplayTarget { + id: TargetID::new(id, REPLAY_TARGET_TYPE.to_string()), + generation, + state: factory_state.clone(), + store: ObservedQueueStore { + generation, + inner: QueueStore::new_with_compression(&factory_queue_dir, 16, ".event", false), + state: factory_state.clone(), + _worker_lease: None, + }, + })) + }, + )); + + let (handoff_tx, mut handoff_rx) = mpsc::unbounded_channel(); + let handoff_count = Arc::new(AtomicUsize::new(0)); + let observer_count = handoff_count.clone(); + let handoff_observer: Arc = Arc::new(move || { + let generation = observer_count.fetch_add(1, Ordering::SeqCst) + 1; + let _ = handoff_tx.send(generation); + }); + let config = config_with_enabled_test_target(REPLAY_TARGET_TYPE, "primary"); + let (coordinator, runtime_view, _) = + real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), Some(handoff_observer)); + + coordinator + .set_mode(true, Some(config.clone())) + .wait() + .await + .expect("initial real runtime enable should succeed"); + assert_eq!(init_rx.recv().await.expect("first target init should be observed"), 1); + assert_eq!(handoff_rx.recv().await.expect("first handoff should be observed"), 1); + recv_until_generation(&mut list_rx, 1, "first replay worker should scan the queue").await; + + let status = runtime_view.runtime_status_snapshot().await; + assert_eq!(status.target_count, 1); + assert_eq!(status.replay_worker_count, 1); + assert_eq!(state.active_workers.load(Ordering::SeqCst), 1); + assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1); + + tokio::time::advance(std::time::Duration::from_secs(6)).await; + assert_eq!(send_rx.recv().await.expect("seeded payload delivery should start"), 1); + + let second_generation = coordinator.update_config(config.clone()); + assert_eq!(init_rx.recv().await.expect("replacement target init should be observed"), 2); + assert_eq!(handoff_rx.recv().await.expect("replacement handoff should be observed"), 2); + + let status_during_handoff = runtime_view.runtime_status_snapshot().await; + assert_eq!(status_during_handoff.target_count, 1); + assert_eq!(status_during_handoff.replay_worker_count, 0); + assert_eq!(state.active_workers.load(Ordering::SeqCst), 1); + assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1); + + let third_generation = coordinator.update_config(config); + state.delivery_release.notify_one(); + second_generation + .wait() + .await + .expect("second generation should commit after crossing the handoff barrier"); + assert_eq!(init_rx.recv().await.expect("third target init should be observed"), 3); + assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 }); + recv_until_generation(&mut list_rx, 2, "second-generation replay worker should scan the queue").await; + + let status_after_second_generation = runtime_view.runtime_status_snapshot().await; + assert_eq!(status_after_second_generation.target_count, 1); + assert_eq!(status_after_second_generation.replay_worker_count, 1); + assert_eq!(state.active_workers.load(Ordering::SeqCst), 1); + assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1); + + state.blocked_init_release.notify_one(); + assert_eq!(handoff_rx.recv().await.expect("third handoff should be observed"), 3); + third_generation + .wait() + .await + .expect("third generation should converge after the committed second generation"); + recv_until_generation(&mut list_rx, 3, "third-generation replay worker should scan the queue").await; + + assert_eq!(state.delivery_attempts.load(Ordering::SeqCst), 1); + assert_eq!(state.completed_deliveries.load(Ordering::SeqCst), 1); + assert_eq!(state.peak_workers.load(Ordering::SeqCst), 1); + assert_eq!(state.active_workers.load(Ordering::SeqCst), 1); + assert_eq!(*state.delivered_bodies.lock().unwrap_or_else(|err| err.into_inner()), vec![body]); + let status_after_reload = runtime_view.runtime_status_snapshot().await; + assert_eq!(status_after_reload.target_count, 1); + assert_eq!(status_after_reload.replay_worker_count, 1); + assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 3 }); + + tokio::time::advance(std::time::Duration::from_secs(6)).await; + recv_until_generation(&mut list_rx, 3, "third-generation replay worker should rescan after the batch timeout").await; + assert!(matches!(send_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + coordinator + .set_mode(false, None) + .wait() + .await + .expect("real runtime disable should stop the replacement worker"); + let stopped = runtime_view.runtime_status_snapshot().await; + assert_eq!(stopped.target_count, 0); + assert_eq!(stopped.replay_worker_count, 0); + assert_eq!(state.active_workers.load(Ordering::SeqCst), 0); + + let verify_store = QueueStore::::new_with_compression(&queue_dir, 16, ".event", false); + verify_store.open().expect("queue should reopen after runtime shutdown"); + assert_eq!(verify_store.len(), 0); + } + + #[tokio::test] + async fn later_config_generation_wins_after_slow_reload() { + let first = config_with_target("first"); + let second = config_with_target("second"); + let runtime = Arc::new(TestRuntime::new(first.clone())); + runtime.block_next_enable(); + let coordinator = coordinator(runtime.clone(), first.clone()); + + let enable = coordinator.set_mode(true, Some(first)); + runtime.enable_entered.notified().await; + let reload = coordinator.update_config(second.clone()); + runtime.enable_release.notify_one(); + + enable.wait().await.expect("first generation should finish"); + reload.wait().await.expect("latest generation should finish"); + + assert_eq!(runtime.last_enabled_config(), Some(second)); + assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 }); + } + + #[tokio::test] + async fn accepted_transition_survives_caller_cancellation() { + let initial = config_with_target("initial"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + runtime.block_next_enable(); + let coordinator = coordinator(runtime.clone(), initial); + + let receipt = coordinator.set_mode(true, None); + runtime.enable_entered.notified().await; + drop(receipt); + runtime.enable_release.notify_one(); + + let apply_guard = coordinator.inner.apply_lock.lock().await; + drop(apply_guard); + assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 })); + } + + #[tokio::test] + async fn disabled_config_update_is_cached_until_reenabled() { + let initial = config_with_target("initial"); + let latest = config_with_target("latest"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime.clone(), initial); + + coordinator + .update_config(latest.clone()) + .wait() + .await + .expect("disabled update should be cached"); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 0); + assert_eq!(coordinator.state(), NotificationRuntimeState::LiveOnly); + + coordinator + .set_mode(true, None) + .wait() + .await + .expect("re-enable should use cached config"); + assert_eq!(runtime.last_enabled_config(), Some(latest)); + } + + #[tokio::test] + async fn concurrent_enable_publications_activate_once() { + let initial = config_with_target("initial"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime.clone(), initial); + + let apply_guard = coordinator.inner.apply_lock.lock().await; + let first = coordinator.set_mode(true, None); + let second = coordinator.set_mode(true, None); + drop(apply_guard); + + first.wait().await.expect("first enable should converge"); + second.wait().await.expect("second enable should converge"); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1); + assert!(matches!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 })); + } + + #[tokio::test] + async fn committed_generation_cleanup_does_not_block_a_newer_generation() { + let initial = config_with_target("initial"); + let latest = config_with_target("latest"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + runtime.block_next_cleanup(); + let coordinator = coordinator(runtime.clone(), initial); + + let first = coordinator.set_mode(true, None); + runtime.cleanup_entered.notified().await; + + coordinator + .update_config(latest.clone()) + .wait() + .await + .expect("newer generation should not wait for old cleanup"); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2); + assert_eq!(runtime.last_enabled_config(), Some(latest)); + + runtime.cleanup_release.notify_one(); + first.wait().await.expect("old generation cleanup should finish"); + } + + #[tokio::test] + async fn superseded_receipt_does_not_retry_a_failed_latest_generation() { + let initial = config_with_target("initial"); + let latest = config_with_target("latest"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime.clone(), initial); + + let apply_guard = coordinator.inner.apply_lock.lock().await; + let superseded = coordinator.set_mode(true, None); + tokio::task::yield_now().await; + let latest_transition = coordinator.update_config(latest.clone()); + tokio::task::yield_now().await; + runtime.fail_next_enable(); + drop(apply_guard); + + superseded + .wait() + .await + .expect("a superseded generation should not apply the latest intent"); + latest_transition + .wait() + .await + .expect_err("the latest generation must own its apply error"); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 1); + assert!(!coordinator.is_converged()); + + coordinator + .update_config(latest.clone()) + .wait() + .await + .expect("an explicit publication should retry the failed configuration"); + assert_eq!(runtime.enable_calls.load(Ordering::SeqCst), 2); + assert_eq!(runtime.last_enabled_config(), Some(latest)); + } + + #[tokio::test] + async fn failed_generation_can_retry_the_same_config() { + let initial = config_with_target("initial"); + let latest = config_with_target("latest"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime.clone(), initial); + + coordinator + .set_mode(true, None) + .wait() + .await + .expect("initial enable should succeed"); + runtime.fail_next_enable(); + coordinator + .update_config(latest.clone()) + .wait() + .await + .expect_err("forced reload should fail"); + assert!(!coordinator.is_converged()); + + coordinator + .update_config(latest.clone()) + .wait() + .await + .expect("same config should be retryable"); + assert!(coordinator.is_converged()); + assert_eq!(runtime.last_enabled_config(), Some(latest)); + } + + #[tokio::test] + async fn suspend_is_restartable_but_terminate_is_final() { + let initial = config_with_target("initial"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime, initial); + + coordinator.set_mode(true, None).wait().await.expect("enable should succeed"); + coordinator + .set_mode(false, None) + .wait() + .await + .expect("suspend should succeed"); + coordinator + .set_mode(true, None) + .wait() + .await + .expect("suspended runtime should restart"); + coordinator.terminate().wait().await.expect("terminate should succeed"); + coordinator.terminate().wait().await.expect("terminate should be idempotent"); + + let err = coordinator + .set_mode(true, None) + .wait() + .await + .expect_err("terminated runtime must reject restart"); + assert!(matches!( + err, + NotificationError::Initialization(detail) if detail == "Notification runtime has terminated" + )); + assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated); + } + + #[tokio::test] + async fn terminate_supersedes_blocked_enable_before_commit() { + let initial = config_with_target("initial"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + runtime.block_next_enable(); + let coordinator = coordinator(runtime.clone(), initial); + + let enable = coordinator.set_mode(true, None); + runtime.enable_entered.notified().await; + let terminate = coordinator.terminate(); + runtime.enable_release.notify_one(); + + enable.wait().await.expect("superseded enable task should finish"); + terminate.wait().await.expect("terminate should finish"); + assert_eq!(runtime.last_enabled_config(), None); + assert_eq!(coordinator.state(), NotificationRuntimeState::Terminated); + } + + #[test] + fn publication_without_tokio_runtime_does_not_mutate_desired_state() { + let initial = config_with_target("initial"); + let latest = config_with_target("latest"); + let runtime = Arc::new(TestRuntime::new(initial.clone())); + let coordinator = coordinator(runtime, initial.clone()); + + let _transition = coordinator.update_config(latest); + let desired = coordinator.inner.desired.lock().unwrap_or_else(|err| err.into_inner()); + assert_eq!(desired.generation, 0); + assert_eq!(desired.config, initial); + } +} diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index 2fce24308..7b1aaf237 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -12,13 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine}; +use crate::error::NotificationError; +use crate::{event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine}; use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY}; +use rustfs_targets::Target; use rustfs_targets::arn::TargetID; use rustfs_targets::target::EntityTarget; -use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager}; -use std::sync::Arc; -use tokio::sync::{RwLock, Semaphore}; +use rustfs_targets::{SharedTarget, TargetRuntimeManager}; +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, Mutex as StdMutex, Weak}; +use tokio::sync::{RwLock, Semaphore, watch}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; const LOG_COMPONENT_NOTIFY: &str = "notify"; @@ -29,8 +34,101 @@ const EVENT_NOTIFY_DISPATCH_STARTED: &str = "notify_dispatch_started"; const EVENT_NOTIFY_DISPATCH_COMPLETED: &str = "notify_dispatch_completed"; const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle"; +async fn wait_for_dispatch_tasks(handles: Vec>) { + for handle in handles { + if let Err(e) = handle.await { + let reason = if e.is_cancelled() { "join_cancelled" } else { "join_panicked" }; + error!( + event = EVENT_NOTIFY_DISPATCH_FAILED, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_DISPATCH, + reason, + "Notify dispatch task failed" + ); + } + } +} + pub type SharedNotifyTargetList = Arc>; +static TARGET_LIST_DISPATCH_GATES: LazyLock>>>> = + LazyLock::new(|| StdMutex::new(HashMap::new())); + +pub(crate) fn shared_dispatch_gate(target_list: &SharedNotifyTargetList, preferred: Option>>) -> Arc> { + let key = Arc::as_ptr(target_list) as usize; + let mut gates = TARGET_LIST_DISPATCH_GATES.lock().unwrap_or_else(|err| err.into_inner()); + gates.retain(|_, gate| gate.strong_count() != 0); + if let Some(gate) = gates.get(&key).and_then(Weak::upgrade) { + return gate; + } + let gate = preferred.unwrap_or_else(|| Arc::new(RwLock::new(()))); + gates.insert(key, Arc::downgrade(&gate)); + gate +} + +pub(crate) struct DirectDispatchTracker { + cancellation: CancellationToken, + inflight: watch::Sender, +} + +impl DirectDispatchTracker { + fn new() -> Self { + let (inflight, _) = watch::channel(0); + Self { + cancellation: CancellationToken::new(), + inflight, + } + } + + fn acquire(self: &Arc) -> DirectDispatchLease { + self.inflight.send_modify(|count| *count += 1); + DirectDispatchLease { + tracker: Arc::clone(self), + } + } + + pub(crate) async fn wait_idle(&self) { + let mut inflight = self.inflight.subscribe(); + while *inflight.borrow_and_update() != 0 { + if inflight.changed().await.is_err() { + return; + } + } + } + + fn cancel_pending(&self) { + self.cancellation.cancel(); + } +} + +struct DirectDispatchLease { + tracker: Arc, +} + +impl DirectDispatchLease { + fn cancellation(&self) -> CancellationToken { + self.tracker.cancellation.clone() + } +} + +impl Drop for DirectDispatchLease { + fn drop(&mut self) { + self.tracker.inflight.send_modify(|count| { + if let Some(next) = count.checked_sub(1) { + *count = next; + } else { + error!( + event = EVENT_NOTIFY_DISPATCH_FAILED, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_DISPATCH, + reason = "direct_lease_underflow", + "Notify direct dispatch lease accounting underflowed" + ); + } + }); + } +} + /// Resolves the effective send concurrency (semaphore permit count). /// /// A value of `0` would build a zero-permit semaphore, so `acquire` never @@ -63,6 +161,8 @@ fn coerce_send_concurrency(configured: usize) -> usize { /// Manages event notification to targets based on rules pub struct EventNotifier { + dispatch_gate: Arc>, + enqueue_limiter: Arc, metrics: Arc, rule_engine: NotifyRuleEngine, target_list: SharedNotifyTargetList, @@ -82,10 +182,14 @@ impl EventNotifier { /// Returns a new instance of EventNotifier. pub fn new(metrics: Arc, rule_engine: NotifyRuleEngine) -> Self { let max_inflight = resolve_send_concurrency(); + let target_list = Arc::new(RwLock::new(TargetList::new())); + let dispatch_gate = shared_dispatch_gate(&target_list, None); EventNotifier { + dispatch_gate, + enqueue_limiter: Arc::new(Semaphore::new(max_inflight)), metrics, rule_engine, - target_list: Arc::new(RwLock::new(TargetList::new())), + target_list, send_limiter: Arc::new(Semaphore::new(max_inflight)), } } @@ -99,6 +203,10 @@ impl EventNotifier { Arc::clone(&self.target_list) } + pub(crate) fn dispatch_gate(&self) -> Arc> { + self.dispatch_gate.clone() + } + /// Returns a list of ARNs for the registered targets /// /// # Arguments @@ -115,12 +223,9 @@ impl EventNotifier { .collect() } - /// Removes all targets pub async fn remove_all_bucket_targets(&self) { let mut target_list_guard = self.target_list.write().await; - // The logic for sending cancel signals via stream_cancel_senders would be removed. - // TargetList::clear_targets_only already handles calling target.close(). - target_list_guard.clear_targets_only().await; // Modified clear to not re-cancel + target_list_guard.clear_targets_only().await; info!( event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, component = LOG_COMPONENT_NOTIFY, @@ -155,7 +260,14 @@ impl EventNotifier { return; } let target_ids_len = target_ids.len(); - let mut handles = vec![]; + let mut deferred_handles = Vec::new(); + let mut direct_handles = Vec::new(); + + // A lifecycle writer holds this gate only while handing queue-store + // ownership from one runtime generation to the next. Taking the read + // guard before cloning targets means the writer both blocks new sends + // and drains every save already using the old generation. + let dispatch_guard = Arc::new(self.dispatch_gate.clone().read_owned().await); // Use scope to limit the borrow scope of target_list let target_list_guard = self.target_list.read().await; @@ -185,10 +297,17 @@ impl EventNotifier { ); continue; } - let limiter = self.send_limiter.clone(); + let is_deferred = target_for_task.store().is_some(); + let direct_dispatch_lease = (!is_deferred).then(|| target_list_guard.direct_dispatch_lease()); + let direct_cancellation = direct_dispatch_lease.as_ref().map(DirectDispatchLease::cancellation); + let deferred_dispatch_guard = is_deferred.then(|| Arc::clone(&dispatch_guard)); + let limiter = if is_deferred { + self.enqueue_limiter.clone() + } else { + self.send_limiter.clone() + }; let metrics = self.metrics.clone(); let event_clone = event.clone(); - let is_deferred = target_for_task.store().is_some(); let target_name_for_task = target_for_task.name(); // Get the name before generating the task debug!( event = EVENT_NOTIFY_DISPATCH_STARTED, @@ -207,9 +326,32 @@ impl EventNotifier { data: event_clone.as_ref().clone(), }); let handle = tokio::spawn(async move { + let _direct_dispatch_lease = direct_dispatch_lease; + let _deferred_dispatch_guard = deferred_dispatch_guard; metrics.increment_processing(); - let _permit = match limiter.acquire_owned().await { - Ok(p) => p, + let permit = if let Some(cancellation) = direct_cancellation { + tokio::select! { + biased; + _ = cancellation.cancelled() => { + metrics.decrement_processing(); + metrics.increment_skipped(); + debug!( + event = EVENT_NOTIFY_DISPATCH_SKIPPED, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_DISPATCH, + target_id = %target_name_for_task, + reason = "runtime_generation_replaced", + "Skipped pending direct notify dispatch" + ); + return; + } + permit = limiter.acquire_owned() => permit, + } + } else { + limiter.acquire_owned().await + }; + let _permit = match permit { + Ok(permit) => permit, Err(e) => { error!( event = EVENT_NOTIFY_DISPATCH_FAILED, @@ -218,7 +360,7 @@ impl EventNotifier { target_id = %target_name_for_task, error = %e, reason = "permit_acquire_failed", - "Failed to acquire notify send permit" + "Failed to acquire notify dispatch permit" ); metrics.increment_failed(); return; @@ -260,7 +402,11 @@ impl EventNotifier { ); } }); - handles.push(handle); + if is_deferred { + deferred_handles.push(handle); + } else { + direct_handles.push(handle); + } } else { warn!( event = EVENT_NOTIFY_DISPATCH_FAILED, @@ -276,19 +422,13 @@ impl EventNotifier { // target_list is automatically released here drop(target_list_guard); - // Wait for all tasks to be completed - for handle in handles { - if let Err(e) = handle.await { - error!( - event = EVENT_NOTIFY_DISPATCH_FAILED, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_DISPATCH, - error = %e, - reason = "join_failed", - "Notify dispatch task failed" - ); - } - } + // Every store-backed save owns a share of the generation guard, so + // caller cancellation cannot race lifecycle handoff with an enqueue. + // Direct targets own no queue store and may finish against the detached + // target while lifecycle progresses. + drop(dispatch_guard); + wait_for_dispatch_tasks(deferred_handles).await; + wait_for_dispatch_tasks(direct_handles).await; debug!( event = EVENT_NOTIFY_DISPATCH_COMPLETED, component = LOG_COMPONENT_NOTIFY, @@ -320,7 +460,7 @@ impl EventNotifier { target_list_guard.add(target)?; } - info!( + tracing::info!( event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, component = LOG_COMPONENT_NOTIFY, subsystem = LOG_SUBSYSTEM_DISPATCH, @@ -334,6 +474,7 @@ impl EventNotifier { /// A thread-safe list of targets pub struct TargetList { + direct_dispatches: Arc, /// Map of TargetID to Target runtime: TargetRuntimeManager, } @@ -348,6 +489,7 @@ impl TargetList { /// Creates a new TargetList pub fn new() -> Self { TargetList { + direct_dispatches: Arc::new(DirectDispatchTracker::new()), runtime: TargetRuntimeManager::new(), } } @@ -435,6 +577,20 @@ impl TargetList { self.runtime.status_snapshot(replay_workers) } + fn direct_dispatch_lease(&self) -> DirectDispatchLease { + self.direct_dispatches.acquire() + } + + pub(crate) fn replace_runtime( + &mut self, + replacement: TargetRuntimeManager, + ) -> (TargetRuntimeManager, Arc) { + let runtime = std::mem::replace(&mut self.runtime, replacement); + let direct_dispatches = std::mem::replace(&mut self.direct_dispatches, Arc::new(DirectDispatchTracker::new())); + direct_dispatches.cancel_pending(); + (runtime, direct_dispatches) + } + pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager { &mut self.runtime } @@ -458,7 +614,7 @@ mod tests { use rustfs_s3_types::EventName; use rustfs_targets::StoreError; use rustfs_targets::{ - TargetError, + ReplayWorkerManager, TargetError, store::{Key, QueueStore, Store}, target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}, }; @@ -466,6 +622,7 @@ mod tests { Arc, atomic::{AtomicUsize, Ordering}, }; + use tokio::sync::Notify; #[tokio::test] async fn encoded_event_key_matches_raw_prefix_suffix_filter() { @@ -525,19 +682,44 @@ mod tests { #[derive(Clone)] struct TestTarget { + block_first_save: Option<(Arc, Arc)>, + close_calls: Arc, + close_entered: Option>, id: TargetID, enabled: bool, save_calls: Arc, + selected_calls: Arc, + store: Option>, } impl TestTarget { fn new(id: &str, name: &str, enabled: bool) -> Self { Self { + block_first_save: None, + close_calls: Arc::new(AtomicUsize::new(0)), + close_entered: None, id: TargetID::new(id.to_string(), name.to_string()), enabled, save_calls: Arc::new(AtomicUsize::new(0)), + selected_calls: Arc::new(AtomicUsize::new(0)), + store: None, } } + + fn with_blocked_first_save(mut self, entered: Arc, release: Arc) -> Self { + self.block_first_save = Some((entered, release)); + self + } + + fn with_store(mut self, store: QueueStore) -> Self { + self.store = Some(store); + self + } + + fn with_close_observer(mut self, close_entered: Arc) -> Self { + self.close_entered = Some(close_entered); + self + } } #[async_trait] @@ -554,7 +736,13 @@ mod tests { } async fn save(&self, _event: Arc>) -> Result<(), TargetError> { - self.save_calls.fetch_add(1, Ordering::SeqCst); + let call = self.save_calls.fetch_add(1, Ordering::SeqCst); + if call == 0 + && let Some((entered, release)) = &self.block_first_save + { + entered.notify_one(); + release.notified().await; + } Ok(()) } @@ -563,11 +751,17 @@ mod tests { } async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + if let Some(close_entered) = &self.close_entered { + close_entered.notify_one(); + } Ok(()) } fn store(&self) -> Option<&(dyn Store + Send + Sync)> { - None + self.store + .as_ref() + .map(|store| store as &(dyn Store + Send + Sync)) } fn clone_dyn(&self) -> Box + Send + Sync> { @@ -580,10 +774,425 @@ mod tests { } fn is_enabled(&self) -> bool { + self.selected_calls.fetch_add(1, Ordering::SeqCst); self.enabled } } + #[tokio::test] + async fn lifecycle_pause_drains_entered_deferred_dispatch_and_blocks_new_dispatch() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let save_entered = Arc::new(Notify::new()); + let save_release = Arc::new(Notify::new()); + let queue_dir = tempfile::tempdir().expect("queue tempdir should be created"); + let target = TestTarget::new("gated-target", "webhook", true) + .with_blocked_first_save(save_entered.clone(), save_release.clone()) + .with_store(QueueStore::new(queue_dir.path(), 16, ".event")); + + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone()); + rule_engine.set_bucket_rules("bucket", rules_map).await; + notifier + .target_list() + .write() + .await + .add(Arc::new(target.clone())) + .expect("target install should succeed"); + + let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate( + notifier.target_list(), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(1)), + metrics, + ); + let first_dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut))) + .await; + } + }); + save_entered.notified().await; + assert_eq!(target.save_calls.load(Ordering::SeqCst), 1); + + let mut pause = Box::pin(facade.pause_dispatch()); + tokio::select! { + biased; + _ = &mut pause => panic!("lifecycle pause crossed an in-flight dispatch"), + _ = std::future::ready(()) => {} + } + + save_release.notify_one(); + first_dispatch.await.expect("first dispatch task should finish"); + let pause_guard = pause.await; + + let replacement = TestTarget::new("gated-target", "webhook", true); + { + let target_list = notifier.target_list(); + let mut target_list = target_list.write().await; + target_list.clear(); + target_list + .add(Arc::new(replacement.clone())) + .expect("replacement target install should succeed"); + } + + let mut second_dispatch = + Box::pin(notifier.send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut)))); + tokio::select! { + biased; + _ = &mut second_dispatch => panic!("dispatch crossed the lifecycle pause"), + _ = std::future::ready(()) => {} + } + assert_eq!( + replacement.selected_calls.load(Ordering::SeqCst), + 0, + "a paused dispatch must not select a target from the replacement generation early" + ); + assert_eq!(target.save_calls.load(Ordering::SeqCst), 1); + assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 0); + + drop(pause_guard); + second_dispatch.await; + assert_eq!(target.selected_calls.load(Ordering::SeqCst), 1); + assert_eq!(target.save_calls.load(Ordering::SeqCst), 1); + assert_eq!(replacement.selected_calls.load(Ordering::SeqCst), 1); + assert_eq!(replacement.save_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn lifecycle_pause_does_not_wait_for_direct_network_dispatch() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let save_entered = Arc::new(Notify::new()); + let save_release = Arc::new(Notify::new()); + let target = + TestTarget::new("direct-target", "webhook", true).with_blocked_first_save(save_entered.clone(), save_release.clone()); + + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone()); + rule_engine.set_bucket_rules("bucket", rules_map).await; + notifier + .target_list() + .write() + .await + .add(Arc::new(target)) + .expect("target install should succeed"); + + let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate( + notifier.target_list(), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(1)), + metrics, + ); + let dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut))) + .await; + } + }); + save_entered.notified().await; + + let mut pause = Box::pin(facade.pause_dispatch()); + let pause_guard = tokio::select! { + biased; + guard = &mut pause => guard, + _ = std::future::ready(()) => panic!("a direct network send blocked lifecycle handoff"), + }; + drop(pause_guard); + save_release.notify_one(); + dispatch.await.expect("direct dispatch should finish after release"); + } + + #[tokio::test] + async fn replacement_cancels_permit_waiting_direct_dispatch_before_closing_target() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier { + send_limiter: Arc::new(Semaphore::new(1)), + ..EventNotifier::new(metrics.clone(), rule_engine.clone()) + }); + let first_entered = Arc::new(Notify::new()); + let first_release = Arc::new(Notify::new()); + let close_entered = Arc::new(Notify::new()); + let target = TestTarget::new("direct-target", "webhook", true) + .with_blocked_first_save(first_entered.clone(), first_release.clone()) + .with_close_observer(close_entered); + + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone()); + rule_engine.set_bucket_rules("bucket", rules_map).await; + notifier + .target_list() + .write() + .await + .add(Arc::new(target.clone())) + .expect("target should install"); + + let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate( + notifier.target_list(), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(1)), + metrics.clone(), + ); + let first = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "first", EventName::ObjectCreatedPut))) + .await; + } + }); + first_entered.notified().await; + + // This task selects the old generation and acquires its lease before + // waiting for the saturated direct-send permit. + let second = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "second", EventName::ObjectCreatedPut))) + .await; + } + }); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while target.selected_calls.load(Ordering::SeqCst) != 2 { + tokio::task::yield_now().await; + } + }) + .await + .expect("both direct sends should select the old generation"); + + let activation = facade.activate_targets_with_replay(Vec::new()).await; + let mut replace = Box::pin(facade.replace_targets(activation)); + tokio::select! { + biased; + result = &mut replace => panic!("replacement closed a generation with selected direct sends: {result:?}"), + _ = std::future::ready(()) => {} + } + assert_eq!(target.close_calls.load(Ordering::SeqCst), 0); + + first_release.notify_one(); + first.await.expect("first direct dispatch should finish"); + second.await.expect("permit-waiting direct dispatch should be cancelled"); + replace.await.expect("replacement should close after direct leases drain"); + assert_eq!(target.save_calls.load(Ordering::SeqCst), 1); + assert_eq!(target.close_calls.load(Ordering::SeqCst), 1); + assert_eq!(metrics.processing_count(), 0); + assert_eq!(metrics.processed_count(), 1); + assert_eq!(metrics.skipped_count(), 1); + } + + #[tokio::test] + async fn caller_abort_does_not_release_deferred_generation_lease() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let save_entered = Arc::new(Notify::new()); + let save_release = Arc::new(Notify::new()); + let queue_dir = tempfile::tempdir().expect("queue tempdir should be created"); + let target = TestTarget::new("deferred", "webhook", true) + .with_blocked_first_save(save_entered.clone(), save_release.clone()) + .with_store(QueueStore::new(queue_dir.path(), 16, ".event")); + + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone()); + rule_engine.set_bucket_rules("bucket", rules_map).await; + notifier + .target_list() + .write() + .await + .add(Arc::new(target)) + .expect("target should install"); + let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate( + notifier.target_list(), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(1)), + metrics, + ); + + let dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut))) + .await; + } + }); + save_entered.notified().await; + dispatch.abort(); + let _ = dispatch.await; + + let mut pause = Box::pin(facade.pause_dispatch()); + tokio::select! { + biased; + _ = &mut pause => panic!("caller abort released the deferred generation lease"), + _ = std::future::ready(()) => {} + } + save_release.notify_one(); + let pause_guard = pause.await; + drop(pause_guard); + } + + #[tokio::test] + async fn deferred_enqueue_concurrency_is_bounded() { + const LIMIT: usize = 2; + const TARGETS: usize = 3; + + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier { + enqueue_limiter: Arc::new(Semaphore::new(LIMIT)), + ..EventNotifier::new(metrics, rule_engine.clone()) + }); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let queue_dir = tempfile::tempdir().expect("queue tempdir should be created"); + let mut targets = Vec::new(); + let mut rules_map = RulesMap::new(); + for index in 0..TARGETS { + let target = TestTarget::new(&format!("deferred-{index}"), "webhook", true) + .with_blocked_first_save(entered.clone(), release.clone()) + .with_store(QueueStore::new(queue_dir.path().join(index.to_string()), 16, ".event")); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target.id.clone()); + notifier + .target_list() + .write() + .await + .add(Arc::new(target.clone())) + .expect("target should install"); + targets.push(target); + } + rule_engine.set_bucket_rules("bucket", rules_map).await; + + let dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut))) + .await; + } + }); + let total_calls = || { + targets + .iter() + .map(|target| target.save_calls.load(Ordering::SeqCst)) + .sum::() + }; + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while total_calls() != LIMIT { + tokio::task::yield_now().await; + } + }) + .await + .expect("the configured number of enqueues should enter"); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert_eq!(total_calls(), LIMIT, "enqueue concurrency exceeded its semaphore capacity"); + + release.notify_waiters(); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while total_calls() != TARGETS { + tokio::task::yield_now().await; + } + }) + .await + .expect("the waiting enqueue should enter after a permit is released"); + release.notify_waiters(); + dispatch.await.expect("all bounded enqueues should finish"); + } + + #[tokio::test] + async fn deferred_enqueue_does_not_wait_for_a_blocked_direct_send_permit() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier { + send_limiter: Arc::new(Semaphore::new(1)), + ..EventNotifier::new(metrics.clone(), rule_engine.clone()) + }); + let direct_entered = Arc::new(Notify::new()); + let direct_release = Arc::new(Notify::new()); + let direct = + TestTarget::new("direct", "webhook", true).with_blocked_first_save(direct_entered.clone(), direct_release.clone()); + let deferred_entered = Arc::new(Notify::new()); + let deferred_release = Arc::new(Notify::new()); + let queue_dir = tempfile::tempdir().expect("queue tempdir should be created"); + let deferred = TestTarget::new("deferred", "webhook", true) + .with_blocked_first_save(deferred_entered.clone(), deferred_release.clone()) + .with_store(QueueStore::new(queue_dir.path(), 16, ".event")); + + let mut direct_rules = RulesMap::new(); + direct_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), direct.id.clone()); + rule_engine.set_bucket_rules("direct-bucket", direct_rules).await; + let mut deferred_rules = RulesMap::new(); + deferred_rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), deferred.id.clone()); + rule_engine.set_bucket_rules("deferred-bucket", deferred_rules).await; + { + let target_list = notifier.target_list(); + let mut target_list = target_list.write().await; + target_list.add(Arc::new(direct)).expect("direct target should install"); + target_list.add(Arc::new(deferred)).expect("deferred target should install"); + } + + let facade = crate::runtime_facade::NotifyRuntimeFacade::new_with_dispatch_gate( + notifier.target_list(), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(1)), + metrics, + ); + let direct_dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("direct-bucket", "object", EventName::ObjectCreatedPut))) + .await; + } + }); + direct_entered.notified().await; + let deferred_dispatch = tokio::spawn({ + let notifier = notifier.clone(); + async move { + notifier + .send(Arc::new(Event::new_test_event("deferred-bucket", "object", EventName::ObjectCreatedPut))) + .await; + } + }); + tokio::time::timeout(std::time::Duration::from_secs(1), deferred_entered.notified()) + .await + .expect("queue persistence must not wait behind a direct network send permit"); + + let mut pause = Box::pin(facade.pause_dispatch()); + tokio::select! { + biased; + _ = &mut pause => panic!("lifecycle pause crossed the blocked deferred enqueue"), + _ = std::future::ready(()) => {} + } + deferred_release.notify_one(); + deferred_dispatch + .await + .expect("deferred dispatch should finish after release"); + let pause_guard = tokio::select! { + biased; + guard = &mut pause => guard, + _ = std::future::ready(()) => panic!("direct network delivery kept the lifecycle gate locked"), + }; + drop(pause_guard); + + direct_release.notify_one(); + direct_dispatch.await.expect("direct dispatch should finish after release"); + } + #[tokio::test] async fn test_send_event_skips_disabled_target() { let rule_engine = NotifyRuleEngine::new(); diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index d5e22638e..e4aa0305b 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -38,6 +38,11 @@ impl TargetRegistry { TargetRegistry { plugins } } + #[cfg(test)] + pub(crate) fn with_plugins(plugins: TargetPluginRegistry) -> Self { + Self { plugins } + } + pub fn supports_target_type(&self, target_type: &str) -> bool { self.plugins.supports_target_type(target_type) } @@ -67,6 +72,15 @@ impl TargetRegistry { ) -> Result + Send + Sync>>, TargetError> { self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await } + + pub(crate) async fn create_dormant_targets_from_config( + &self, + config: &Config, + ) -> Result<(Vec + Send + Sync>>, Vec), TargetError> { + self.plugins + .create_dormant_targets_from_config(config, NOTIFY_ROUTE_PREFIX) + .await + } } #[cfg(test)] diff --git a/crates/notify/src/runtime_facade.rs b/crates/notify/src/runtime_facade.rs index c6383844a..683a01ba6 100644 --- a/crates/notify/src/runtime_facade.rs +++ b/crates/notify/src/runtime_facade.rs @@ -12,13 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList}; -use rustfs_targets::{ - BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target, +use crate::{ + Event, NotificationError, + error::transition_join_error, + integration::NotificationMetrics, + notifier::{DirectDispatchTracker, SharedNotifyTargetList, shared_dispatch_gate}, +}; +use rustfs_targets::{ + BuiltinPluginRuntimeAdapter, OpenedActivation, PluginRuntimeAdapter, PreparedActivation, ReplayEvent, ReplayWorkerManager, + RuntimeActivation, Target, TargetRuntimeManager, +}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, }; -use std::sync::Arc; use std::time::Duration; -use tokio::sync::{RwLock, Semaphore}; +use tokio::sync::{OwnedRwLockWriteGuard, RwLock, Semaphore}; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; const LOG_COMPONENT_NOTIFY: &str = "notify"; @@ -26,12 +36,25 @@ const LOG_SUBSYSTEM_RUNTIME: &str = "runtime"; const EVENT_NOTIFY_RUNTIME_LIFECYCLE: &str = "notify_runtime_lifecycle"; const EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED: &str = "notify_runtime_shutdown_failed"; const EVENT_NOTIFY_REPLAY_RETRY_EXHAUSTED: &str = "notify_replay_retry_exhausted"; +const TARGET_CLOSE_TIMEOUT: Duration = Duration::from_secs(10); +pub(crate) struct DetachedNotifyRuntime { + direct_dispatches: Arc, + runtime: TargetRuntimeManager, + replay_workers: ReplayWorkerManager, +} + +// Multi-lock publication order: replay_workers -> target_list -> +// publication_gate. The lifecycle may already hold dispatch_gate while +// entering this facade; no path that needs these three locks may acquire +// publication_gate before either runtime lock. #[derive(Clone)] pub struct NotifyRuntimeFacade { + dispatch_gate: Arc>, + legacy_terminated: Arc, target_list: SharedNotifyTargetList, replay_workers: Arc>, - runtime_adapter: Arc>, + runtime_adapter: Arc>, } impl NotifyRuntimeFacade { @@ -41,6 +64,18 @@ impl NotifyRuntimeFacade { concurrency_limiter: Arc, metrics: Arc, ) -> Self { + let dispatch_gate = shared_dispatch_gate(&target_list, None); + Self::new_with_dispatch_gate(target_list, replay_workers, dispatch_gate, concurrency_limiter, metrics) + } + + pub(crate) fn new_with_dispatch_gate( + target_list: SharedNotifyTargetList, + replay_workers: Arc>, + dispatch_gate: Arc>, + concurrency_limiter: Arc, + metrics: Arc, + ) -> Self { + let dispatch_gate = shared_dispatch_gate(&target_list, Some(dispatch_gate)); let replay_metrics = metrics; let runtime_adapter = BuiltinPluginRuntimeAdapter::new( Arc::new(move |event: ReplayEvent| { @@ -97,36 +132,166 @@ impl NotifyRuntimeFacade { ); Self { + dispatch_gate, + legacy_terminated: Arc::new(AtomicBool::new(false)), target_list, replay_workers, runtime_adapter: Arc::new(runtime_adapter), } } + pub(crate) async fn pause_dispatch(&self) -> OwnedRwLockWriteGuard<()> { + self.dispatch_gate.clone().write_owned().await + } + pub async fn activate_targets_with_replay( &self, targets: Vec + Send + Sync>>, ) -> RuntimeActivation { - self.runtime_adapter.activate_with_replay(targets).await + // The compatibility pair must not start replacement replay before + // replace_targets has stopped and joined the current generation. + self.runtime_adapter.prepare_dormant_compat_activation(targets).await } - pub async fn replace_targets(&self, activation: RuntimeActivation) -> Result<(), NotificationError> { - // Lock order: replay_workers -> target_list (matches notify AGENTS.md). + pub(crate) async fn prepare_targets( + &self, + targets: Vec + Send + Sync>>, + cancellation: &CancellationToken, + ) -> PreparedActivation { + self.runtime_adapter.prepare_targets_cancellable(targets, cancellation).await + } + + pub(crate) async fn open_prepared_stores( + &self, + prepared: PreparedActivation, + ) -> Result<(OpenedActivation, PreparedActivation), NotificationError> { + let runtime_adapter = self.runtime_adapter.clone(); + tokio::task::spawn_blocking(move || runtime_adapter.open_prepared_stores(prepared)) + .await + .map_err(transition_join_error) + } + + pub(crate) async fn commit_prepared( + &self, + opened: OpenedActivation, + on_committed: Committed, + ) -> (DetachedNotifyRuntime, PreparedActivation) + where + Committed: FnOnce(bool), + { + // The lifecycle coordinator validates the generation immediately before + // entering the non-cancellable handoff. Once old replay workers have + // been joined, this generation must publish before a later accepted + // intent can run; abandoning it here would leave the old runtime + // visible without replay workers. let mut replay_workers = self.replay_workers.write().await; let mut target_list = self.target_list.write().await; - self.runtime_adapter - .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation) - .await - .map_err(NotificationError::Target)?; + + let (activation, rejected) = self.runtime_adapter.try_activate_prepared(opened); + let fully_activated = rejected.failure_summary().is_none(); + let (runtime, replay_workers, direct_dispatches) = + Self::swap_activation(&mut target_list, &mut replay_workers, activation); + on_committed(fully_activated); + ( + DetachedNotifyRuntime { + direct_dispatches, + runtime, + replay_workers, + }, + rejected, + ) + } + + pub(crate) async fn commit_disabled(&self, on_committed: Committed) -> DetachedNotifyRuntime + where + Committed: FnOnce(), + { + // Lock order: replay_workers -> target_list. The lifecycle coordinator + // crosses its publication barrier before stopping the old workers, so + // this non-cancellable commit must publish even if a newer intent was + // accepted while the workers joined. + let mut replay_workers = self.replay_workers.write().await; + let mut target_list = self.target_list.write().await; + + let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new()); + let detached = DetachedNotifyRuntime { + direct_dispatches, + runtime, + replay_workers: std::mem::take(&mut *replay_workers), + }; + on_committed(); + detached + } + + pub async fn replace_targets(&self, mut activation: RuntimeActivation) -> Result<(), NotificationError> { + // A caller may supply an activation created outside the compatibility + // prepare method. Stop any already-running replacement workers before + // entering the ordered handoff; the supported activate→replace pair is + // dormant here and therefore never overlaps the old generation. + self.runtime_adapter.stop_replay_workers(&mut activation.replay_workers).await; + let dispatch_guard = self.pause_dispatch().await; + if self.legacy_terminated.load(Ordering::Acquire) { + drop(dispatch_guard); + self.runtime_adapter + .close_compat_activation(activation) + .await + .map_err(NotificationError::Target)?; + return Err(NotificationError::Initialization("Notification runtime has terminated".to_string())); + } + self.stop_active_replay_workers().await; + let (activation, open_rejected, activation_rejected) = self.runtime_adapter.start_dormant_compat_activation(activation); + let activation_failures = [open_rejected.failure_summary(), activation_rejected.failure_summary()] + .into_iter() + .flatten() + .collect::>(); + let (old_runtime, old_replay_workers, old_direct_dispatches) = { + // Lock order: replay_workers -> target_list (matches notify AGENTS.md). + let mut replay_workers = self.replay_workers.write().await; + let mut target_list = self.target_list.write().await; + Self::swap_activation(&mut target_list, &mut replay_workers, activation) + }; + drop(dispatch_guard); + let close_old = self.close_detached_targets(DetachedNotifyRuntime { + direct_dispatches: old_direct_dispatches, + runtime: old_runtime, + replay_workers: old_replay_workers, + }); + let close_open_rejected = self.close_prepared(open_rejected); + let close_activation_rejected = self.close_prepared(activation_rejected); + let (close_old, close_open_rejected, close_activation_rejected) = + tokio::join!(close_old, close_open_rejected, close_activation_rejected); + close_old?; + close_open_rejected?; + close_activation_rejected?; + if !activation_failures.is_empty() { + return Err(NotificationError::Initialization(format!( + "one or more notification targets failed to activate: {}", + activation_failures.join("; ") + ))); + } Ok(()) } pub async fn stop_replay_workers(&self) { - let mut replay_workers = self.replay_workers.write().await; - self.runtime_adapter.stop_replay_workers(&mut replay_workers).await; + let _dispatch_guard = self.pause_dispatch().await; + self.stop_active_replay_workers().await; + } + + pub(crate) async fn stop_active_replay_workers(&self) { + let mut detached = { + let mut replay_workers = self.replay_workers.write().await; + std::mem::take(&mut *replay_workers) + }; + self.runtime_adapter.stop_replay_workers(&mut detached).await; } pub async fn shutdown(&self) { + let _ = self.shutdown_checked().await; + } + + pub async fn shutdown_checked(&self) -> Result<(), NotificationError> { + let _dispatch_guard = self.pause_dispatch().await; + self.legacy_terminated.store(true, Ordering::Release); info!( event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, component = LOG_COMPONENT_NOTIFY, @@ -135,7 +300,14 @@ impl NotifyRuntimeFacade { "notify runtime lifecycle" ); - let active_targets = self.replay_workers.read().await.len(); + let (detached_runtime, detached_replay_workers, detached_direct_dispatches) = { + // Lock order: replay_workers -> target_list (matches notify AGENTS.md). + let mut replay_workers = self.replay_workers.write().await; + let mut target_list = self.target_list.write().await; + let (runtime, direct_dispatches) = target_list.replace_runtime(TargetRuntimeManager::new()); + (runtime, std::mem::take(&mut *replay_workers), direct_dispatches) + }; + let active_targets = detached_replay_workers.len(); info!( event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, component = LOG_COMPONENT_NOTIFY, @@ -145,25 +317,22 @@ impl NotifyRuntimeFacade { "notify runtime lifecycle" ); - { - // Lock order: replay_workers -> target_list (matches notify AGENTS.md). - let mut replay_workers = self.replay_workers.write().await; - let mut target_list = self.target_list.write().await; - if let Err(err) = self - .runtime_adapter - .shutdown(target_list.runtime_mut(), &mut replay_workers) - .await - { - tracing::error!( - event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED, - component = LOG_COMPONENT_NOTIFY, - subsystem = LOG_SUBSYSTEM_RUNTIME, - error = %err, - "Failed to shutdown notify runtime cleanly" - ); - } + let shutdown_result = self + .close_detached(DetachedNotifyRuntime { + direct_dispatches: detached_direct_dispatches, + runtime: detached_runtime, + replay_workers: detached_replay_workers, + }) + .await; + if let Err(err) = &shutdown_result { + tracing::error!( + event = EVENT_NOTIFY_RUNTIME_SHUTDOWN_FAILED, + component = LOG_COMPONENT_NOTIFY, + subsystem = LOG_SUBSYSTEM_RUNTIME, + error = %err, + "Failed to shutdown notify runtime cleanly" + ); } - tokio::time::sleep(Duration::from_millis(500)).await; info!( event = EVENT_NOTIFY_RUNTIME_LIFECYCLE, @@ -172,6 +341,64 @@ impl NotifyRuntimeFacade { state = "stopped", "notify runtime lifecycle" ); + shutdown_result + } + + fn swap_activation( + target_list: &mut crate::notifier::TargetList, + replay_workers: &mut ReplayWorkerManager, + activation: RuntimeActivation, + ) -> (TargetRuntimeManager, ReplayWorkerManager, Arc) { + let mut replacement = TargetRuntimeManager::new(); + for target in activation.targets { + replacement.add_arc(target); + } + let (runtime, direct_dispatches) = target_list.replace_runtime(replacement); + (runtime, std::mem::replace(replay_workers, activation.replay_workers), direct_dispatches) + } + + pub(crate) async fn stop_detached_replay(&self, detached: &mut DetachedNotifyRuntime) { + // Replay join is intentionally not wrapped in the target-close timeout: + // returning before a worker is confirmed stopped could let a + // replacement drain the same persistent queue concurrently. + self.runtime_adapter.stop_replay_workers(&mut detached.replay_workers).await; + } + + pub(crate) async fn close_detached_targets(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> { + // Direct sends are allowed to finish after the replacement runtime is + // published, but the old targets must remain open until every task that + // selected that generation has released its lease. + detached.direct_dispatches.wait_idle().await; + match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, detached.runtime.clear_and_close()).await { + Ok(close_errors) if close_errors.is_empty() => Ok(()), + Ok(close_errors) => { + let targets = close_errors + .into_iter() + .map(|(target_id, _)| target_id) + .collect::>() + .join("; "); + Err(NotificationError::Target(rustfs_targets::TargetError::Storage(format!( + "Failed to close {targets}" + )))) + } + Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout( + "Timed out closing replaced notification targets".to_string(), + ))), + } + } + + async fn close_detached(&self, mut detached: DetachedNotifyRuntime) -> Result<(), NotificationError> { + self.stop_detached_replay(&mut detached).await; + self.close_detached_targets(detached).await + } + + pub(crate) async fn close_prepared(&self, prepared: PreparedActivation) -> Result<(), NotificationError> { + match tokio::time::timeout(TARGET_CLOSE_TIMEOUT, self.runtime_adapter.close_prepared(prepared)).await { + Ok(result) => result.map_err(NotificationError::Target), + Err(_) => Err(NotificationError::Target(rustfs_targets::TargetError::Timeout( + "Timed out closing superseded notification targets".to_string(), + ))), + } } } @@ -184,26 +411,50 @@ mod tests { }; use async_trait::async_trait; use rustfs_targets::arn::TargetID; - use rustfs_targets::store::{Key, Store}; + use rustfs_targets::store::{Key, QueueStore, Store}; use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; - use tokio::sync::{RwLock, Semaphore}; + use tokio::sync::{Notify, RwLock, Semaphore}; #[derive(Clone)] struct TestTarget { + close_entered: Option>, + close_error: bool, + close_release: Option>, close_calls: Arc, id: TargetID, + store: Option>, } impl TestTarget { fn new(id: &str, name: &str) -> Self { Self { + close_entered: None, + close_error: false, + close_release: None, close_calls: Arc::new(AtomicUsize::new(0)), id: TargetID::new(id.to_string(), name.to_string()), + store: None, } } + + fn with_blocking_close(mut self, entered: Arc, release: Arc) -> Self { + self.close_entered = Some(entered); + self.close_release = Some(release); + self + } + + fn with_close_error(mut self) -> Self { + self.close_error = true; + self + } + + fn with_store(mut self, store: QueueStore) -> Self { + self.store = Some(store); + self + } } #[async_trait] @@ -229,11 +480,22 @@ mod tests { async fn close(&self) -> Result<(), TargetError> { self.close_calls.fetch_add(1, Ordering::SeqCst); + if let Some(entered) = &self.close_entered { + entered.notify_one(); + } + if let Some(release) = &self.close_release { + release.notified().await; + } + if self.close_error { + return Err(TargetError::Storage("forced close failure".to_string())); + } Ok(()) } fn store(&self) -> Option<&(dyn Store + Send + Sync)> { - None + self.store + .as_ref() + .map(|store| store as &(dyn Store + Send + Sync)) } fn clone_dyn(&self) -> Box + Send + Sync> { @@ -254,7 +516,13 @@ mod tests { let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new())); let target_list = notifier.target_list(); let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new())); - let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics); + let facade = NotifyRuntimeFacade::new_with_dispatch_gate( + target_list, + replay_workers.clone(), + notifier.dispatch_gate(), + Arc::new(Semaphore::new(4)), + metrics, + ); (facade, notifier, replay_workers) } @@ -273,6 +541,26 @@ mod tests { assert_eq!(activation.replay_workers.len(), 0); } + #[tokio::test] + async fn compatibility_activation_stays_dormant_until_ordered_replace() { + let (facade, _, replay_workers) = build_facade(); + let queue_root = tempfile::tempdir().expect("queue root"); + let store = QueueStore::new_with_compression(queue_root.path(), 16, ".event", false); + let target = TestTarget::new("primary", "webhook").with_store(store); + + let activation = facade.activate_targets_with_replay(vec![Box::new(target)]).await; + assert_eq!(activation.targets.len(), 1); + assert_eq!(activation.replay_workers.len(), 0, "compatibility prepare must not start replay early"); + assert_eq!(replay_workers.read().await.len(), 0); + + facade + .replace_targets(activation) + .await + .expect("ordered compatibility replace should succeed"); + assert_eq!(replay_workers.read().await.len(), 1); + facade.shutdown_checked().await.expect("test runtime should shut down"); + } + #[tokio::test] async fn runtime_facade_replace_targets_commits_runtime_state() { let (facade, notifier, replay_workers) = build_facade(); @@ -292,4 +580,129 @@ mod tests { assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]); assert_eq!(replay_workers.read().await.len(), 0); } + + #[tokio::test] + async fn runtime_queries_do_not_wait_for_target_close() { + let (facade, notifier, replay_workers) = build_facade(); + let close_entered = Arc::new(Notify::new()); + let close_release = Arc::new(Notify::new()); + let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), close_release.clone()); + facade + .replace_targets(rustfs_targets::RuntimeActivation { + replay_workers: ReplayWorkerManager::new(), + targets: vec![Arc::new(target) as SharedTarget], + }) + .await + .expect("target install should succeed"); + + let shutdown = tokio::spawn({ + let facade = facade.clone(); + async move { facade.shutdown_checked().await } + }); + close_entered.notified().await; + + let target_list = notifier.target_list(); + assert!(target_list.try_read().is_ok(), "target list lock must not be held during close"); + assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during close"); + close_release.notify_one(); + shutdown + .await + .expect("shutdown task should not panic") + .expect("shutdown should succeed"); + + let snapshot = NotifyRuntimeView::new(target_list, replay_workers) + .runtime_status_snapshot() + .await; + assert_eq!(snapshot.target_count, 0); + assert_eq!(snapshot.replay_worker_count, 0); + } + + #[tokio::test] + async fn runtime_locks_are_released_while_replay_worker_joins() { + let (facade, notifier, replay_workers) = build_facade(); + let cancel_received = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel(1); + let join = tokio::spawn({ + let cancel_received = cancel_received.clone(); + let release = release.clone(); + async move { + let _ = cancel_rx.recv().await; + cancel_received.notify_one(); + release.notified().await; + } + }); + replay_workers + .write() + .await + .insert_with_handle("blocked-worker".to_string(), cancel_tx, join); + + let stop = tokio::spawn({ + let facade = facade.clone(); + async move { facade.stop_replay_workers().await } + }); + cancel_received.notified().await; + + assert!(replay_workers.try_read().is_ok(), "replay manager lock must not be held during join"); + let target_list = notifier.target_list(); + assert!(target_list.try_read().is_ok(), "target list lock must not be held during replay join"); + + release.notify_one(); + stop.await.expect("stop task should finish"); + } + + #[tokio::test] + async fn shutdown_returns_close_error_after_detaching_runtime() { + let (facade, notifier, replay_workers) = build_facade(); + let target = TestTarget::new("primary", "webhook").with_close_error(); + facade + .replace_targets(rustfs_targets::RuntimeActivation { + replay_workers: ReplayWorkerManager::new(), + targets: vec![Arc::new(target) as SharedTarget], + }) + .await + .expect("target install should succeed"); + + let err = facade.shutdown_checked().await.expect_err("close failure should propagate"); + assert!(matches!(err, crate::NotificationError::Target(TargetError::Storage(_)))); + + let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers) + .runtime_status_snapshot() + .await; + assert_eq!(snapshot.target_count, 0); + assert_eq!(snapshot.replay_worker_count, 0); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_bounds_a_target_that_never_closes() { + let (facade, notifier, replay_workers) = build_facade(); + let close_entered = Arc::new(Notify::new()); + let never_release = Arc::new(Notify::new()); + let target = TestTarget::new("primary", "webhook").with_blocking_close(close_entered.clone(), never_release); + facade + .replace_targets(rustfs_targets::RuntimeActivation { + replay_workers: ReplayWorkerManager::new(), + targets: vec![Arc::new(target) as SharedTarget], + }) + .await + .expect("target install should succeed"); + + let shutdown = tokio::spawn({ + let facade = facade.clone(); + async move { facade.shutdown_checked().await } + }); + close_entered.notified().await; + tokio::time::advance(super::TARGET_CLOSE_TIMEOUT).await; + + let err = shutdown + .await + .expect("shutdown task should not panic") + .expect_err("blocked close must time out"); + assert!(matches!(err, crate::NotificationError::Target(TargetError::Timeout(_)))); + let snapshot = NotifyRuntimeView::new(notifier.target_list(), replay_workers) + .runtime_status_snapshot() + .await; + assert_eq!(snapshot.target_count, 0); + assert_eq!(snapshot.replay_worker_count, 0); + } } diff --git a/crates/notify/src/services.rs b/crates/notify/src/services.rs index 4ac1450dc..7b25b8ba7 100644 --- a/crates/notify/src/services.rs +++ b/crates/notify/src/services.rs @@ -57,7 +57,13 @@ impl NotifyServices { live_event_history: Arc>, ) -> Self { let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone()); - let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone()); + let runtime_facade = NotifyRuntimeFacade::new_with_dispatch_gate( + target_list, + stream_cancellers, + notifier.dispatch_gate(), + concurrency_limiter, + metrics.clone(), + ); let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone()); let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view); let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history); diff --git a/crates/notify/src/storage_api.rs b/crates/notify/src/storage_api.rs index fb8d4fe2b..9f4a44368 100644 --- a/crates/notify/src/storage_api.rs +++ b/crates/notify/src/storage_api.rs @@ -15,35 +15,70 @@ use std::sync::Arc; use rustfs_ecstore::api::config::com::{ - read_config_without_migrate as read_notify_config_without_migrate_from_backend, - save_server_config as save_notify_server_config_to_backend, + read_config_without_migrate_no_lock as read_notify_config_without_migrate_from_backend_no_lock, + read_existing_server_config_no_lock as read_existing_notify_config_from_backend_no_lock, + save_server_config_no_lock as save_notify_server_config_to_backend_no_lock, + with_server_config_read_lock as with_notify_server_config_read_lock_from_backend, + with_server_config_write_lock as with_notify_server_config_write_lock_from_backend, }; use rustfs_ecstore::api::runtime::object_store_handle as resolve_notify_object_store_handle_from_backend; -pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore; +pub use rustfs_ecstore::api::storage::ECStore as NotifyStore; pub(crate) fn resolve_notify_object_store_handle() -> Option> { resolve_notify_object_store_handle_from_backend() } -pub(crate) async fn read_notify_server_config_without_migrate( +pub(crate) async fn read_notify_server_config_without_migrate_no_lock( store: Arc, ) -> Result { - read_notify_config_without_migrate_from_backend(store) + read_notify_config_without_migrate_from_backend_no_lock(store) .await .map_err(|err| err.to_string()) } -pub(crate) async fn save_notify_server_config( +pub(crate) async fn read_existing_notify_server_config_no_lock( + store: Arc, +) -> Result { + read_existing_notify_config_from_backend_no_lock(store) + .await + .map_err(|err| err.to_string()) +} + +pub(crate) async fn save_notify_server_config_no_lock( store: Arc, config: &rustfs_config::server_config::Config, ) -> Result<(), String> { - save_notify_server_config_to_backend(store, config) + save_notify_server_config_to_backend_no_lock(store, config) + .await + .map_err(|err| err.to_string()) +} + +pub(crate) async fn with_notify_server_config_write_lock(store: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + with_notify_server_config_write_lock_from_backend(store, operation) + .await + .map_err(|err| err.to_string()) +} + +pub(crate) async fn with_notify_server_config_read_lock(store: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + with_notify_server_config_read_lock_from_backend(store, operation) .await .map_err(|err| err.to_string()) } pub(crate) mod crate_boundary { pub(crate) use super::{ - read_notify_server_config_without_migrate, resolve_notify_object_store_handle, save_notify_server_config, + read_existing_notify_server_config_no_lock, read_notify_server_config_without_migrate_no_lock, + resolve_notify_object_store_handle, save_notify_server_config_no_lock, with_notify_server_config_read_lock, + with_notify_server_config_write_lock, }; } diff --git a/crates/notify/tests/global_lifecycle.rs b/crates/notify/tests/global_lifecycle.rs new file mode 100644 index 000000000..713aed874 --- /dev/null +++ b/crates/notify/tests/global_lifecycle.rs @@ -0,0 +1,80 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_config::server_config::Config; +use rustfs_notify::{NotificationError, NotificationRuntimeState, ensure_live_events, notification_system, reconcile}; +use std::sync::Arc; + +fn assert_terminated(error: NotificationError) { + assert!(matches!( + error, + NotificationError::Initialization(detail) if detail == "Notification runtime has terminated" + )); +} + +#[tokio::test] +async fn global_singleton_survives_suspend_but_not_process_termination() { + let system = ensure_live_events(); + let same_system = ensure_live_events(); + assert!(Arc::ptr_eq(&system, &same_system)); + assert!(Arc::ptr_eq( + &system, + ¬ification_system().expect("global notification system should exist") + )); + + system + .set_targets_enabled(true, Some(Config::new())) + .await + .expect("empty target runtime should enable"); + assert!(matches!( + system.runtime_lifecycle_state(), + NotificationRuntimeState::TargetsEnabled { .. } + )); + + system + .set_targets_enabled(false, None) + .await + .expect("disable should suspend targets without terminating the singleton"); + assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly); + + system + .set_targets_enabled(true, None) + .await + .expect("a suspended target runtime should be restartable"); + assert!(matches!( + system.runtime_lifecycle_state(), + NotificationRuntimeState::TargetsEnabled { .. } + )); + + system + .shutdown_checked() + .await + .expect("process shutdown should terminate the target runtime"); + assert_eq!(system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated); + + let lazy_system = ensure_live_events(); + assert!(Arc::ptr_eq(&system, &lazy_system)); + assert_terminated( + reconcile(Config::new()) + .await + .expect_err("lazy reconciliation must not restart a terminated process runtime"), + ); + assert_terminated( + lazy_system + .reload_config(Config::new()) + .await + .expect_err("config reload must not restart a terminated process runtime"), + ); + assert_eq!(lazy_system.runtime_lifecycle_state(), NotificationRuntimeState::Terminated); +} diff --git a/crates/notify/tests/legacy_initialize_retry.rs b/crates/notify/tests/legacy_initialize_retry.rs new file mode 100644 index 000000000..6a4cc3720 --- /dev/null +++ b/crates/notify/tests/legacy_initialize_retry.rs @@ -0,0 +1,44 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; +use rustfs_config::server_config::{Config, KVS}; +use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT}; +use rustfs_notify::{initialize, notification_system}; + +#[tokio::test] +async fn failed_legacy_initialize_can_retry_the_stable_singleton() { + let mut invalid_target = KVS::new(); + invalid_target.insert(ENABLE_KEY.to_string(), "on".to_string()); + invalid_target.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string()); + let mut invalid_config = Config::new(); + invalid_config + .0 + .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string()) + .or_default() + .insert("primary".to_string(), invalid_target); + + initialize(invalid_config) + .await + .expect_err("invalid first target activation should fail"); + initialize(Config::new()) + .await + .expect("legacy initialize should retry after the configuration is fixed"); + + notification_system() + .expect("stable singleton should remain available") + .shutdown_checked() + .await + .expect("test runtime should terminate"); +} diff --git a/crates/targets/Cargo.toml b/crates/targets/Cargo.toml index 027d882e3..aa3e88a79 100644 --- a/crates/targets/Cargo.toml +++ b/crates/targets/Cargo.toml @@ -18,6 +18,7 @@ rustfs-tls-runtime = { workspace = true } rustfs-s3-types = { workspace = true } rustfs-utils = { workspace = true, features = ["egress"] } async-trait = { workspace = true } +futures-util = { workspace = true } async-nats = { workspace = true } deadpool-postgres = { workspace = true } hyper = { workspace = true, features = ["http2", "http1", "server"] } @@ -26,6 +27,7 @@ lapin = { workspace = true, default-features = false, features = ["tokio", "rust libc = { workspace = true } pulsar = { workspace = true, default-features = false, features = ["tokio-rustls-runtime", "telemetry"] } regex = { workspace = true } +rayon = { workspace = true } reqwest = { workspace = true } rumqttc = { workspace = true, features = ["websocket"] } redis = { workspace = true, features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] } @@ -38,6 +40,7 @@ sha2 = { workspace = true } snap = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] } +tokio-util = { workspace = true } tokio-postgres = { workspace = true, default-features = false, features = ["runtime", "with-serde_json-1"] } tokio-postgres-rustls = { workspace = true } tracing = { workspace = true } diff --git a/crates/targets/src/config/common.rs b/crates/targets/src/config/common.rs index 46f365410..4bbbf32e6 100644 --- a/crates/targets/src/config/common.rs +++ b/crates/targets/src/config/common.rs @@ -47,16 +47,13 @@ pub(super) fn split_env_field_and_instance(rest: &str, valid_fields: &HashSet bool { - config - .lookup(ENABLE_KEY) - .map(|v| { - EnableState::from_str(v.as_str()) - .ok() - .map(|s| s.is_enabled()) - .unwrap_or(false) - }) - .unwrap_or(false) +pub(super) fn is_target_enabled(config: &KVS) -> Result { + let Some(value) = config.lookup(ENABLE_KEY) else { + return Ok(false); + }; + EnableState::from_str(value.as_str()) + .map(EnableState::is_enabled) + .map_err(|_| TargetError::Configuration(format!("Invalid {ENABLE_KEY} value '{value}'"))) } pub(super) fn parse_target_bool(value: Option<&str>) -> Option { diff --git a/crates/targets/src/config/instance.rs b/crates/targets/src/config/instance.rs index 09d1d45a7..260ed1cc2 100644 --- a/crates/targets/src/config/instance.rs +++ b/crates/targets/src/config/instance.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::loader::collect_merged_target_configs_from_env; +use super::loader::{ + MergedTargetConfigRecord, collect_merged_target_configs_compat_from_env, collect_merged_target_configs_from_env, +}; +use crate::TargetError; use crate::domain::TargetDomain; use rustfs_config::server_config::{Config, KVS}; use std::collections::HashSet; @@ -86,6 +89,13 @@ pub fn normalize_target_plugin_instances( normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars()) } +pub fn try_normalize_target_plugin_instances( + config: &Config, + descriptor: &TargetPluginInstanceCompatDescriptor<'_>, +) -> Result, TargetError> { + try_normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars()) +} + pub fn normalize_target_plugin_instances_from_env( config: &Config, descriptor: &TargetPluginInstanceCompatDescriptor<'_>, @@ -100,7 +110,7 @@ where .map(|field| (*field).to_string()) .collect::>(); - collect_merged_target_configs_from_env( + collect_merged_target_configs_compat_from_env( config, descriptor.subsystem, descriptor.route_prefix, @@ -109,7 +119,42 @@ where env_vars, ) .into_iter() - .map(|record| TargetPluginInstanceRecord { + .map(|record| target_plugin_instance_record(descriptor, record)) + .collect() +} + +pub fn try_normalize_target_plugin_instances_from_env( + config: &Config, + descriptor: &TargetPluginInstanceCompatDescriptor<'_>, + env_vars: I, +) -> Result, TargetError> +where + I: IntoIterator, +{ + let valid_fields = descriptor + .valid_fields + .iter() + .map(|field| (*field).to_string()) + .collect::>(); + + Ok(collect_merged_target_configs_from_env( + config, + descriptor.subsystem, + descriptor.route_prefix, + descriptor.target_type, + &valid_fields, + env_vars, + )? + .into_iter() + .map(|record| target_plugin_instance_record(descriptor, record)) + .collect()) +} + +fn target_plugin_instance_record( + descriptor: &TargetPluginInstanceCompatDescriptor<'_>, + record: MergedTargetConfigRecord, +) -> TargetPluginInstanceRecord { + TargetPluginInstanceRecord { domain: descriptor.domain, plugin_id: descriptor.plugin_id.to_string(), target_type: descriptor.target_type.to_string(), @@ -123,8 +168,7 @@ where has_env_instance: record.has_env_instance, }, effective_config: record.effective_config, - }) - .collect() + } } pub fn normalize_legacy_target_instances( @@ -149,8 +193,9 @@ where mod tests { use super::{ TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env, - normalize_target_plugin_instances_from_env, + try_normalize_target_plugin_instances_from_env, }; + use crate::TargetError; use crate::domain::TargetDomain; use crate::manifest::builtin_target_manifest; use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; @@ -338,9 +383,50 @@ mod tests { let descriptor = notify_webhook_descriptor(); let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())]; - let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone()); + let canonical = try_normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone()) + .expect("canonical normalization should succeed"); let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env); assert_eq!(canonical, compatibility); } + + #[test] + fn normalize_instances_rejects_invalid_enable_value() { + let error = try_normalize_target_plugin_instances_from_env( + &Config(HashMap::new()), + ¬ify_webhook_descriptor(), + vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())], + ) + .expect_err("invalid enable value must be propagated by the public normalizer"); + + match error { + TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"), + other => panic!("expected a configuration error, got {other}"), + } + } + + #[test] + fn legacy_normalizer_keeps_valid_instance_when_one_enable_is_invalid() { + let instances = normalize_legacy_target_instances_from_env( + &Config(HashMap::new()), + ¬ify_webhook_descriptor(), + vec![ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()), + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()), + ], + ); + + assert_eq!(instances.len(), 2); + let good = instances + .iter() + .find(|instance| instance.instance_id == "good") + .expect("valid instance should remain present"); + let bad = instances + .iter() + .find(|instance| instance.instance_id == "bad") + .expect("invalid legacy instance should remain visible"); + assert!(good.enabled); + assert!(!bad.enabled); + } } diff --git a/crates/targets/src/config/loader.rs b/crates/targets/src/config/loader.rs index 3c372dbd7..a47646351 100644 --- a/crates/targets/src/config/loader.rs +++ b/crates/targets/src/config/loader.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::common::{is_target_enabled, split_env_field_and_instance}; +use crate::TargetError; use rustfs_config::server_config::{Config, KVS}; use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX}; use std::collections::{HashMap, HashSet}; @@ -27,6 +28,15 @@ pub fn collect_target_configs( collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars()) } +pub fn try_collect_target_configs( + config: &Config, + route_prefix: &str, + target_type: &str, + valid_fields: &HashSet, +) -> Result, TargetError> { + try_collect_target_configs_from_env(config, route_prefix, target_type, valid_fields, std::env::vars()) +} + fn is_sensitive_target_field(field_name: &str) -> bool { let field_name = field_name.to_ascii_lowercase(); field_name.contains("password") @@ -123,7 +133,7 @@ pub fn collect_target_configs_from_env( where I: IntoIterator, { - collect_merged_target_configs_from_env( + collect_merged_target_configs_compat_from_env( config, &format!("{route_prefix}{target_type}").to_lowercase(), route_prefix, @@ -137,6 +147,30 @@ where .collect() } +pub fn try_collect_target_configs_from_env( + config: &Config, + route_prefix: &str, + target_type: &str, + valid_fields: &HashSet, + env_vars: I, +) -> Result, TargetError> +where + I: IntoIterator, +{ + Ok(collect_merged_target_configs_from_env( + config, + &format!("{route_prefix}{target_type}").to_lowercase(), + route_prefix, + target_type, + valid_fields, + env_vars, + )? + .into_iter() + .filter(|record| record.enabled) + .map(|record| (record.instance_id, record.effective_config)) + .collect()) +} + pub(crate) fn collect_merged_target_configs_from_env( config: &Config, section_name: &str, @@ -144,7 +178,52 @@ pub(crate) fn collect_merged_target_configs_from_env( target_type: &str, valid_fields: &HashSet, env_vars: I, +) -> Result, TargetError> +where + I: IntoIterator, +{ + collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars) + .into_iter() + .map(|result| result.map_err(|(_, err)| err)) + .collect() +} + +pub(crate) fn collect_merged_target_configs_compat_from_env( + config: &Config, + section_name: &str, + route_prefix: &str, + target_type: &str, + valid_fields: &HashSet, + env_vars: I, ) -> Vec +where + I: IntoIterator, +{ + collect_merged_target_config_results_from_env(config, section_name, route_prefix, target_type, valid_fields, env_vars) + .into_iter() + .map(|result| match result { + Ok(record) => record, + Err((record, err)) => { + warn!( + target_type, + instance_id = %record.instance_id, + error = %err, + "Treating target instance with invalid enable configuration as disabled" + ); + record + } + }) + .collect() +} + +fn collect_merged_target_config_results_from_env( + config: &Config, + section_name: &str, + route_prefix: &str, + target_type: &str, + valid_fields: &HashSet, + env_vars: I, +) -> Vec> where I: IntoIterator, { @@ -220,14 +299,28 @@ where let redacted_config = redacted_target_config(&merged_config); debug!(instance_id = %id, ?redacted_config, "Merged target configuration"); } - merged_configs.push(MergedTargetConfigRecord { - instance_id: id, - enabled: is_target_enabled(&merged_config), - effective_config: merged_config, - has_file_default, - has_file_instance, - has_env_default, - has_env_instance, + merged_configs.push(match is_target_enabled(&merged_config) { + Ok(enabled) => Ok(MergedTargetConfigRecord { + instance_id: id, + enabled, + effective_config: merged_config, + has_file_default, + has_file_instance, + has_env_default, + has_env_instance, + }), + Err(err) => Err(( + MergedTargetConfigRecord { + instance_id: id, + enabled: false, + effective_config: merged_config, + has_file_default, + has_file_instance, + has_env_default, + has_env_instance, + }, + err, + )), }); } @@ -238,8 +331,9 @@ where mod tests { use super::{ collect_env_target_instance_ids_from_env, collect_target_configs_from_env, redact_target_field_value, - redacted_target_config, + redacted_target_config, try_collect_target_configs_from_env, }; + use crate::TargetError; use rustfs_config::notify::{ ENV_NOTIFY_REDIS_ENABLE, ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS, ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE, ENV_NOTIFY_REDIS_URL, NOTIFY_REDIS_KEYS, NOTIFY_ROUTE_PREFIX, @@ -269,7 +363,7 @@ mod tests { cfg.0.insert("notify_webhook".to_string(), subsystem); - let configs = collect_target_configs_from_env( + let configs = try_collect_target_configs_from_env( &cfg, NOTIFY_ROUTE_PREFIX, "webhook", @@ -282,7 +376,8 @@ mod tests { ("RUSTFS_NOTIFY_WEBHOOK_ENABLE".to_string(), "on".to_string()), ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()), ], - ); + ) + .expect("valid env target"); let configs: HashMap = configs.into_iter().collect(); assert_eq!(configs.len(), 2); @@ -295,7 +390,7 @@ mod tests { #[test] fn collect_target_configs_discovers_enabled_instance_from_env() { let cfg = Config(HashMap::new()); - let configs = collect_target_configs_from_env( + let configs = try_collect_target_configs_from_env( &cfg, NOTIFY_ROUTE_PREFIX, "webhook", @@ -307,7 +402,8 @@ mod tests { "https://example.com/from-env".to_string(), ), ], - ); + ) + .expect("valid target configs"); assert_eq!(configs.len(), 1); assert_eq!(configs[0].0, "primary"); @@ -323,7 +419,7 @@ mod tests { subsystem.insert("_".to_string(), default_kvs); cfg.0.insert("notify_webhook".to_string(), subsystem); - let configs = collect_target_configs_from_env( + let configs = try_collect_target_configs_from_env( &cfg, NOTIFY_ROUTE_PREFIX, "webhook", @@ -332,7 +428,8 @@ mod tests { "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(), "https://example.com/secondary".to_string(), )], - ); + ) + .expect("valid target configs"); assert!(configs.is_empty()); } @@ -361,7 +458,7 @@ mod tests { let cfg = Config(HashMap::new()); let valid_fields = NOTIFY_REDIS_KEYS.iter().map(|key| (*key).to_string()).collect(); - let configs = collect_target_configs_from_env( + let configs = try_collect_target_configs_from_env( &cfg, NOTIFY_ROUTE_PREFIX, "redis", @@ -372,7 +469,8 @@ mod tests { (format!("{ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS}_PRIMARY"), "9".to_string()), (format!("{ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE}_PRIMARY"), "off".to_string()), ], - ); + ) + .expect("valid redis target config"); let configs: HashMap = configs.into_iter().collect(); let redis_config = configs.get("primary").expect("redis env target should be discovered"); @@ -383,6 +481,57 @@ mod tests { assert_eq!(redis_config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(), Some("off")); } + #[test] + fn collect_target_configs_rejects_invalid_instance_enable_value() { + let err = try_collect_target_configs_from_env( + &Config(HashMap::new()), + NOTIFY_ROUTE_PREFIX, + "webhook", + &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]), + vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), "invalid".to_string())], + ) + .expect_err("invalid enable value must not look like a disabled target"); + + match err { + TargetError::Configuration(detail) => assert_eq!(detail, "Invalid enable value 'invalid'"), + other => panic!("expected a configuration error, got {other}"), + } + } + + #[test] + fn legacy_collection_keeps_valid_instances_when_one_enable_is_invalid() { + let configs = collect_target_configs_from_env( + &Config(HashMap::new()), + NOTIFY_ROUTE_PREFIX, + "webhook", + &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]), + vec![ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_GOOD".to_string(), "on".to_string()), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_GOOD".to_string(), "https://example.com/good".to_string()), + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_BAD".to_string(), "invalid".to_string()), + ], + ); + + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].0, "good"); + } + + #[test] + fn collect_target_configs_preserves_whitespace_padded_legacy_value() { + let configs = try_collect_target_configs_from_env( + &Config(HashMap::new()), + NOTIFY_ROUTE_PREFIX, + "webhook", + &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]), + vec![("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY".to_string(), " on ".to_string())], + ) + .expect("the shared enable parser accepts surrounding whitespace"); + + assert_eq!(configs.len(), 1); + assert_eq!(configs[0].0, "primary"); + assert_eq!(configs[0].1.lookup(ENABLE_KEY).as_deref(), Some(" on ")); + } + #[test] fn redact_target_field_value_redacts_sensitive_fields() { assert_eq!(redact_target_field_value("password", "secret"), "***redacted***"); diff --git a/crates/targets/src/config/mod.rs b/crates/targets/src/config/mod.rs index 0c255441d..df083edec 100644 --- a/crates/targets/src/config/mod.rs +++ b/crates/targets/src/config/mod.rs @@ -21,10 +21,11 @@ pub use instance::{ LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance, TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances, normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env, + try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env, }; pub use loader::{ collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs, - collect_target_configs_from_env, + collect_target_configs_from_env, try_collect_target_configs, try_collect_target_configs_from_env, }; pub use target_args::{ build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index a45d2968f..e3a410b36 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -43,6 +43,7 @@ pub use config::{ LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance, TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances, normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env, + try_normalize_target_plugin_instances, try_normalize_target_plugin_instances_from_env, }; pub use control_plane::{ TargetPluginEnableState, TargetPluginExternalAction, TargetPluginExternalActionDecision, TargetPluginExternalActionError, @@ -65,8 +66,9 @@ pub use plugin::{ TargetPluginRegistry, TargetRequestValidator, boxed_target, }; pub use runtime::{ - ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, - RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay, + OpenedActivation, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, + RuntimeTargetHealthSnapshot, RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, + activate_targets_with_replay, adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter}, init_target_and_optionally_start_replay, ops_diagnostics::{ diff --git a/crates/targets/src/plugin.rs b/crates/targets/src/plugin.rs index ca7e94108..a95d414ce 100644 --- a/crates/targets/src/plugin.rs +++ b/crates/targets/src/plugin.rs @@ -14,8 +14,9 @@ use crate::{ PluginRuntimeAdapter, RuntimeActivation, Target, TargetError, - config::collect_target_configs, + config::try_collect_target_configs, manifest::{TargetPluginManifest, builtin_target_manifest}, + target::with_deferred_queue_store_open, }; use hashbrown::HashMap; use rustfs_config::server_config::{Config, KVS}; @@ -317,39 +318,68 @@ where config: &Config, route_prefix: &str, ) -> Result>, TargetError> { + self.create_targets_from_config_with_store_mode(config, route_prefix, false) + .await + .map(|(targets, _)| targets) + } + + /// Creates targets while deferring queue-store open until runtime handoff. + /// Unlike the compatibility activation API, lifecycle preparation reports + /// any invalid or unconstructable configured instance so the originating + /// Admin request cannot report a false success. + pub async fn create_dormant_targets_from_config( + &self, + config: &Config, + route_prefix: &str, + ) -> Result<(Vec>, Vec), TargetError> { + self.create_targets_from_config_with_store_mode(config, route_prefix, true) + .await + } + + async fn create_targets_from_config_with_store_mode( + &self, + config: &Config, + route_prefix: &str, + defer_store_open: bool, + ) -> Result<(Vec>, Vec), TargetError> { let mut successful_targets = Vec::new(); - let mut failed_targets = 0usize; + let mut failures = Vec::new(); for (target_type, plugin) in &self.plugins { info!(target_type = %target_type, "Start working on target type"); - for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) { + for (id, merged_config) in try_collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set())? { info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create"); - match self.create_target(target_type, id.clone(), &merged_config) { + let created = if defer_store_open { + with_deferred_queue_store_open(|| self.create_target(target_type, id.clone(), &merged_config)) + } else { + self.create_target(target_type, id.clone(), &merged_config) + }; + match created { Ok(target) => { info!(target_type = %target.id().name, instance_id = %id, "Create target successfully"); successful_targets.push(target); } - Err(err) => { - failed_targets += 1; - error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target"); + Err(_) => { + failures.push(format!("{target_type}/{id}: target construction failed")); + error!(target_type = %target_type, instance_id = %id, reason = "construction_failed", "Failed to create target"); } } } } - if failed_targets > 0 { + if !failures.is_empty() { warn!( created = successful_targets.len(), - failed = failed_targets, + failed = failures.len(), "Some configured targets failed to create and were skipped" ); } info!( count = successful_targets.len(), - failed = failed_targets, + failed = failures.len(), "All target processing completed" ); - Ok(successful_targets) + Ok((successful_targets, failures)) } pub async fn create_activation_from_config( diff --git a/crates/targets/src/runtime/adapter.rs b/crates/targets/src/runtime/adapter.rs index 9e7485634..87c2a23ff 100644 --- a/crates/targets/src/runtime/adapter.rs +++ b/crates/targets/src/runtime/adapter.rs @@ -13,21 +13,53 @@ // limitations under the License. use super::{ - ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, - TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker, + OpenedActivation, PrepareTargetResult, PreparedActivation, ReplayEvent, ReplayWorkerManager, RuntimeActivation, + RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, TargetActivationFailure, TargetRuntimeManager, prepare_target, + start_replay_worker, }; use crate::plugin::PluginEvent; -use crate::{Target, TargetError}; +use crate::{SharedTarget, Target, TargetError}; use async_trait::async_trait; +use rayon::prelude::*; use std::future::Future; +use std::panic::{AssertUnwindSafe, catch_unwind}; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::time::Duration; use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; type ReplayHook = Arc) -> Pin + Send>> + Send + Sync>; type ReplayStartObserver = Arc; +const MAX_PARALLEL_STORE_OPENS: usize = 4; + +static STORE_OPEN_POOL: LazyLock> = LazyLock::new(|| { + rayon::ThreadPoolBuilder::new() + .num_threads(MAX_PARALLEL_STORE_OPENS) + .thread_name(|index| format!("rustfs-target-store-open-{index}")) + .build() +}); + +enum StoreOpenOutcome +where + E: PluginEvent, +{ + Accepted(SharedTarget), + Rejected { panicked: bool, target: SharedTarget }, +} + +fn open_target_store(target: SharedTarget) -> StoreOpenOutcome +where + E: PluginEvent, +{ + match catch_unwind(AssertUnwindSafe(|| target.store().map(|store| store.open()))) { + Ok(None | Some(Ok(()))) => StoreOpenOutcome::Accepted(target), + Ok(Some(Err(_))) => StoreOpenOutcome::Rejected { panicked: false, target }, + Err(_) => StoreOpenOutcome::Rejected { panicked: true, target }, + } +} + /// Shared runtime contract for target plugins. #[async_trait] pub trait PluginRuntimeAdapter: Send + Sync @@ -96,6 +128,234 @@ where stop_log_prefix: stop_log_prefix.into(), } } + + pub async fn prepare_targets(&self, targets: Vec + Send + Sync>>) -> PreparedActivation { + self.prepare_targets_inner(targets, None).await + } + + pub async fn prepare_targets_cancellable( + &self, + targets: Vec + Send + Sync>>, + cancellation: &CancellationToken, + ) -> PreparedActivation { + self.prepare_targets_inner(targets, Some(cancellation)).await + } + + async fn prepare_targets_inner( + &self, + targets: Vec + Send + Sync>>, + cancellation: Option<&CancellationToken>, + ) -> PreparedActivation { + let mut prepared = Vec::with_capacity(targets.len()); + let mut failures = Vec::new(); + let mut rejected_targets = Vec::new(); + let mut targets = targets.into_iter(); + while let Some(target) = targets.next() { + match prepare_target(target, cancellation).await { + PrepareTargetResult::Ready(target) => prepared.push(target), + PrepareTargetResult::Degraded { error, target } => { + drop(error); + tracing::warn!( + target_id = %target.id(), + reason = "initialization_failed", + "Target initialization failed during lifecycle preparation" + ); + failures.push(TargetActivationFailure { + detail: format!("{}: initialization failed", target.id()), + }); + prepared.push(target); + } + PrepareTargetResult::Failed { error, target } => { + drop(error); + let target_id = target.id().to_string(); + tracing::warn!( + target_id, + reason = "initialization_failed", + "Target initialization failed during lifecycle preparation" + ); + failures.push(TargetActivationFailure { + detail: format!("{target_id}: initialization failed"), + }); + rejected_targets.push(Arc::from(target)); + } + PrepareTargetResult::Cancelled(target) => { + prepared.push(Arc::from(target)); + prepared.extend(targets.map(Arc::from)); + break; + } + } + } + PreparedActivation { + failures, + rejected_targets, + targets: prepared, + } + } + + /// Opens queue stores only after the previous runtime generation has been + /// quiesced. Targets whose stores cannot be opened retain the established + /// fault-isolation behavior and are returned for lock-free shutdown. + pub fn open_prepared_stores(&self, prepared: PreparedActivation) -> (OpenedActivation, PreparedActivation) { + let mut accepted = Vec::with_capacity(prepared.targets.len()); + let mut failures = prepared.failures; + let mut rejected = prepared.rejected_targets; + let outcomes = if prepared.targets.len() < 2 { + prepared.targets.into_iter().map(open_target_store).collect() + } else { + match STORE_OPEN_POOL.as_ref() { + // Vec's indexed parallel iterator preserves configuration + // order in collect, keeping failure summaries deterministic. + Ok(pool) => pool.install(|| prepared.targets.into_par_iter().map(open_target_store).collect::>()), + Err(err) => { + tracing::warn!(error = %err, "Failed to create target store open pool; opening stores serially"); + prepared.targets.into_iter().map(open_target_store).collect() + } + } + }; + for outcome in outcomes { + match outcome { + StoreOpenOutcome::Accepted(target) => accepted.push(target), + StoreOpenOutcome::Rejected { panicked, target } => { + if panicked { + tracing::error!( + target_id = %target.id(), + reason = "store_open_panicked", + "Target queue store panicked while opening during runtime handoff" + ); + } else { + tracing::error!( + target_id = %target.id(), + reason = "store_open_failed", + "Failed to open target queue store during runtime handoff" + ); + } + failures.push(TargetActivationFailure { + detail: format!("{}: queue store open failed", target.id()), + }); + rejected.push(target); + } + } + } + ( + OpenedActivation { targets: accepted }, + PreparedActivation { + failures, + rejected_targets: rejected, + targets: Vec::new(), + }, + ) + } + + pub fn try_activate_prepared(&self, opened: OpenedActivation) -> (RuntimeActivation, PreparedActivation) { + let mut replay_workers = ReplayWorkerManager::new(); + let mut accepted = Vec::with_capacity(opened.targets.len()); + let mut failures = Vec::new(); + let mut rejected_targets = Vec::new(); + for target in opened.targets { + let target_id = target.id().to_string(); + let replay = catch_unwind(AssertUnwindSafe(|| { + target.store().filter(|_| target.is_enabled()).map(|store| { + start_replay_worker( + store.boxed_clone(), + Arc::clone(&target), + Arc::clone(&self.replay_hook), + self.replay_semaphore.clone(), + self.batch_timeout, + self.idle_sleep, + ) + }) + })); + let replay = match replay { + Ok(replay) => replay, + Err(_) => { + tracing::error!( + target_id, + reason = "replay_activation_panicked", + "Target replay activation panicked during runtime handoff" + ); + failures.push(TargetActivationFailure { + detail: format!("{target_id}: replay activation failed"), + }); + rejected_targets.push(target); + continue; + } + }; + (self.replay_start_observer)(&target_id, replay.is_some()); + if let Some((cancel_tx, join)) = replay { + replay_workers.insert_with_handle(target_id, cancel_tx, join); + } + accepted.push(target); + } + + ( + RuntimeActivation { + replay_workers, + targets: accepted, + }, + PreparedActivation { + failures, + rejected_targets, + targets: Vec::new(), + }, + ) + } + + #[doc(hidden)] + pub async fn prepare_dormant_compat_activation( + &self, + targets: Vec + Send + Sync>>, + ) -> RuntimeActivation { + let PreparedActivation { + failures, + rejected_targets, + targets, + } = self.prepare_targets(targets).await; + let rejected = PreparedActivation { + failures, + rejected_targets, + targets: Vec::new(), + }; + if let Err(err) = self.close_prepared(rejected).await { + tracing::warn!(error = %err, "Failed to close targets rejected while preparing compatibility activation"); + } + RuntimeActivation { + replay_workers: ReplayWorkerManager::new(), + targets, + } + } + + #[doc(hidden)] + pub fn start_dormant_compat_activation( + &self, + activation: RuntimeActivation, + ) -> (RuntimeActivation, PreparedActivation, PreparedActivation) { + let prepared = PreparedActivation { + failures: Vec::new(), + rejected_targets: Vec::new(), + targets: activation.targets, + }; + let (opened, open_rejected) = self.open_prepared_stores(prepared); + let (activation, activation_rejected) = self.try_activate_prepared(opened); + (activation, open_rejected, activation_rejected) + } + + #[doc(hidden)] + pub async fn close_compat_activation(&self, mut activation: RuntimeActivation) -> Result<(), TargetError> { + let mut runtime = TargetRuntimeManager::new(); + for target in activation.targets { + runtime.add_arc(target); + } + self.shutdown(&mut runtime, &mut activation.replay_workers).await + } + + pub async fn close_prepared(&self, prepared: PreparedActivation) -> Result<(), TargetError> { + let mut runtime = TargetRuntimeManager::new(); + for target in prepared.targets.into_iter().chain(prepared.rejected_targets) { + runtime.add_arc(target); + } + let mut replay_workers = ReplayWorkerManager::new(); + self.shutdown(&mut runtime, &mut replay_workers).await + } } #[async_trait] @@ -104,36 +364,16 @@ where E: PluginEvent, { async fn activate_with_replay(&self, targets: Vec + Send + Sync>>) -> RuntimeActivation { - let replay_hook = Arc::clone(&self.replay_hook); - let replay_start_observer = Arc::clone(&self.replay_start_observer); - let replay_semaphore = self.replay_semaphore.clone(); - let batch_timeout = self.batch_timeout; - let idle_sleep = self.idle_sleep; - - activate_targets_with_replay(targets, move |target| { - let replay_hook = Arc::clone(&replay_hook); - let replay_start_observer = Arc::clone(&replay_start_observer); - let replay_semaphore = replay_semaphore.clone(); - - async move { - init_target_and_optionally_start_replay( - target, - move |target_id, has_replay| replay_start_observer(target_id, has_replay), - move |store, target| { - start_replay_worker( - store, - target, - Arc::clone(&replay_hook), - replay_semaphore.clone(), - batch_timeout, - idle_sleep, - ) - }, - ) - .await - } - }) - .await + let prepared = self.prepare_targets(targets).await; + let (opened, rejected) = self.open_prepared_stores(prepared); + if let Err(err) = self.close_prepared(rejected).await { + tracing::warn!(error = %err, "Failed to close targets whose queue stores could not be opened"); + } + let (activation, rejected) = self.try_activate_prepared(opened); + if let Err(err) = self.close_prepared(rejected).await { + tracing::warn!(error = %err, "Failed to close targets rejected during replay activation"); + } + activation } async fn replace_runtime_targets( @@ -189,7 +429,7 @@ where if !close_errors.is_empty() { let detail = close_errors .into_iter() - .map(|(target_id, err)| format!("{target_id}: {err}")) + .map(|(target_id, _)| target_id) .collect::>() .join("; "); return Err(TargetError::Storage(format!("Failed to close {detail}"))); @@ -200,24 +440,143 @@ where #[cfg(test)] mod tests { - use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter}; + use super::{BuiltinPluginRuntimeAdapter, MAX_PARALLEL_STORE_OPENS, PluginRuntimeAdapter}; use crate::PluginEvent; use crate::arn::TargetID; use crate::store::{Key, QueueStore, Store}; use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; use crate::{StoreError, Target, TargetError}; use async_trait::async_trait; - use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; use tempfile::tempdir; + use tokio::sync::Notify; + use tokio_util::sync::CancellationToken; + + type TestStore = dyn Store + Send + Sync; + + type BeforeOpen = Arc; + + #[derive(Clone)] + struct TestOpenStore { + before_clone: BeforeOpen, + before_open: BeforeOpen, + store: QueueStore, + } + + impl Store for TestOpenStore { + type Error = StoreError; + type Key = Key; + + fn open(&self) -> Result<(), Self::Error> { + (self.before_open)(); + self.store.open() + } + + fn put(&self, item: Arc) -> Result { + self.store.put(item) + } + + fn put_multiple(&self, items: Vec) -> Result { + self.store.put_multiple(items) + } + + fn put_raw(&self, data: &[u8]) -> Result { + self.store.put_raw(data) + } + + fn get(&self, key: &Self::Key) -> Result { + self.store.get(key) + } + + fn get_multiple(&self, key: &Self::Key) -> Result, Self::Error> { + self.store.get_multiple(key) + } + + fn get_raw(&self, key: &Self::Key) -> Result, Self::Error> { + self.store.get_raw(key) + } + + fn del(&self, key: &Self::Key) -> Result<(), Self::Error> { + self.store.del(key) + } + + fn delete(&self) -> Result<(), Self::Error> { + self.store.delete() + } + + fn list(&self) -> Vec { + self.store.list() + } + + fn len(&self) -> usize { + self.store.len() + } + + fn is_empty(&self) -> bool { + self.store.is_empty() + } + + fn boxed_clone(&self) -> Box + Send + Sync> { + (self.before_clone)(); + Box::new(self.clone()) + } + } + + #[derive(Default)] + struct StoreOpenGate { + changed: Condvar, + state: Mutex, + } + + #[derive(Default)] + struct StoreOpenGateState { + active: usize, + max_active: usize, + released: bool, + } + + impl StoreOpenGate { + fn enter(&self) { + let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner()); + state.active += 1; + state.max_active = state.max_active.max(state.active); + self.changed.notify_all(); + while !state.released { + state = self.changed.wait(state).unwrap_or_else(|err| err.into_inner()); + } + state.active -= 1; + } + + fn wait_for_active(&self, expected: usize, timeout: Duration) -> bool { + let state = self.state.lock().unwrap_or_else(|err| err.into_inner()); + let (state, _) = self + .changed + .wait_timeout_while(state, timeout, |state| state.max_active < expected) + .unwrap_or_else(|err| err.into_inner()); + state.max_active >= expected + } + + fn release(&self) { + let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner()); + state.released = true; + self.changed.notify_all(); + } + + fn max_active(&self) -> usize { + self.state.lock().unwrap_or_else(|err| err.into_inner()).max_active + } + } #[derive(Clone)] struct TestTarget { close_calls: Arc, id: TargetID, + init_calls: Arc, + init_entered: Option>, init_fails: bool, - store: Option>, + store: Option>, } impl TestTarget { @@ -225,6 +584,8 @@ mod tests { Self { close_calls: Arc::new(AtomicUsize::new(0)), id: TargetID::new(id.to_string(), name.to_string()), + init_calls: Arc::new(AtomicUsize::new(0)), + init_entered: None, init_fails: false, store: None, } @@ -235,11 +596,16 @@ mod tests { self } + fn with_pending_init(mut self, init_entered: Arc) -> Self { + self.init_entered = Some(init_entered); + self + } + fn with_store(mut self) -> Self { let dir = tempdir().expect("tempdir should be created for queue store tests"); let store = QueueStore::::new(dir.path(), 16, ".queue"); store.open().expect("queue store should open"); - self.store = Some(store); + self.store = Some(Arc::new(store)); self } } @@ -271,9 +637,7 @@ mod tests { } fn store(&self) -> Option<&(dyn Store + Send + Sync)> { - self.store - .as_ref() - .map(|store| store as &(dyn Store + Send + Sync)) + self.store.as_deref() } fn clone_dyn(&self) -> Box + Send + Sync> { @@ -281,6 +645,11 @@ mod tests { } async fn init(&self) -> Result<(), TargetError> { + self.init_calls.fetch_add(1, Ordering::SeqCst); + if let Some(init_entered) = &self.init_entered { + init_entered.notify_one(); + return std::future::pending().await; + } if self.init_fails { return Err(TargetError::Configuration("forced init failure".to_string())); } @@ -334,6 +703,212 @@ mod tests { assert_eq!(activation.replay_workers.len(), 1); } + #[tokio::test] + async fn prepared_store_target_reports_init_failure_without_dropping_queue_runtime() { + let adapter = builtin_adapter(); + let target = TestTarget::new("primary", "webhook").with_failed_init().with_store(); + + let prepared = adapter.prepare_targets(vec![Box::new(target)]).await; + assert_eq!(prepared.targets.len(), 1); + assert!( + prepared + .failure_summary() + .is_some_and(|summary| summary.contains("initialization failed") && !summary.contains("forced init failure")) + ); + + let (opened, rejected) = adapter.open_prepared_stores(prepared); + assert!(rejected.failure_summary().is_some()); + let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened); + assert!(activation_rejected.failure_summary().is_none()); + assert_eq!(activation.targets.len(), 1); + assert_eq!(activation.replay_workers.len(), 1); + activation.replay_workers.stop_all("stop degraded target replay worker").await; + } + + #[tokio::test] + async fn cancellable_preparation_returns_current_and_remaining_targets_for_shutdown() { + let adapter = builtin_adapter(); + let init_entered = Arc::new(Notify::new()); + let first = TestTarget::new("first", "webhook").with_pending_init(init_entered.clone()); + let first_close_calls = first.close_calls.clone(); + let second = TestTarget::new("second", "webhook"); + let second_close_calls = second.close_calls.clone(); + let second_init_calls = second.init_calls.clone(); + let cancellation = CancellationToken::new(); + let prepare_adapter = adapter.clone(); + let prepare_cancellation = cancellation.clone(); + let prepare = tokio::spawn(async move { + prepare_adapter + .prepare_targets_cancellable(vec![Box::new(first), Box::new(second)], &prepare_cancellation) + .await + }); + + init_entered.notified().await; + cancellation.cancel(); + let prepared = tokio::time::timeout(Duration::from_secs(1), prepare) + .await + .expect("cancellation should interrupt target initialization") + .expect("preparation task should finish"); + assert_eq!(prepared.targets.len(), 2); + + adapter + .close_prepared(prepared) + .await + .expect("cancelled targets should close"); + assert_eq!(first_close_calls.load(Ordering::SeqCst), 1); + assert_eq!(second_close_calls.load(Ordering::SeqCst), 1); + assert_eq!(second_init_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn prepared_activation_opens_store_before_starting_replay() { + let adapter = builtin_adapter(); + let dir = tempdir().expect("tempdir should be created"); + let queue_path = dir.path().join("queue"); + let mut target = TestTarget::new("primary", "webhook"); + target.store = Some(Arc::new(QueueStore::::new(&queue_path, 16, ".queue"))); + + let prepared = adapter.prepare_targets(vec![Box::new(target)]).await; + assert!(!queue_path.exists(), "dormant preparation must not open the queue store"); + + let (opened, rejected) = adapter.open_prepared_stores(prepared); + assert!(rejected.targets.is_empty()); + assert!(queue_path.is_dir(), "handoff must open the queue store before activation"); + let (mut activation, activation_rejected) = adapter.try_activate_prepared(opened); + assert!(activation_rejected.failure_summary().is_none()); + assert_eq!(activation.replay_workers.len(), 1); + activation + .replay_workers + .stop_all("stop prepared activation test worker") + .await; + } + + #[tokio::test] + async fn activation_closes_a_target_when_its_queue_store_cannot_open() { + let adapter = builtin_adapter(); + let dir = tempdir().expect("tempdir should be created"); + let invalid_base = dir.path().join("not-a-directory"); + std::fs::write(&invalid_base, b"file").expect("invalid queue base should be created"); + let mut target = TestTarget::new("primary", "webhook"); + let close_calls = target.close_calls.clone(); + target.store = Some(Arc::new(QueueStore::::new(&invalid_base, 16, ".queue"))); + + let activation = adapter.activate_with_replay(vec![Box::new(target)]).await; + + assert!(activation.targets.is_empty()); + assert!(activation.replay_workers.is_empty()); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn prepared_stores_open_with_bounded_parallelism_and_stable_order() { + const TARGETS: usize = MAX_PARALLEL_STORE_OPENS * 2; + + let adapter = builtin_adapter(); + let dir = tempdir().expect("tempdir should be created"); + let gate = Arc::new(StoreOpenGate::default()); + let mut targets: Vec + Send + Sync>> = Vec::with_capacity(TARGETS); + let mut expected_ids = Vec::with_capacity(TARGETS); + for index in 0..TARGETS { + let mut target = TestTarget::new(&format!("target-{index}"), "webhook"); + expected_ids.push(target.id.to_string()); + let open_gate = gate.clone(); + target.store = Some(Arc::new(TestOpenStore { + before_clone: Arc::new(|| {}), + before_open: Arc::new(move || open_gate.enter()), + store: QueueStore::new(dir.path().join(index.to_string()), 16, ".queue"), + })); + targets.push(Box::new(target)); + } + + let prepared = adapter.prepare_targets(targets).await; + let open_adapter = adapter.clone(); + let opening = tokio::task::spawn_blocking(move || open_adapter.open_prepared_stores(prepared)); + let wait_gate = gate.clone(); + let reached_bound = + tokio::task::spawn_blocking(move || wait_gate.wait_for_active(MAX_PARALLEL_STORE_OPENS, Duration::from_secs(30))) + .await + .expect("store-open observer should not panic"); + gate.release(); + let (opened, rejected) = opening.await.expect("bounded store opens should not panic"); + let opened_ids = opened + .targets + .iter() + .map(|target| target.id().to_string()) + .collect::>(); + + assert!(reached_bound, "store opens did not use the configured parallelism"); + assert_eq!(gate.max_active(), MAX_PARALLEL_STORE_OPENS); + assert_eq!(opened_ids, expected_ids, "parallel store opens must preserve configuration order"); + assert!(rejected.targets.is_empty()); + assert!(rejected.failure_summary().is_none()); + } + + #[tokio::test] + async fn panicking_store_open_rejects_and_closes_only_that_target() { + let adapter = builtin_adapter(); + let dir = tempdir().expect("tempdir should be created"); + let mut target = TestTarget::new("panicking", "webhook"); + let close_calls = target.close_calls.clone(); + target.store = Some(Arc::new(TestOpenStore { + before_clone: Arc::new(|| {}), + before_open: Arc::new(|| panic!("forced store open panic: do-not-expose-payload")), + store: QueueStore::new(dir.path(), 16, ".queue"), + })); + + let prepared = adapter.prepare_targets(vec![Box::new(target)]).await; + let (opened, rejected) = adapter.open_prepared_stores(prepared); + let summary = rejected + .failure_summary() + .expect("panicking store should report a generic activation failure"); + let (activation, activation_rejected) = adapter.try_activate_prepared(opened); + + assert!(activation.targets.is_empty(), "a target without an open store must not become visible"); + assert!( + activation.replay_workers.is_empty(), + "a rejected target must not publish without a replay worker" + ); + assert!(activation_rejected.failure_summary().is_none()); + assert!(summary.contains("queue store open failed")); + assert!(!summary.contains("do-not-expose-payload")); + adapter + .close_prepared(rejected) + .await + .expect("a target rejected after a store panic should close"); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn panicking_store_clone_cannot_publish_target_without_replay_worker() { + let adapter = builtin_adapter(); + let dir = tempdir().expect("tempdir should be created"); + let mut target = TestTarget::new("panicking-clone", "webhook"); + let close_calls = target.close_calls.clone(); + target.store = Some(Arc::new(TestOpenStore { + before_clone: Arc::new(|| panic!("forced store clone panic: do-not-expose-payload")), + before_open: Arc::new(|| {}), + store: QueueStore::new(dir.path(), 16, ".queue"), + })); + + let prepared = adapter.prepare_targets(vec![Box::new(target)]).await; + let (opened, open_rejected) = adapter.open_prepared_stores(prepared); + assert!(open_rejected.failure_summary().is_none()); + let (activation, rejected) = adapter.try_activate_prepared(opened); + let summary = rejected + .failure_summary() + .expect("panicking store clone should report a generic activation failure"); + + assert!(activation.targets.is_empty(), "a target without a replay worker must not become visible"); + assert!(activation.replay_workers.is_empty()); + assert!(summary.contains("replay activation failed")); + assert!(!summary.contains("do-not-expose-payload")); + adapter + .close_prepared(rejected) + .await + .expect("a target rejected during replay activation should close"); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() { let adapter = builtin_adapter(); diff --git a/crates/targets/src/runtime/mod.rs b/crates/targets/src/runtime/mod.rs index 4f83740eb..bfad78dd1 100644 --- a/crates/targets/src/runtime/mod.rs +++ b/crates/targets/src/runtime/mod.rs @@ -27,11 +27,21 @@ use crate::store::{Key, Store, ensure_store_entry_raw_readable}; use crate::target::QueuedPayload; use crate::target::TargetDeliverySnapshot; use crate::{StoreError, TargetError}; +use futures_util::stream::{FuturesUnordered, StreamExt}; use std::sync::Arc; use std::{collections::HashMap, fmt::Debug}; use std::{future::Future, pin::Pin, time::Duration}; use tokio::sync::{Semaphore, mpsc}; use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str { + if error.is_cancelled() { + "join_cancelled" + } else { + "join_panicked" + } +} /// Maximum number of replay attempts before a stored entry is exhausted. Each attempt runs one full /// send (one ack wait at the configured timeout for a JetStream entry), then a backoff sleep before @@ -68,11 +78,21 @@ pub(crate) fn inter_attempt_backoff_sum(attempts: usize) -> Duration { pub type SharedTarget = Arc + Send + Sync>; type ReplayHook = Arc) -> Pin + Send>> + Send + Sync>; -/// Upper bound on how long [`ReplayWorkerManager::stop_all`] waits for a single -/// replay worker to observe its cancel signal and exit before it is forcibly -/// aborted. Workers observe cancellation promptly (including during retry -/// backoff), so this only guards against a wedged task. -const STOP_JOIN_TIMEOUT: Duration = Duration::from_secs(5); +pub(crate) enum PrepareTargetResult +where + E: PluginEvent, +{ + Ready(SharedTarget), + Degraded { + error: TargetError, + target: SharedTarget, + }, + Failed { + error: TargetError, + target: Box + Send + Sync>, + }, + Cancelled(Box + Send + Sync>), +} /// Tracks a running replay worker: its cancel channel and, when the worker was /// spawned in-process, the [`JoinHandle`] used to await its exit on shutdown. @@ -113,7 +133,7 @@ impl ReplayWorkerManager { } /// Registers a cancel channel together with the worker's join handle so - /// `stop_all` can await the worker's exit (bounded by [`STOP_JOIN_TIMEOUT`]). + /// `stop_all` can await the worker's exit. pub fn insert_with_handle(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>, join: JoinHandle<()>) { self.cancellers.insert( target_id, @@ -140,37 +160,36 @@ impl ReplayWorkerManager { } /// Stops every replay worker: it first signals cancellation to all of them, - /// then awaits each worker's exit (bounded by [`STOP_JOIN_TIMEOUT`], after - /// which the task is aborted). Signalling before joining lets all workers - /// wind down concurrently, and joining guarantees no worker keeps draining - /// the shared store after this returns — preventing duplicate delivery and - /// orphaned tasks across reloads and shutdown. + /// then strictly awaits each worker's exit. A worker already awaiting a + /// delivery acknowledgement is allowed to finish; aborting it at an + /// arbitrary deadline could leave an acknowledged queue entry undeleted and + /// make the replacement worker deliver it again. Signalling before joining + /// lets all workers wind down concurrently. Legacy joinless registrations + /// can only be signalled. pub async fn stop_all(&mut self, log_prefix: &str) { - let mut handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect(); + let handles: Vec<(String, ReplayWorkerHandle)> = self.cancellers.drain().collect(); + let mut joins = std::collections::VecDeque::new(); // Phase 1: signal cancellation to all workers. - for (target_id, handle) in &handles { + for (target_id, handle) in handles { tracing::info!(target_id = %target_id, "{log_prefix}"); - let _ = handle.cancel_tx.send(()).await; + let _ = handle.cancel_tx.try_send(()); + if let Some(join) = handle.join { + joins.push_back((target_id, join)); + } else { + tracing::warn!( + target_id = %target_id, + "Replay worker has no join handle; cancellation was signalled but exit cannot be verified" + ); + } } - // Phase 2: await each worker's exit, forcibly aborting any that overrun. - for (target_id, handle) in handles.drain(..) { - let Some(mut join) = handle.join else { - continue; - }; - match tokio::time::timeout(STOP_JOIN_TIMEOUT, &mut join).await { - Ok(Ok(())) => {} - Ok(Err(err)) => { - tracing::warn!(target_id = %target_id, error = %err, "Replay worker terminated abnormally"); - } - Err(_) => { - join.abort(); - tracing::warn!( - target_id = %target_id, - "Timed out awaiting replay worker exit; task aborted" - ); - } + // Phase 2: strict join. Delivery operations own their own protocol + // deadlines; lifecycle must not invent a shorter deadline that turns an + // acknowledgement race into duplicate delivery. + while let Some((target_id, join)) = joins.pop_front() { + if let Err(err) = join.await { + tracing::warn!(target_id = %target_id, reason = join_failure_reason(&err), "Replay worker terminated abnormally"); } } } @@ -184,6 +203,55 @@ where pub targets: Vec>, } +/// Targets whose persistent queue stores are open and are ready to start +/// replay. This distinct stage prevents activation from skipping store open. +pub struct OpenedActivation +where + E: PluginEvent, +{ + pub(crate) targets: Vec>, +} + +struct TargetActivationFailure { + detail: String, +} + +/// Targets that have completed initialization but have not started replay +/// workers yet. Keeping preparation dormant lets lifecycle orchestration stop +/// the previous workers before the replacement workers are spawned. +pub struct PreparedActivation +where + E: PluginEvent, +{ + failures: Vec, + rejected_targets: Vec>, + pub(crate) targets: Vec>, +} + +impl PreparedActivation +where + E: PluginEvent, +{ + pub fn failure_summary(&self) -> Option { + if self.failures.is_empty() { + return None; + } + + Some( + self.failures + .iter() + .map(|failure| failure.detail.clone()) + .collect::>() + .join("; "), + ) + } + + pub fn extend_creation_failures(&mut self, failures: impl IntoIterator) { + self.failures + .extend(failures.into_iter().map(|detail| TargetActivationFailure { detail })); + } +} + #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct RuntimeStatusSnapshot { pub replay_worker_count: usize, @@ -339,17 +407,19 @@ where /// Surfacing them lets a caller fail an explicit shutdown while still tearing down the rest of the /// runtime. pub async fn clear_and_close(&mut self) -> Vec<(String, TargetError)> { - let target_ids: Vec = self.targets.keys().cloned().collect(); + let targets = std::mem::take(&mut self.targets); + let mut closes = FuturesUnordered::new(); + for (target_id, target) in targets { + closes.push(async move { (target_id, target.close().await) }); + } + let mut errors = Vec::new(); - for target_id in target_ids { - if let Some(target) = self.targets.remove(&target_id) - && let Err(err) = target.close().await - { + while let Some((target_id, result)) = closes.next().await { + if let Err(err) = result { tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown"); errors.push((target_id, err)); } } - self.targets.clear(); errors } @@ -440,21 +510,16 @@ where SharedTarget, ) -> (mpsc::Sender<()>, JoinHandle<()>), { - let target_id = target.id().to_string(); - let has_store = target.store().is_some(); - - if let Err(err) = target.init().await { - tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target"); - if !has_store { + let shared = match prepare_target(target, None).await { + PrepareTargetResult::Ready(target) => target, + PrepareTargetResult::Degraded { target, .. } => target, + PrepareTargetResult::Failed { target, .. } => { + let _ = target.close().await; return None; } - tracing::warn!( - target_id = %target_id, - "Proceeding with store-backed target despite init failure" - ); - } - - let shared: SharedTarget = Arc::from(target); + PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"), + }; + let target_id = shared.id().to_string(); if !shared.is_enabled() { on_replay_start(&target_id, false); return Some((shared, None)); @@ -467,6 +532,45 @@ where Some((shared, cancel)) } +pub(crate) async fn prepare_target( + target: Box + Send + Sync>, + cancellation: Option<&CancellationToken>, +) -> PrepareTargetResult +where + E: PluginEvent, +{ + let target_id = target.id().to_string(); + let has_store = target.store().is_some(); + + let init_result = match cancellation { + Some(cancellation) => { + tokio::select! { + biased; + _ = cancellation.cancelled() => return PrepareTargetResult::Cancelled(target), + result = target.init() => result, + } + } + None => target.init().await, + }; + + if let Err(err) = init_result { + tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target"); + if !has_store { + return PrepareTargetResult::Failed { error: err, target }; + } + tracing::warn!( + target_id = %target_id, + "Proceeding with store-backed target despite init failure" + ); + return PrepareTargetResult::Degraded { + error: err, + target: Arc::from(target), + }; + } + + PrepareTargetResult::Ready(Arc::from(target)) +} + type ActivatedTarget = (SharedTarget, Option<(mpsc::Sender<()>, JoinHandle<()>)>); pub async fn activate_targets_with_replay( @@ -584,7 +688,11 @@ async fn stream_replay_worker( } Ok(Ok(_)) => {} Err(join_err) => { - tracing::warn!(target_id = %target.id(), error = %join_err, "The failed-events maintenance task failed to join"); + tracing::warn!( + target_id = %target.id(), + reason = join_failure_reason(&join_err), + "The failed-events maintenance task failed to join" + ); } } last_prune = tokio::time::Instant::now(); @@ -835,7 +943,8 @@ mod tests { use crate::{Target, TargetError}; use async_trait::async_trait; use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use tokio::sync::{Notify, Semaphore}; #[tokio::test(start_paused = true)] async fn seed_interval_start_backdates_by_one_interval() { @@ -900,14 +1009,20 @@ mod tests { #[derive(Clone)] struct TestTarget { id: TargetID, + block_on_close: Arc, + close_gate: Arc, close_calls: Arc, + close_started: Arc, } impl TestTarget { fn new(id: &str, name: &str) -> Self { Self { id: TargetID::new(id.to_string(), name.to_string()), + block_on_close: Arc::new(AtomicBool::new(false)), + close_gate: Arc::new(Semaphore::new(0)), close_calls: Arc::new(AtomicUsize::new(0)), + close_started: Arc::new(Notify::new()), } } } @@ -935,6 +1050,10 @@ mod tests { async fn close(&self) -> Result<(), TargetError> { self.close_calls.fetch_add(1, Ordering::SeqCst); + self.close_started.notify_one(); + if self.block_on_close.load(Ordering::SeqCst) { + let _permit = self.close_gate.acquire().await.expect("close gate should remain open"); + } Ok(()) } @@ -966,6 +1085,45 @@ mod tests { assert_eq!(close_calls.load(Ordering::SeqCst), 1); } + #[tokio::test(start_paused = true)] + async fn runtime_manager_starts_all_target_closes_before_waiting_for_completion() { + let mut manager = TargetRuntimeManager::::new(); + let first = TestTarget::new("first", "webhook"); + let second = TestTarget::new("second", "webhook"); + let first_observer = first.clone(); + let second_observer = second.clone(); + manager.add_boxed(Box::new(first)); + manager.add_boxed(Box::new(second)); + + let first_close_key = manager + .keys() + .into_iter() + .next() + .expect("two targets should have a first close key"); + let (blocked, unblocked) = if first_close_key == first_observer.id.to_string() { + (first_observer, second_observer) + } else { + (second_observer, first_observer) + }; + blocked.block_on_close.store(true, Ordering::SeqCst); + + let close_task = tokio::spawn(async move { manager.clear_and_close().await }); + + tokio::time::timeout(std::time::Duration::from_secs(1), blocked.close_started.notified()) + .await + .expect("the first target close should start"); + tokio::time::timeout(std::time::Duration::from_secs(1), unblocked.close_started.notified()) + .await + .expect("a blocked first close must not prevent the second close from starting"); + assert!(!close_task.is_finished(), "clear_and_close must still await the blocked target"); + + blocked.close_gate.add_permits(1); + let errors = close_task.await.expect("clear_and_close task should join"); + assert!(errors.is_empty()); + assert_eq!(blocked.close_calls.load(Ordering::SeqCst), 1); + assert_eq!(unblocked.close_calls.load(Ordering::SeqCst), 1); + } + #[test] fn runtime_manager_snapshots_targets() { let mut manager = TargetRuntimeManager::::new(); @@ -1030,6 +1188,51 @@ mod tests { assert!(exited.load(Ordering::SeqCst), "stop_all must await the worker to completion"); } + #[tokio::test(start_paused = true)] + async fn stop_all_does_not_abort_delivery_awaiting_acknowledgement() { + use super::ReplayWorkerManager; + use std::sync::atomic::{AtomicBool, Ordering}; + use tokio::sync::Notify; + + let mut manager = ReplayWorkerManager::new(); + let acknowledgement = Arc::new(Notify::new()); + let worker_started = Arc::new(Notify::new()); + let exited = Arc::new(AtomicBool::new(false)); + let (cancel_tx, mut cancel_rx) = tokio::sync::mpsc::channel::<()>(1); + let worker_acknowledgement = acknowledgement.clone(); + let worker_started_signal = worker_started.clone(); + let worker_exited = exited.clone(); + let join = tokio::spawn(async move { + worker_started_signal.notify_one(); + let _ = cancel_rx.recv().await; + // Model a protocol operation that has accepted the request but has + // not returned its acknowledgement yet. Lifecycle must not abort + // this future or the same durable entry can be sent twice. + worker_acknowledgement.notified().await; + worker_exited.store(true, Ordering::SeqCst); + }); + manager.insert_with_handle("primary:webhook".to_string(), cancel_tx, join); + worker_started.notified().await; + + let mut stop = Box::pin(manager.stop_all("stopping ack-pending test worker")); + tokio::select! { + biased; + _ = &mut stop => panic!("stop_all returned before the pending acknowledgement"), + _ = std::future::ready(()) => {} + } + tokio::time::advance(std::time::Duration::from_secs(60)).await; + tokio::select! { + biased; + _ = &mut stop => panic!("stop_all aborted an acknowledgement-pending delivery"), + _ = std::future::ready(()) => {} + } + + acknowledgement.notify_one(); + stop.await; + assert!(exited.load(Ordering::SeqCst)); + assert!(manager.is_empty()); + } + mod classifier { use super::super::{ReplayEvent, stream_replay_worker}; use crate::arn::TargetID; diff --git a/crates/targets/src/target/kafka.rs b/crates/targets/src/target/kafka.rs index 9b087f349..7888fd13b 100644 --- a/crates/targets/src/target/kafka.rs +++ b/crates/targets/src/target/kafka.rs @@ -24,7 +24,7 @@ use crate::{ store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, invalidate_cache_on_connectivity_error, + TargetTlsState, TargetType, build_queued_payload, build_target_tls_fingerprint, is_connectivity_error, open_target_queue_store, persist_queued_payload_to_store, }, }; @@ -32,13 +32,84 @@ use async_trait::async_trait; use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError, KafkaCode}; use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SaslConfig, SecurityConfig}; use rustfs_tls_runtime::{load_cert_bundle_der_bytes, load_private_key}; -use std::{fmt, marker::PhantomData, sync::Arc, time::Duration}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{fmt, future::Future, marker::PhantomData, sync::Arc, time::Duration}; use tokio::sync::Mutex; use tracing::{debug, error, info, instrument, warn}; pub(crate) const KAFKA_SASL_PLAIN: &str = "PLAIN"; pub(crate) const KAFKA_SASL_SCRAM_SHA_256: &str = "SCRAM-SHA-256"; pub(crate) const KAFKA_SASL_SCRAM_SHA_512: &str = "SCRAM-SHA-512"; +const KAFKA_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30); + +struct KafkaDeliveryAttempt<'a> { + armed: bool, + poisoned: &'a AtomicBool, +} + +impl KafkaDeliveryAttempt<'_> { + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for KafkaDeliveryAttempt<'_> { + fn drop(&mut self) { + if self.armed { + self.poisoned.store(true, Ordering::Release); + } + } +} + +fn kafka_delivery_timeout() -> TargetError { + TargetError::Timeout(format!("Kafka delivery timed out after {KAFKA_DELIVERY_TIMEOUT:?}")) +} + +async fn with_serialized_kafka_delivery( + delivery_lock: &Mutex<()>, + delivery_poisoned: &AtomicBool, + select_producer: Select, + deliver: Deliver, + invalidate: Invalidate, +) -> Result +where + P: Send, + T: Send, + Select: FnOnce() -> SelectFuture + Send, + SelectFuture: Future> + Send, + Deliver: FnOnce(P) -> DeliveryFuture + Send, + DeliveryFuture: Future> + Send, + Invalidate: Fn() -> InvalidateFuture + Send, + InvalidateFuture: Future + Send, +{ + let deadline = tokio::time::Instant::now() + KAFKA_DELIVERY_TIMEOUT; + let _delivery_guard = tokio::time::timeout_at(deadline, delivery_lock.lock()) + .await + .map_err(|_| kafka_delivery_timeout())?; + let mut attempt = KafkaDeliveryAttempt { + armed: true, + poisoned: delivery_poisoned, + }; + + if delivery_poisoned.load(Ordering::Acquire) { + tokio::time::timeout_at(deadline, invalidate()) + .await + .map_err(|_| kafka_delivery_timeout())?; + delivery_poisoned.store(false, Ordering::Release); + } + + let result = tokio::time::timeout_at(deadline, async { deliver(select_producer().await?).await }) + .await + .map_err(|_| kafka_delivery_timeout())?; + if result.as_ref().is_err_and(is_connectivity_error) { + tokio::time::timeout_at(deadline, invalidate()) + .await + .map_err(|_| kafka_delivery_timeout())?; + delivery_poisoned.store(false, Ordering::Release); + } + attempt.disarm(); + result +} /// Arguments for configuring a Kafka target #[derive(Clone)] @@ -233,6 +304,8 @@ where args: KafkaArgs, store: Option + Send + Sync>>, producer: Arc>>>, + delivery_lock: Arc>, + delivery_poisoned: Arc, tls_state: Arc>, /// Adapter that bridges this target to the TLS reload coordinator. /// When `Some`, the target uses coordinator-managed material; when `None`, @@ -291,6 +364,8 @@ where args, store: queue_store, producer: Arc::new(Mutex::new(None)), + delivery_lock: Arc::new(Mutex::new(())), + delivery_poisoned: Arc::new(AtomicBool::new(false)), tls_state: Arc::new(Mutex::new(TargetTlsState::default())), tls_adapter: None, delivery_counters: Arc::new(TargetDeliveryCounters::default()), @@ -307,7 +382,7 @@ where }; let mut config = AsyncProducerConfig::new() - .with_ack_timeout(Duration::from_secs(30)) + .with_ack_timeout(KAFKA_DELIVERY_TIMEOUT) .with_required_acks(acks); if let Some(security) = self.args.security_config(true)? { @@ -388,20 +463,26 @@ where "Sending Kafka payload" ); - let producer = self.get_or_build_producer().await?; - - // Use "/" as the message key so all events for the same - // object hash to the same partition and preserve per-object ordering - // across multiple partitions (backlog#983). - let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name); - if let Err(err) = producer - .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice())) - .await - { - let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka"); - invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await; - return Err(mapped); - } + // rustfs-kafka-async does not validate response correlation IDs. Keep + // producer selection, send, and timeout invalidation serialized so a + // waiter cannot reuse a connection with an unread timed-out response. + with_serialized_kafka_delivery( + &self.delivery_lock, + &self.delivery_poisoned, + || self.get_or_build_producer(), + |producer| async move { + // Use "/" as the message key so all events for the same + // object hash to the same partition and preserve per-object ordering + // across multiple partitions (backlog#983). + let partition_key = format!("{}/{}", meta.bucket_name, meta.object_name); + producer + .send(&Record::from_key_value(&self.args.topic, partition_key, body.as_slice())) + .await + .map_err(|err| Self::map_kafka_error(err, "Failed to send message to Kafka")) + }, + || self.invalidate_cached_producer(), + ) + .await?; debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic"); self.delivery_counters.record_success(); @@ -415,6 +496,8 @@ where args: self.args.clone(), store: self.store.as_ref().map(|s| s.boxed_clone()), producer: Arc::clone(&self.producer), + delivery_lock: Arc::clone(&self.delivery_lock), + delivery_poisoned: Arc::clone(&self.delivery_poisoned), tls_state: Arc::clone(&self.tls_state), tls_adapter: self.tls_adapter.clone(), delivery_counters: Arc::clone(&self.delivery_counters), @@ -559,6 +642,8 @@ where #[cfg(test)] mod tests { use super::*; + use std::sync::atomic::AtomicUsize; + use tokio::sync::Notify; fn base_args() -> KafkaArgs { KafkaArgs { @@ -580,6 +665,158 @@ mod tests { } } + #[tokio::test(start_paused = true)] + async fn timeout_invalidates_before_the_next_delivery_selects_a_producer() { + let delivery_lock = Arc::new(Mutex::new(())); + let delivery_poisoned = Arc::new(AtomicBool::new(false)); + let generation = Arc::new(AtomicUsize::new(1)); + let first_entered = Arc::new(Notify::new()); + + let first = { + let delivery_lock = Arc::clone(&delivery_lock); + let delivery_poisoned = Arc::clone(&delivery_poisoned); + let generation = Arc::clone(&generation); + let first_entered = Arc::clone(&first_entered); + tokio::spawn(async move { + with_serialized_kafka_delivery( + &delivery_lock, + &delivery_poisoned, + { + let generation = Arc::clone(&generation); + move || async move { Ok(generation.load(Ordering::SeqCst)) } + }, + move |selected| async move { + assert_eq!(selected, 1); + first_entered.notify_one(); + std::future::pending::>().await + }, + move || { + let generation = Arc::clone(&generation); + async move { generation.store(2, Ordering::SeqCst) } + }, + ) + .await + }) + }; + + first_entered.notified().await; + tokio::time::advance(Duration::from_secs(1)).await; + let second = { + let delivery_lock = Arc::clone(&delivery_lock); + let delivery_poisoned = Arc::clone(&delivery_poisoned); + let generation = Arc::clone(&generation); + tokio::spawn(async move { + with_serialized_kafka_delivery( + &delivery_lock, + &delivery_poisoned, + { + let generation = Arc::clone(&generation); + move || async move { Ok(generation.load(Ordering::SeqCst)) } + }, + |selected| async move { Ok(selected) }, + move || { + let generation = Arc::clone(&generation); + async move { generation.store(2, Ordering::SeqCst) } + }, + ) + .await + }) + }; + + assert!(matches!( + first.await.expect("first delivery task should not panic"), + Err(TargetError::Timeout(_)) + )); + assert_eq!( + second + .await + .expect("second delivery task should not panic") + .expect("second delivery should succeed"), + 2, + "the waiter must select a fresh producer generation after timeout invalidation" + ); + } + + #[tokio::test(start_paused = true)] + async fn delivery_deadline_includes_waiting_for_the_serialization_lock() { + let delivery_lock = Arc::new(Mutex::new(())); + let delivery_poisoned = AtomicBool::new(false); + let selected = Arc::new(AtomicBool::new(false)); + let _held = delivery_lock.lock().await; + + let error = with_serialized_kafka_delivery( + &delivery_lock, + &delivery_poisoned, + { + let selected = Arc::clone(&selected); + move || async move { + selected.store(true, Ordering::SeqCst); + Ok(()) + } + }, + |()| async { Ok(()) }, + || async {}, + ) + .await + .expect_err("lock admission must share the absolute delivery deadline"); + + assert!(matches!(error, TargetError::Timeout(_))); + assert!(!selected.load(Ordering::SeqCst), "a timed-out waiter must not select a producer"); + assert!(!delivery_poisoned.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn cancelled_delivery_poisons_the_connection_before_the_next_selection() { + let delivery_lock = Arc::new(Mutex::new(())); + let delivery_poisoned = Arc::new(AtomicBool::new(false)); + let generation = Arc::new(AtomicUsize::new(1)); + let first_entered = Arc::new(Notify::new()); + let first = { + let delivery_lock = Arc::clone(&delivery_lock); + let delivery_poisoned = Arc::clone(&delivery_poisoned); + let first_entered = Arc::clone(&first_entered); + tokio::spawn(async move { + with_serialized_kafka_delivery( + &delivery_lock, + &delivery_poisoned, + || async { Ok(1usize) }, + move |_| async move { + first_entered.notify_one(); + std::future::pending::>().await + }, + || async {}, + ) + .await + }) + }; + first_entered.notified().await; + first.abort(); + assert!(first.await.expect_err("first delivery should be cancelled").is_cancelled()); + assert!(delivery_poisoned.load(Ordering::Acquire)); + + let selected = with_serialized_kafka_delivery( + &delivery_lock, + &delivery_poisoned, + { + let generation = Arc::clone(&generation); + move || async move { Ok(generation.load(Ordering::SeqCst)) } + }, + |selected| async move { Ok(selected) }, + { + let generation = Arc::clone(&generation); + move || { + let generation = Arc::clone(&generation); + async move { generation.store(2, Ordering::SeqCst) } + } + }, + ) + .await + .expect("the next delivery should recover from cancellation poisoning"); + + assert_eq!(selected, 2); + assert!(!delivery_poisoned.load(Ordering::Acquire)); + } + #[test] fn test_validate_empty_brokers() { let args = KafkaArgs { diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index d3635b666..9e3e2abe1 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -19,12 +19,14 @@ use crate::{StoreError, TargetError, TargetLog}; use async_trait::async_trait; use rustfs_s3_types::EventName; use serde::{Deserialize, Serialize}; +use std::cell::Cell; use std::fmt::Formatter; use std::future::Future; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::thread_local; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::{debug, warn}; pub mod amqp; @@ -111,7 +113,12 @@ where /// Checks if the target is active and reachable async fn is_active(&self) -> Result; - /// Saves an event (either sends it immediately or stores it for later) + /// Saves an event (either sends it immediately or stores it for later). + /// + /// A target whose [`Self::store`] returns `Some` must only persist the event + /// here; network delivery belongs to its replay worker. Runtime lifecycle + /// handoff drains these durable enqueues while allowing a direct network + /// send to finish against a detached target. async fn save(&self, event: Arc>) -> Result<(), TargetError>; /// Sends an event from the store using the queued raw body and metadata. @@ -605,6 +612,24 @@ pub(crate) fn open_target_queue_store( Ok(store.map(|store| Box::new(store) as BoxedQueuedStore)) } +thread_local! { + static DEFER_QUEUE_STORE_OPEN: Cell = const { Cell::new(false) }; +} + +pub(crate) fn with_deferred_queue_store_open(operation: impl FnOnce() -> T) -> T { + struct Reset(bool); + + impl Drop for Reset { + fn drop(&mut self) { + DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.set(self.0)); + } + } + + let previous = DEFER_QUEUE_STORE_OPEN.with(|deferred| deferred.replace(true)); + let _reset = Reset(previous); + operation() +} + /// Opens the queue store and returns the concrete QueueStore, so a target that needs its typed /// failed-store capability holds it directly rather than through the type-erased Store handle. pub(crate) fn open_target_queue_store_typed( @@ -625,9 +650,11 @@ pub(crate) fn open_target_queue_store_typed( TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, }; let store = QueueStore::::new(queue_dir, queue_limit, extension); - store - .open() - .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?; + if !DEFER_QUEUE_STORE_OPEN.with(Cell::get) { + store + .open() + .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?; + } Ok(Some(store)) } @@ -649,6 +676,26 @@ pub(crate) fn is_connectivity_error(err: &TargetError) -> bool { matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) } +/// Applies an absolute deadline to one protocol delivery attempt. +/// +/// Target clients expose different timeout controls, and several of them only +/// place a timeout value in the wire request without bounding the local socket +/// future. Keeping the outer deadline here gives every caller the same typed, +/// retryable timeout without changing the target-specific error mapping. +pub(crate) async fn with_delivery_deadline( + deadline: Duration, + operation: &'static str, + delivery: F, +) -> Result +where + F: Future>, +{ + match tokio::time::timeout(deadline, delivery).await { + Ok(result) => result, + Err(_) => Err(TargetError::Timeout(format!("{operation} timed out after {deadline:?}"))), + } +} + pub(crate) async fn invalidate_cache_on_connectivity_error(err: &TargetError, invalidate: F) where F: FnOnce() -> Fut, @@ -1137,6 +1184,29 @@ mod tests { let _ = fs::remove_file(base); } + #[test] + fn deferred_queue_store_creation_does_not_touch_the_filesystem() { + let base = std::env::temp_dir().join(format!("rustfs-target-store-deferred-{}", Uuid::new_v4())); + fs::write(&base, b"not-a-directory").expect("failed to create file base"); + let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string()); + + let store = with_deferred_queue_store_open(|| { + open_target_queue_store( + base.to_str().unwrap(), + 100, + TargetType::NotifyEvent, + ChannelTargetType::Kafka.as_str(), + &target_id, + "deferred open", + ) + }) + .expect("deferred construction must not open the queue directory") + .expect("non-empty queue directory should create a dormant store"); + + assert!(store.open().is_err(), "the invalid path must fail when handoff explicitly opens it"); + let _ = fs::remove_file(base); + } + #[test] fn persist_queued_payload_to_store_writes_encoded_payload() { let store = MockQueuedStore::new(false); @@ -1182,6 +1252,19 @@ mod tests { assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string()))); } + #[tokio::test(start_paused = true)] + async fn delivery_deadline_cuts_off_a_stalled_protocol_operation() { + let error = with_delivery_deadline( + Duration::from_secs(30), + "test delivery", + std::future::pending::>(), + ) + .await + .expect_err("a stalled delivery must hit its hard deadline"); + + assert!(matches!(error, TargetError::Timeout(message) if message == "test delivery timed out after 30s")); + } + #[tokio::test] async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() { let marker = Arc::new(AtomicBool::new(false)); diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index bc4bf26a0..3120d85b1 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -765,11 +765,6 @@ where #[instrument(skip(self, body, meta), fields(target_id = %self.id))] async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { - let client_guard = self.client.lock().await; - let client = client_guard - .as_ref() - .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; - debug!( event = EVENT_MQTT_DELIVERY_STATE, component = LOG_COMPONENT_TARGETS, @@ -790,10 +785,22 @@ where // silently dropped the event while its durable copy was already deleted // (backlog#971). Error classification now matches on the typed error // instead of substring matching on the display string. - let notice = match client.publish_tracked(&self.args.topic, self.args.qos, false, body).await { - Ok(notice) => notice, - Err(e) => { - let err = classify_mqtt_client_error(&e); + let notice = match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, async { + let client_guard = self.client.lock().await; + let client = client_guard + .as_ref() + .ok_or_else(|| TargetError::Configuration("MQTT client not initialized".to_string()))?; + let notice = client + .publish_tracked(&self.args.topic, self.args.qos, false, body) + .await + .map_err(|error| classify_mqtt_client_error(&error))?; + drop(client_guard); + Ok(notice) + }) + .await + { + Ok(Ok(notice)) => notice, + Ok(Err(err)) => { warn!( event = EVENT_MQTT_DELIVERY_STATE, component = LOG_COMPONENT_TARGETS, @@ -801,18 +808,29 @@ where target_id = %self.id, state = "publish_failed", reason = "enqueue_error", - error = %e, + error = %err, "mqtt delivery state" ); mark_target_disconnected_on_connectivity_error(&self.connected, &err); return Err(err); } + Err(_) => { + warn!( + event = EVENT_MQTT_DELIVERY_STATE, + component = LOG_COMPONENT_TARGETS, + subsystem = LOG_SUBSYSTEM_MQTT, + target_id = %self.id, + state = "publish_failed", + reason = "enqueue_timeout", + "mqtt delivery state" + ); + // Admission can time out because the local bounded request + // channel is full while the MQTT session remains connected. + // Only protocol/client failures are evidence of disconnect. + return Err(TargetError::Timeout("MQTT publish enqueue timed out".to_string())); + } }; - // Release the client lock before awaiting the broker acknowledgement so a - // slow/hung broker never blocks other senders from queueing publishes. - drop(client_guard); - match tokio::time::timeout(MQTT_PUBLISH_CONFIRM_TIMEOUT, notice.wait_completion_async()).await { Ok(Ok(())) => { debug!( @@ -1708,9 +1726,9 @@ where #[cfg(test)] mod tests { use super::{ - ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTlsConfig, PublishNoticeError, QoS, - classify_mqtt_client_error, classify_mqtt_notice_error, next_reconnect_backoff, reconnect_supervisor, - validate_mqtt_broker_url, + AsyncClient, ClientError, MQTT_RECONNECT_BACKOFF_MAX, MQTT_RECONNECT_BACKOFF_MIN, MQTTArgs, MQTTTarget, MQTTTlsConfig, + MqttOptions, PublishNoticeError, QoS, QueuedPayloadMeta, classify_mqtt_client_error, classify_mqtt_notice_error, + next_reconnect_backoff, reconnect_supervisor, validate_mqtt_broker_url, }; use crate::error::TargetError; use crate::target::{REDACTED_SECRET, TargetType}; @@ -1720,6 +1738,23 @@ mod tests { use tokio::sync::mpsc; use url::Url; + fn base_mqtt_args() -> MQTTArgs { + MQTTArgs { + enable: true, + broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"), + topic: "rustfs/events".to_string(), + qos: QoS::AtLeastOnce, + username: String::new(), + password: String::new(), + tls: MQTTTlsConfig::default(), + max_reconnect_interval: Duration::from_secs(1), + keep_alive: Duration::from_secs(30), + queue_dir: String::new(), + queue_limit: 0, + target_type: TargetType::NotifyEvent, + } + } + #[test] fn mqtt_client_error_classified_as_not_connected() { // A publish that cannot be handed to the event loop means the client is @@ -1752,6 +1787,38 @@ mod tests { assert!(matches!(classify_mqtt_notice_error(&err), TargetError::Request(_))); } + #[tokio::test(start_paused = true)] + async fn enqueue_timeout_keeps_a_live_session_connected() { + let target = MQTTTarget::::new("mqtt:test".to_string(), base_mqtt_args()).expect("target should build"); + let (client, _event_loop) = AsyncClient::builder(MqttOptions::new("mqtt-timeout-test", ("localhost", 1883))) + .capacity(1) + .build(); + client + .publish("fill", QoS::AtLeastOnce, false, b"fill".as_slice()) + .await + .expect("first publish should fill the local channel"); + *target.client.lock().await = Some(client); + target.connected.store(true, Ordering::SeqCst); + let meta = QueuedPayloadMeta::new( + rustfs_s3_types::EventName::ObjectCreatedPut, + "bucket".to_string(), + "object".to_string(), + "application/json", + 2, + ); + + let error = target + .send_body(b"{}".to_vec(), &meta) + .await + .expect_err("a full local request channel should hit the enqueue deadline"); + + assert!(matches!(error, TargetError::Timeout(_))); + assert!( + target.connected.load(Ordering::SeqCst), + "local admission pressure is not evidence that the MQTT session disconnected" + ); + } + #[test] fn next_reconnect_backoff_doubles_until_capped() { let mut backoff = MQTT_RECONNECT_BACKOFF_MIN; @@ -1864,21 +1931,13 @@ mod tests { #[test] fn debug_redacts_mqtt_secret_fields() { let args = MQTTArgs { - enable: true, - broker: Url::parse("mqtt://broker.example.com:1883").expect("valid broker"), - topic: "rustfs/events".to_string(), - qos: QoS::AtLeastOnce, username: "mqtt-user".to_string(), password: "mqtt-password".to_string(), tls: MQTTTlsConfig { client_key_path: "/etc/rustfs/mqtt.key".to_string(), ..MQTTTlsConfig::default() }, - max_reconnect_interval: Duration::from_secs(1), - keep_alive: Duration::from_secs(30), - queue_dir: String::new(), - queue_limit: 0, - target_type: TargetType::NotifyEvent, + ..base_mqtt_args() }; let rendered = format!("{args:?}"); diff --git a/crates/targets/src/target/mysql.rs b/crates/targets/src/target/mysql.rs index b14e252e8..eb070140f 100644 --- a/crates/targets/src/target/mysql.rs +++ b/crates/targets/src/target/mysql.rs @@ -25,7 +25,7 @@ use crate::{ target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store, - persist_queued_payload_to_store, redacted_secret, + persist_queued_payload_to_store, redacted_secret, with_delivery_deadline, }, }; use async_trait::async_trait; @@ -47,6 +47,8 @@ use uuid::Uuid; /// `TargetError::Timeout`, a connectivity error, so the payload stays queued /// for replay. const MYSQL_CONN_CHECKOUT_TIMEOUT: Duration = Duration::from_secs(15); +/// Absolute ceiling for one INSERT, including pool checkout and server execution. +const MYSQL_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30); /// Name of the optional idempotency-key column / primary key. Present on tables /// created by this target; absent on legacy two-column tables. @@ -784,29 +786,33 @@ where "Inserting MySQL event" ); - let pool = self.get_or_init_pool().await?; - // At this point the pool has already been initialized (get_or_init_pool - // succeeded above), so get_conn() failures are always transient: the - // connection was lost or the pool is temporarily exhausted. - let mut conn = checkout_conn(&pool).await?; - let event_time = extract_event_time(body)?; let event_data = std::str::from_utf8(body).map_err(|e| TargetError::Serialization(format!("Event body is not valid UTF-8: {e}")))?; let quoted_table = quote_table_name(&self.args.table)?; + with_delivery_deadline(MYSQL_DELIVERY_TIMEOUT, "MySQL delivery", async { + let pool = self.get_or_init_pool().await?; + // At this point the pool has already been initialized (get_or_init_pool + // succeeded above), so get_conn() failures are always transient: the + // connection was lost or the pool is temporarily exhausted. + let mut conn = checkout_conn(&pool).await?; - if self.idempotency_supported.load(Ordering::Relaxed) { - let sql = mysql_insert_sql_with_event_id("ed_table); - conn.exec_drop(sql, (event_id, event_time.as_str(), event_data)) - .await - .map_err(|err| map_mysql_error(err, "Failed to insert event"))?; - } else { - let sql = mysql_insert_sql_legacy("ed_table); - conn.exec_drop(sql, (event_time.as_str(), event_data)) - .await - .map_err(|err| map_mysql_error(err, "Failed to insert event"))?; - } + if self.idempotency_supported.load(Ordering::Relaxed) { + let sql = mysql_insert_sql_with_event_id("ed_table); + conn.exec_drop(sql, (event_id, event_time.as_str(), event_data)) + .await + .map_err(|err| map_mysql_error(err, "Failed to insert event"))?; + } else { + let sql = mysql_insert_sql_legacy("ed_table); + conn.exec_drop(sql, (event_time.as_str(), event_data)) + .await + .map_err(|err| map_mysql_error(err, "Failed to insert event"))?; + } + + Ok(()) + }) + .await?; self.delivery_counters.record_success(); debug!(target_id = %self.id, "MySQL event inserted"); diff --git a/crates/targets/src/target/nats/mod.rs b/crates/targets/src/target/nats/mod.rs index 81cc37c8f..ab8067115 100644 --- a/crates/targets/src/target/nats/mod.rs +++ b/crates/targets/src/target/nats/mod.rs @@ -25,7 +25,7 @@ use crate::{ target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error, - open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, + open_target_queue_store_typed, persist_queued_payload_to_store, redacted_secret, with_delivery_deadline, }, }; use async_trait::async_trait; @@ -52,6 +52,8 @@ use publish_error::{classify_nats_flush_error, classify_nats_publish_error}; pub(crate) use jetstream::resolve_dedup_id; pub(crate) use validation::{validate_jetstream_settings, validate_jetstream_stream}; +const NATS_CORE_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30); + #[derive(Clone)] pub struct NATSArgs { pub enable: bool, @@ -397,9 +399,21 @@ where } async fn send_body(&self, body: Vec) -> Result<(), TargetError> { - let client = self.get_or_connect().await?; - if let Err(e) = client.publish(self.args.subject.clone(), body.into()).await { - let err = classify_nats_publish_error(&e); + let result = with_delivery_deadline(NATS_CORE_DELIVERY_TIMEOUT, "NATS delivery", async { + let client = self.get_or_connect().await?; + client + .publish(self.args.subject.clone(), body.into()) + .await + .map_err(|err| classify_nats_publish_error(&err))?; + + // publish only enqueues the message on the client's outbound channel. Flush to confirm the + // message reached the server before delivery is treated as successful (backlog#971). + client.flush().await.map_err(|err| classify_nats_flush_error(&err))?; + Ok(()) + }) + .await; + + if let Err(err) = result { if is_connectivity_error(&err) { self.invalidate_cached_client_connection().await; self.connected.store(false, Ordering::SeqCst); @@ -407,15 +421,6 @@ where return Err(err); } - // publish only enqueues the message on the client's outbound channel. Flush to confirm the - // message reached the server before delivery is treated as successful (backlog#971). - if let Err(e) = client.flush().await { - let err = classify_nats_flush_error(&e); - self.invalidate_cached_client_connection().await; - self.connected.store(false, Ordering::SeqCst); - return Err(err); - } - self.delivery_counters.record_success(); Ok(()) } diff --git a/crates/targets/src/target/postgres.rs b/crates/targets/src/target/postgres.rs index 6469c20a0..982be02e7 100644 --- a/crates/targets/src/target/postgres.rs +++ b/crates/targets/src/target/postgres.rs @@ -38,7 +38,7 @@ use crate::{ target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, redacted_optional_secret, - redacted_secret, + redacted_secret, with_delivery_deadline, }, }; use async_trait::async_trait; @@ -70,6 +70,8 @@ const POSTGRES_POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10); /// Absolute ceiling on a single checkout, wrapping `pool.get()` in a Tokio /// timeout as a belt-and-suspenders guard on top of the deadpool timeouts. const POSTGRES_POOL_CHECKOUT_HARD_LIMIT: Duration = Duration::from_secs(20); +/// Absolute ceiling for one SQL delivery, including pool checkout and execution. +const POSTGRES_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30); /// Returns `true` for any `s3:ObjectRemoved:*` event. /// @@ -730,30 +732,29 @@ where let key = resolve_payload_key(&payload, meta); - let result = match self.args.format { - // For the single-row `namespace` format, an object removal must - // delete the row rather than UPSERT it, otherwise stale state - // lingers in the table after the object is gone. - PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => { - client.execute(&self.namespace_delete_sql, &[&key]).await + with_delivery_deadline(POSTGRES_DELIVERY_TIMEOUT, "PostgreSQL delivery", async { + match self.args.format { + // For the single-row `namespace` format, an object removal must + // delete the row rather than UPSERT it, otherwise stale state + // lingers in the table after the object is gone. + PostgresFormat::Namespace if is_object_removed_event(&meta.event_name) => { + client.execute(&self.namespace_delete_sql, &[&key]).await + } + PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await, + PostgresFormat::Access => { + let event_name_str = meta.event_name.to_string(); + let queued_at_ms = meta.queued_at_unix_ms as i64; + client + .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms]) + .await + } } - PostgresFormat::Namespace => client.execute(&self.namespace_sql, &[&key, &payload]).await, - PostgresFormat::Access => { - let event_name_str = meta.event_name.to_string(); - let queued_at_ms = meta.queued_at_unix_ms as i64; - client - .execute(&self.access_sql, &[&event_id, &event_name_str, &key, &payload, &queued_at_ms]) - .await - } - }; + .map_err(|err| map_pg_error(&err, "PostgreSQL insert failed")) + }) + .await?; - match result { - Ok(_) => { - self.delivery_counters.record_success(); - Ok(()) - } - Err(err) => Err(map_pg_error(&err, "PostgreSQL insert failed")), - } + self.delivery_counters.record_success(); + Ok(()) } /// Probes the table from `init()`. Failure is non-fatal when a queue is diff --git a/crates/targets/src/target/pulsar.rs b/crates/targets/src/target/pulsar.rs index f94cee68e..027f9d9c9 100644 --- a/crates/targets/src/target/pulsar.rs +++ b/crates/targets/src/target/pulsar.rs @@ -24,8 +24,9 @@ use crate::{ store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, open_target_queue_store, - persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component, + TargetTlsState, TargetType, build_queued_payload_with_records, build_target_tls_fingerprint, is_connectivity_error, + open_target_queue_store, persist_queued_payload_to_store, redacted_secret, sanitize_queue_dir_component, + with_delivery_deadline, }, }; use async_trait::async_trait; @@ -39,11 +40,15 @@ use std::fmt; use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use tokio::sync::Mutex as AsyncMutex; -use tracing::{info, instrument}; +use tracing::{info, instrument, warn}; use url::Url; use uuid::Uuid; +const PULSAR_DELIVERY_TIMEOUT: Duration = Duration::from_secs(30); +const PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT: Duration = Duration::from_secs(1); + #[derive(Clone)] pub struct PulsarArgs { pub enable: bool, @@ -276,6 +281,23 @@ where self.tls_state.lock().reset(); } + async fn clear_failed_delivery_state(&self) { + match tokio::time::timeout(PULSAR_FAILED_DELIVERY_CLEANUP_TIMEOUT, self.producer.lock()).await { + Ok(mut producer) => { + producer.take(); + } + Err(_) => { + warn!( + target_id = %self.id, + reason = "producer_cleanup_lock_timeout", + "Timed out clearing the Pulsar producer after a failed delivery" + ); + } + } + self.clear_cached_client(); + self.connected.store(false, Ordering::SeqCst); + } + async fn get_or_connect_client(&self) -> Result, TargetError> { // When a TLS reload adapter is attached, it drives client rebuilds // in the background. The inline per-send fingerprint check is skipped. @@ -334,20 +356,30 @@ where } async fn send_body(&self, body: Vec) -> Result<(), TargetError> { - self.init_producer().await?; - let mut guard = self.producer.lock().await; - let producer = guard - .as_mut() - .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?; - let receipt = producer - .send_non_blocking(body) - .await - .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?; - receipt - .await - .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?; - self.delivery_counters.record_success(); - Ok(()) + let result = with_delivery_deadline(PULSAR_DELIVERY_TIMEOUT, "Pulsar delivery", async { + self.init_producer().await?; + let mut guard = self.producer.lock().await; + let producer = guard + .as_mut() + .ok_or_else(|| TargetError::Configuration("Pulsar producer not initialized".to_string()))?; + let receipt = producer + .send_non_blocking(body) + .await + .map_err(|e| TargetError::Request(format!("Failed to send Pulsar message: {e}")))?; + receipt + .await + .map_err(|e| TargetError::Request(format!("Failed to receive Pulsar receipt: {e}")))?; + self.delivery_counters.record_success(); + Ok(()) + }) + .await; + + if let Err(err) = &result + && is_connectivity_error(err) + { + self.clear_failed_delivery_state().await; + } + result } } @@ -525,6 +557,22 @@ mod tests { } } + #[tokio::test(start_paused = true)] + async fn failed_delivery_cleanup_is_bounded_when_the_producer_lock_is_busy() { + let target = Arc::new(PulsarTarget::::new("pulsar:test".to_string(), base_args()).expect("target should build")); + target.connected.store(true, Ordering::SeqCst); + let producer_guard = target.producer.lock().await; + let cleanup = { + let target = Arc::clone(&target); + tokio::spawn(async move { target.clear_failed_delivery_state().await }) + }; + + cleanup.await.expect("cleanup task should not panic"); + + assert!(!target.connected.load(Ordering::SeqCst)); + drop(producer_guard); + } + #[test] fn debug_redacts_pulsar_secret_fields() { let args = PulsarArgs { diff --git a/crates/targets/src/target/redis.rs b/crates/targets/src/target/redis.rs index 8d30823cc..dd146e44c 100644 --- a/crates/targets/src/target/redis.rs +++ b/crates/targets/src/target/redis.rs @@ -26,6 +26,7 @@ use crate::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error, mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store, + with_delivery_deadline, }, }; use async_trait::async_trait; @@ -48,6 +49,19 @@ use tokio::sync::Mutex; use tracing::{debug, info, instrument, warn}; use url::Url; +const REDIS_CONNECTION_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5); +const REDIS_RESPONSE_TIMEOUT_DEFAULT: Duration = Duration::from_secs(5); + +fn redis_total_delivery_timeout(args: &RedisArgs) -> Duration { + let attempts = u32::try_from(args.max_retry_attempts).unwrap_or(u32::MAX); + let per_attempt = args + .connection_timeout + .unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT) + .saturating_add(args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT)) + .saturating_add(args.max_retry_delay.unwrap_or(Duration::from_secs(2))); + per_attempt.saturating_mul(attempts) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RedisTlsPolicy { SystemCa, @@ -214,6 +228,16 @@ impl RedisArgs { )); } + if self.connection_timeout == Some(Duration::ZERO) { + return Err(TargetError::Configuration( + "Redis connection_timeout must be greater than zero".to_string(), + )); + } + + if self.response_timeout == Some(Duration::ZERO) { + return Err(TargetError::Configuration("Redis response_timeout must be greater than zero".to_string())); + } + if self.pipeline_buffer_size == Some(0) { return Err(TargetError::Configuration( "Redis pipeline_buffer_size must be greater than zero".to_string(), @@ -464,71 +488,97 @@ where "Sending Redis payload" ); - let mut attempt = 0usize; - let mut last_error = None; - while attempt < self.args.max_retry_attempts { - attempt += 1; + let result = with_delivery_deadline(redis_total_delivery_timeout(&self.args), "Redis delivery", async { + let mut attempt = 0usize; + let mut last_error = None; + while attempt < self.args.max_retry_attempts { + attempt += 1; - let mut publisher = self.get_or_create_publisher().await?; - match publisher - .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice()) + let connection_timeout = self.args.connection_timeout.unwrap_or(REDIS_CONNECTION_TIMEOUT_DEFAULT); + let mut publisher = match with_delivery_deadline( + connection_timeout, + "Redis connection", + self.get_or_create_publisher(), + ) .await - { - Ok(receiver_count) => { - // PUBLISH returns the number of subscribers that received the - // message. Redis pub/sub is best-effort: with zero subscribers - // the event is delivered to no one, yet the durable copy is - // deleted. Warn so operators relying on reliable delivery are - // not silently losing events (backlog#982). - if receiver_count == 0 { + { + Ok(publisher) => publisher, + Err(err) => { + invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await; + return Err(err); + } + }; + let response_timeout = self.args.response_timeout.unwrap_or(REDIS_RESPONSE_TIMEOUT_DEFAULT); + match with_delivery_deadline(response_timeout, "Redis publish response", async { + publisher + .publish::<_, _, i64>(self.args.channel.as_str(), body.as_slice()) + .await + .map_err(map_redis_error) + }) + .await + { + Ok(receiver_count) => { + // PUBLISH returns the number of subscribers that received the + // message. Redis pub/sub is best-effort: with zero subscribers + // the event is delivered to no one, yet the durable copy is + // deleted. Warn so operators relying on reliable delivery are + // not silently losing events (backlog#982). + if receiver_count == 0 { + warn!( + target_id = %self.id, + channel = %self.args.channel, + "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)" + ); + } + debug!( + target_id = %self.id, + channel = %self.args.channel, + attempt, + receiver_count, + "Event published to Redis channel" + ); + self.delivery_counters.record_success(); + return Ok(()); + } + Err(mapped) => { + invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await; + warn!( target_id = %self.id, channel = %self.args.channel, - "Redis PUBLISH reached 0 subscribers; the event was not received by any consumer (pub/sub is best-effort)" + attempt, + max_attempts = self.args.max_retry_attempts, + error = %mapped, + "Redis publish attempt failed" ); - } - debug!( - target_id = %self.id, - channel = %self.args.channel, - attempt, - receiver_count, - "Event published to Redis channel" - ); - self.delivery_counters.record_success(); - return Ok(()); - } - Err(err) => { - let mapped = map_redis_error(err); - invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await; - warn!( - target_id = %self.id, - channel = %self.args.channel, - attempt, - max_attempts = self.args.max_retry_attempts, - error = %mapped, - "Redis publish attempt failed" - ); + if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts { + last_error = Some(mapped); + break; + } - if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts { last_error = Some(mapped); - break; + tokio::time::sleep(compute_retry_delay( + attempt, + self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)), + self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)), + )) + .await; } - - last_error = Some(mapped); - tokio::time::sleep(compute_retry_delay( - attempt, - self.args.min_retry_delay.unwrap_or(Duration::from_millis(100)), - self.args.max_retry_delay.unwrap_or(Duration::from_secs(2)), - )) - .await; } } + + Err(last_error.unwrap_or(TargetError::Unknown( + "Redis publish failed without a captured error".to_string(), + ))) + }) + .await; + + if let Err(err) = &result { + invalidate_cache_on_connectivity_error(err, || self.invalidate_cached_publisher()).await; + self.connected.store(false, Ordering::SeqCst); } - - self.connected.store(false, Ordering::SeqCst); - - Err(last_error.unwrap_or(TargetError::Unknown("Redis publish failed without a captured error".to_string()))) + result } } @@ -551,7 +601,7 @@ where // thus a fresh TCP+TLS handshake — on every health check (backlog#982). // ensure_publisher_ready already invalidates the cached manager on a // connectivity error so the next attempt rebuilds it. - match tokio::time::timeout(Duration::from_secs(5), self.ensure_publisher_ready()).await { + match tokio::time::timeout(REDIS_CONNECTION_TIMEOUT_DEFAULT, self.ensure_publisher_ready()).await { Ok(Ok(())) => { self.connected.store(true, Ordering::SeqCst); Ok(true) @@ -917,6 +967,26 @@ mod tests { assert!(args.validate().is_err()); } + #[test] + fn validate_rejects_zero_connection_timeout() { + let args = RedisArgs { + connection_timeout: Some(Duration::ZERO), + ..base_args() + }; + + assert!(args.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_response_timeout() { + let args = RedisArgs { + response_timeout: Some(Duration::ZERO), + ..base_args() + }; + + assert!(args.validate().is_err()); + } + #[test] fn validate_accepts_custom_ca_tls_policy() { let args = RedisArgs { @@ -1237,6 +1307,23 @@ mod tests { assert_eq!(target.delivery_snapshot().total_messages, 1); } + #[tokio::test(start_paused = true)] + async fn delivery_budget_respects_response_timeout_longer_than_sixty_seconds() { + let mut args = base_args(); + args.max_retry_attempts = 1; + args.response_timeout = Some(Duration::from_secs(90)); + let manager_config = build_redis_connection_manager_config(&args); + + assert_eq!(manager_config.response_timeout(), Some(Duration::from_secs(90))); + assert_eq!(redis_total_delivery_timeout(&args), Duration::from_secs(97)); + with_delivery_deadline(redis_total_delivery_timeout(&args), "Redis delivery", async { + tokio::time::sleep(Duration::from_secs(70)).await; + Ok::<_, TargetError>(()) + }) + .await + .expect("the configured delivery budget must not impose a fixed sixty-second cap"); + } + #[tokio::test] async fn send_body_sets_connected_false_after_retry_exhaustion() { let listener = match TcpListener::bind("127.0.0.1:0").await { diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index 7c45006bb..cfa64ba08 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -952,7 +952,7 @@ mod tests { .expect("https webhook probe should trust configured ca"); assert_eq!(resp.status(), reqwest::StatusCode::OK); - assert_eq!(resp.text_with_charset("utf-8").await.expect("read response body"), ""); + assert!(resp.bytes().await.expect("read response body").is_empty()); handle.join().expect("tls server thread"); } } diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index c2c7ec543..d72e4b329 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -19,19 +19,24 @@ |---|---|---| | admin_auth_test | 3 | ✅ | | admin_iam_crud_test | 2 | ✅ | +| admin_pools_test | 1 | ✅ | | admin_timeout_regression_test | 1 | | | anonymous_access_test | 3 | ✅ | +| api_rate_limit_test | 3 | | | archive_download_integrity_test | 13 | | | bucket_logging_test | 3 | | | bucket_policy_check_test | 1 | ✅ | | checksum_upload_test | 7 | | | cluster_concurrency_test | 2 | | -| common | 3 | | +| cluster_multidrive_pool_test | 2 | | +| common | 10 | | | compression_test | 1 | | +| connection_cap_test | 2 | | | console_smoke_test | 1 | ✅ | | content_encoding_test | 3 | ✅ | +| copy_object_checksum_test | 3 | | | copy_object_metadata_test | 1 | ✅ | -| copy_object_version_restore_test | 1 | | +| copy_object_version_restore_test | 2 | | | copy_source_invalid_date_test | 1 | ✅ | | create_bucket_region_test | 2 | ✅ | | degraded_read_eof_regression_test | 3 | | @@ -40,6 +45,7 @@ | delete_objects_versioning_test | 2 | ✅ | | existing_object_tag_policy_test | 4 | | | fake_s3_target | 4 | ✅ | +| fault_proxy | 7 | | | get_codec_streaming_compat_test | 1 | | | head_object_consistency_test | 1 | ✅ | | head_object_range_test | 1 | ✅ | @@ -56,13 +62,13 @@ | multipart_auth_test | 109 | | | namespace_lock_quorum_test | 2 | | | negative_sigv4_test | 6 | ✅ | -| notification_webhook_test | 1 | ✅ | +| notification_webhook_test | 2 | ✅ | | object_lambda_test | 16 | | | object_lock | 33 | | | overwrite_cleanup_regression_test | 1 | | | presigned_negative_test | 7 | ✅ | | protocols | 16 | | -| quota_test | 13 | | +| quota_test | 14 | | | reliability_disk_fault_test | 3 | | | reliant | 10 | 4 ✅ | | replication_extension_test | 47 | 20 ✅ +27 🌙 | @@ -75,4 +81,6 @@ | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -**Total listed: 439 tests across 57 modules · PR smoke subset: 112 tests / 27 modules** (25 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-16. +`notification_webhook_test` also has 1 ignored store-and-forward regression tracked by rustfs#4852; ignored tests are excluded from the active counts above. + +**Total listed: 467 tests across 63 modules · PR smoke subset: 114 tests / 28 modules** (26 full modules + 4 `reliant` tests + 20 of `replication_extension_test`) **· nightly `e2e-repl-nightly`: 27 tests** · generated 2026-07-21. diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs index bf5e656c2..d6515a709 100644 --- a/rustfs/src/admin/handlers/audit.rs +++ b/rustfs/src/admin/handlers/audit.rs @@ -221,7 +221,7 @@ async fn authorize_audit_admin_request(req: &S3Request, action: AdminActio validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await } -fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { +fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result> { shared_target_mutation_block_reason( audit_target_specs(), AUDIT_ROUTE_PREFIX, @@ -248,16 +248,18 @@ async fn audit_target_operation_block_reason(action: &str) -> Option { target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action) } -fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap) -> Vec { - shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses) - .into_iter() - .map(|endpoint| AuditEndpoint { - account_id: endpoint.account_id, - service: endpoint.service, - status: endpoint.status, - source: endpoint.source, - }) - .collect() +fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap) -> S3Result> { + Ok( + shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)? + .into_iter() + .map(|endpoint| AuditEndpoint { + account_id: endpoint.account_id, + service: endpoint.service, + status: endpoint.status, + source: endpoint.source, + }) + .collect(), + ) } fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> { @@ -279,7 +281,7 @@ impl Operation for AuditTargetConfig { return Err(s3_error!(InvalidRequest, "{reason}")); } let config_snapshot = load_server_config_from_store().await?; - if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) { + if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? { log_audit_target_operation_blocked!("set_audit_target_config", Some(target_type), Some(target_name), &reason); return Err(s3_error!(InvalidRequest, "{reason}")); } @@ -305,12 +307,14 @@ impl Operation for AuditTargetConfig { ) .await?; - update_audit_config_and_reload(audit_target_specs(), |config| { + let mutation_target_type = target_type.to_lowercase(); + let mutation_target_name = target_name.to_lowercase(); + update_audit_config_and_reload(audit_target_specs(), move |config| { config .0 - .entry(target_type.to_lowercase()) + .entry(mutation_target_type.clone()) .or_default() - .insert(target_name.to_lowercase(), kvs.clone()); + .insert(mutation_target_name.clone(), kvs.clone()); true }) .await @@ -344,7 +348,7 @@ impl Operation for ListAuditTargets { } let config = load_server_config_from_store().await?; - let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses); + let audit_endpoints = merge_audit_endpoints(&config, runtime_statuses)?; let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints }).map_err(|e| { log_audit_target_request_failed!("list_audit_targets", "serialize_audit_targets_failed", None, None, e); s3_error!(InternalError, "failed to serialize audit targets: {}", e) @@ -369,19 +373,21 @@ impl Operation for RemoveAuditTarget { return Err(s3_error!(InvalidRequest, "{reason}")); } let config_snapshot = load_server_config_from_store().await?; - if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name) { + if let Some(reason) = audit_target_mutation_block_reason(&config_snapshot, target_type, target_name)? { log_audit_target_operation_blocked!("remove_audit_target_config", Some(target_type), Some(target_name), &reason); return Err(s3_error!(InvalidRequest, "{reason}")); } - update_audit_config_and_reload(audit_target_specs(), |config| { + let mutation_target_type = target_type.to_lowercase(); + let mutation_target_name = target_name.to_lowercase(); + update_audit_config_and_reload(audit_target_specs(), move |config| { let mut changed = false; - if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) { - if targets.remove(&target_name.to_lowercase()).is_some() { + if let Some(targets) = config.0.get_mut(&mutation_target_type) { + if targets.remove(&mutation_target_name).is_some() { changed = true; } if targets.is_empty() { - config.0.remove(&target_type.to_lowercase()); + config.0.remove(&mutation_target_type); } } changed @@ -469,7 +475,7 @@ mod tests { (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()), (("env-only".to_string(), "webhook".to_string()), "online".to_string()), ]); - let merged = merge_audit_endpoints(&config, runtime); + let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints"); let mixed = merged .iter() @@ -512,7 +518,7 @@ mod tests { (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()), (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()), ]); - let merged = merge_audit_endpoints(&config, runtime); + let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints"); let mixed = merged .iter() @@ -549,7 +555,7 @@ mod tests { (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()), (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()), ]); - let merged = merge_audit_endpoints(&config, runtime); + let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints"); let mixed = merged .iter() @@ -576,7 +582,8 @@ mod tests { ], || { let config = Config(HashMap::new()); - let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary"); + let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary") + .expect("audit target mutation block reason"); assert!(reason.is_some()); assert!(reason.unwrap().contains("managed by environment variables")); }, @@ -613,7 +620,8 @@ mod tests { AUDIT_WEBHOOK_SUB_SYS.to_string(), HashMap::from([("primary".to_string(), enabled_kvs("on"))]), )])); - let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary"); + let reason = audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary") + .expect("audit target mutation block reason"); assert!(reason.is_some()); assert!(reason.unwrap().contains("both persisted config and environment variables")); }); @@ -633,7 +641,7 @@ mod tests { ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")), ], || { - let merged = merge_audit_endpoints(&config, HashMap::new()); + let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints"); let mixed = merged .iter() .find(|entry| entry.account_id == "mixed-disabled") @@ -655,7 +663,7 @@ mod tests { ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), ], || { - let merged = merge_audit_endpoints(&config, HashMap::new()); + let merged = merge_audit_endpoints(&config, HashMap::new()).expect("merge audit endpoints"); let env_only = merged .iter() .find(|entry| entry.account_id == "env-only") @@ -768,7 +776,7 @@ mod tests { ], || { let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]); - let merged = merge_audit_endpoints(&config, runtime); + let merged = merge_audit_endpoints(&config, runtime).expect("merge audit endpoints"); let mixed = merged .iter() .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook") @@ -787,7 +795,11 @@ mod tests { )])); with_audit_webhook_target_env_cleared("primarycase", || { - assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase").is_none()); + assert!( + audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primarycase") + .expect("audit target mutation block reason") + .is_none() + ); }); } @@ -795,7 +807,11 @@ mod tests { fn audit_target_mutation_block_reason_allows_runtime_only_target() { with_audit_webhook_target_env_cleared("primary", || { let config = Config(HashMap::new()); - assert!(audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary").is_none()); + assert!( + audit_target_mutation_block_reason(&config, AUDIT_WEBHOOK_SUB_SYS, "primary") + .expect("audit target mutation block reason") + .is_none() + ); }); } diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs index 5f038c118..0b6853832 100644 --- a/rustfs/src/admin/handlers/audit_runtime_config.rs +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -14,11 +14,15 @@ use crate::admin::handlers::target_descriptor::AdminTargetSpec; use crate::admin::runtime_sources::{AppContext, current_app_context, current_object_store_handle_for_context}; -use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config}; +use crate::admin::storage_api::config::{ + read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock, + with_admin_server_config_write_lock, +}; use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::server_config::Config; use s3s::{S3Result, s3_error}; +use tracing::warn; pub(crate) async fn load_server_config_from_store_for_context(context: Option<&AppContext>) -> S3Result { let Some(store) = current_object_store_handle_for_context(context) else { @@ -51,30 +55,31 @@ pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config match system.get_state().await { AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { if has_targets { - system - .reload_config(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?; + system.reload_config(config).await.map_err(|_| { + warn!(reason = "reload_failed", "Failed to reload local audit runtime"); + s3_error!(InternalError, "failed to reload audit config") + })?; } else { - system - .close() - .await - .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?; + system.close().await.map_err(|_| { + warn!(reason = "stop_failed", "Failed to stop local audit runtime"); + s3_error!(InternalError, "failed to stop audit system") + })?; } } AuditSystemState::Stopped | AuditSystemState::Stopping => { if has_targets { - system - .start(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + system.start(config).await.map_err(|_| { + warn!(reason = "start_failed", "Failed to start local audit runtime"); + s3_error!(InternalError, "failed to start audit system") + })?; } } } } else if has_targets { - start_global_audit_system(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + start_global_audit_system(config).await.map_err(|_| { + warn!(reason = "start_failed", "Failed to start global audit runtime"); + s3_error!(InternalError, "failed to start audit system") + })?; } Ok(()) @@ -86,30 +91,40 @@ async fn update_audit_config_and_reload_for_context( mut modifier: F, ) -> S3Result<()> where - F: FnMut(&mut Config) -> bool, + F: FnMut(&mut Config) -> bool + Send + 'static, { let Some(store) = current_object_store_handle_for_context(context) else { return Err(s3_error!(InternalError, "server storage not initialized")); }; - let mut config = read_admin_config_without_migrate(store.clone()) - .await - .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; + let specs = specs.to_vec(); + let lock_store = store.clone(); + with_admin_server_config_write_lock(lock_store, move || async move { + let mut config = read_admin_config_without_migrate_no_lock(store.clone()) + .await + .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; - if !modifier(&mut config) { - return Ok(()); - } + if !modifier(&mut config) { + return Ok(()); + } - save_admin_server_config(store, &config) - .await - .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; + save_admin_server_config_no_lock(store, &config) + .await + .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; - apply_audit_runtime_config(specs, config).await + // Keep persistence and runtime publication in one detached, serialized + // mutation. Otherwise a cancelled caller or two concurrent updates can + // leave the persisted config and active audit generation disagreeing. + apply_audit_runtime_config(&specs, config).await + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; + Ok(()) } pub(crate) async fn update_audit_config_and_reload(specs: &[AdminTargetSpec], modifier: F) -> S3Result<()> where - F: FnMut(&mut Config) -> bool, + F: FnMut(&mut Config) -> bool + Send + 'static, { let context = current_app_context(); update_audit_config_and_reload_for_context(context.as_deref(), specs, modifier).await @@ -121,26 +136,30 @@ pub(crate) async fn set_audit_target_config( target_name: &str, kvs: rustfs_config::server_config::KVS, ) -> S3Result<()> { - update_audit_config_and_reload(specs, |config| { + let subsystem = subsystem.to_lowercase(); + let target_name = target_name.to_lowercase(); + update_audit_config_and_reload(specs, move |config| { config .0 - .entry(subsystem.to_lowercase()) + .entry(subsystem.clone()) .or_default() - .insert(target_name.to_lowercase(), kvs.clone()); + .insert(target_name.clone(), kvs.clone()); true }) .await } pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> { - update_audit_config_and_reload(specs, |config| { + let subsystem = subsystem.to_lowercase(); + let target_name = target_name.to_lowercase(); + update_audit_config_and_reload(specs, move |config| { let mut changed = false; - if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) { - if targets.remove(&target_name.to_lowercase()).is_some() { + if let Some(targets) = config.0.get_mut(&subsystem) { + if targets.remove(&target_name).is_some() { changed = true; } if targets.is_empty() { - config.0.remove(&subsystem.to_lowercase()); + config.0.remove(&subsystem); } } changed diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index c51966ca9..c11f18266 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::admin::auth::validate_admin_request; +use crate::admin::handlers::supervise_admin_mutation; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ current_app_context, current_object_store_handle_for_context, current_server_config_for_context, publish_server_config, @@ -20,12 +21,14 @@ use crate::admin::runtime_sources::{ use crate::admin::service::config::{ CONFIG_WORKER_RELOAD_FAILURE_STATE, EVENT_CONFIG_WORKER_RELOAD_FAILED, FULL_CONFIG_WORKER_SUBSYSTEMS, LOG_COMPONENT_ADMIN, LOG_SUBSYSTEM_CONFIG, PreparedRuntimeConfig, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, - prepare_server_config, signal_config_snapshot_reload, signal_dynamic_config_reload, + preflight_dynamic_config_reload, prepare_server_config, signal_config_snapshot_reload_checked, + signal_dynamic_config_reload_checked, }; use crate::admin::storage_api::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_api::config::{ RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, read_admin_config, read_admin_config_without_migrate, - save_admin_config, save_admin_server_config, + read_admin_config_without_migrate_no_lock, save_admin_config, save_admin_server_config_no_lock, + with_admin_server_config_write_lock, }; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; @@ -47,7 +50,7 @@ use rustfs_config::notify::{ ENV_NOTIFY_MQTT_QOS, ENV_NOTIFY_MQTT_QUEUE_DIR, ENV_NOTIFY_MQTT_QUEUE_LIMIT, ENV_NOTIFY_MQTT_RECONNECT_INTERVAL, ENV_NOTIFY_MQTT_TOPIC, ENV_NOTIFY_MQTT_USERNAME, ENV_NOTIFY_WEBHOOK_AUTH_TOKEN, ENV_NOTIFY_WEBHOOK_CLIENT_CERT, ENV_NOTIFY_WEBHOOK_CLIENT_KEY, ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, ENV_NOTIFY_WEBHOOK_QUEUE_DIR, - ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + ENV_NOTIFY_WEBHOOK_QUEUE_LIMIT, NOTIFY_MQTT_SUB_SYS, NOTIFY_SUB_SYSTEMS, NOTIFY_WEBHOOK_SUB_SYS, }; use rustfs_config::oidc::{ ENV_IDENTITY_OPENID_CLAIM_NAME, ENV_IDENTITY_OPENID_CLAIM_PREFIX, ENV_IDENTITY_OPENID_CLIENT_ID, @@ -726,6 +729,14 @@ async fn load_server_config_from_store() -> S3Result { .map_err(Into::into) } +async fn load_server_config_from_store_locked() -> S3Result { + let store = object_store()?; + read_admin_config_without_migrate_no_lock(store) + .await + .map_err(ApiError::from) + .map_err(Into::into) +} + async fn load_active_server_config() -> S3Result { if let Ok(config) = load_server_config_from_store().await { return Ok(config); @@ -736,9 +747,9 @@ async fn load_active_server_config() -> S3Result { .ok_or_else(|| s3_error!(InternalError, "server config is not initialized")) } -async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> { +async fn save_server_config_to_store_locked(config: &ServerConfig) -> S3Result<()> { let store = object_store()?; - save_admin_server_config(store, config) + save_admin_server_config_no_lock(store, config) .await .map_err(ApiError::from) .map_err(Into::into) @@ -1563,20 +1574,144 @@ async fn commit_prepared_config( /// Re-apply local mutable worker families after a full-config replacement. /// Peers receive one full-snapshot signal after this returns; signaling each /// family here as well would recreate audit/scanner targets twice per peer. -async fn apply_dynamic_subsystems(config: &ServerConfig) { +fn publish_notify_config_intent( + config: &ServerConfig, + sub_system: Option<&str>, +) -> Option { + (sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system))) + .then(|| rustfs_notify::ensure_live_events().publish_config(config.clone())) +} + +async fn preflight_notify_config_intent(sub_system: Option<&str>) -> S3Result<()> { + if sub_system.is_none() || sub_system.is_some_and(|sub_system| NOTIFY_SUB_SYSTEMS.contains(&sub_system)) { + preflight_dynamic_config_reload(sub_system.unwrap_or(NOTIFY_WEBHOOK_SUB_SYS)).await?; + } + Ok(()) +} + +async fn wait_notify_config_intent(transition: Option) -> S3Result { + let Some(transition) = transition else { + return Ok(false); + }; + transition.wait().await.map_err(|err| { + warn!(error = %err, "Failed to apply local notification config"); + s3_error!(InternalError, "failed to apply notification config") + })?; + Ok(true) +} + +async fn apply_non_notify_dynamic_subsystems(config: &ServerConfig) -> Vec { + let mut failures = Vec::new(); for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if let Err(err) = apply_dynamic_config_for_subsystem(config, sub_system).await { + if NOTIFY_SUB_SYSTEMS.contains(&sub_system) { + continue; + } + if apply_dynamic_config_for_subsystem(config, sub_system).await.is_err() { + failures.push(format!("local {sub_system}")); warn!( event = EVENT_CONFIG_WORKER_RELOAD_FAILED, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_CONFIG, config_subsystem = sub_system, state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - error = %err, + reason = "apply_failed", "Published server config but failed to reload a local worker subsystem" ); } } + failures +} + +fn finish_config_reconciliation(errors: Vec) -> S3Result<()> { + if errors.is_empty() { + Ok(()) + } else { + Err(s3_error!( + InternalError, + "server config persisted but runtime convergence failed: {}", + errors.join("; ") + )) + } +} + +async fn reconcile_targeted_config( + config: ServerConfig, + sub_system: Option, + storage_class_applied: bool, + notify_transition: Option, +) -> S3Result { + let mut errors = Vec::new(); + let notify_applied = notify_transition.is_some(); + if let Err(err) = wait_notify_config_intent(notify_transition).await { + warn!(error = %err, "Local notification config failed to converge"); + errors.push("local notify".to_string()); + } + + let config_applied = if notify_applied { + if let Some(sub_system) = sub_system.as_deref() + && let Err(err) = signal_dynamic_config_reload_checked(sub_system).await + { + warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed"); + errors.push(format!("peer {sub_system}")); + } + true + } else if storage_class_applied { + if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { + warn!(error = %err, "Peer storage-class reload failed"); + errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); + } + true + } else if let Some(sub_system) = sub_system.as_deref() + && is_dynamic_config_subsystem(sub_system) + { + let config_applied = match apply_dynamic_config_for_subsystem(&config, sub_system).await { + Ok(applied) => applied, + Err(_) => { + warn!(config_subsystem = sub_system, reason = "apply_failed", "Local config reload failed"); + errors.push(format!("local {sub_system}")); + false + } + }; + if let Err(err) = signal_dynamic_config_reload_checked(sub_system).await { + warn!(config_subsystem = sub_system, error = %err, "Peer config reload failed"); + errors.push(format!("peer {sub_system}")); + } + config_applied + } else { + if let Err(err) = signal_config_snapshot_reload_checked().await { + warn!(error = %err, "Peer config snapshot reload failed"); + errors.push("peer config snapshot".to_string()); + } + false + }; + + finish_config_reconciliation(errors)?; + Ok(config_applied) +} + +async fn reconcile_full_config( + config: ServerConfig, + notify_transition: Option, +) -> S3Result<()> { + let mut errors = Vec::new(); + if let Err(err) = wait_notify_config_intent(notify_transition).await { + warn!(error = %err, "Local notification config failed to converge"); + errors.push("local notify".to_string()); + } + if let Err(err) = signal_dynamic_config_reload_checked(STORAGE_CLASS_SUB_SYS).await { + warn!(error = %err, "Peer storage-class reload failed"); + errors.push(format!("peer {STORAGE_CLASS_SUB_SYS}")); + } + errors.extend(apply_non_notify_dynamic_subsystems(&config).await); + if let Err(err) = signal_dynamic_config_reload_checked(NOTIFY_WEBHOOK_SUB_SYS).await { + warn!(error = %err, "Peer notification config reload failed"); + errors.push("peer notify".to_string()); + } + if let Err(err) = signal_config_snapshot_reload_checked().await { + warn!(error = %err, "Peer config snapshot reload failed"); + errors.push("peer config snapshot".to_string()); + } + finish_config_reconciliation(errors) } pub struct GetConfigKVHandler {} @@ -1611,37 +1746,39 @@ impl Operation for SetConfigKVHandler { } validate_config_directives(&directives)?; - let sub_system = config_update_sub_system(&directives)?; - let mut config = load_server_config_from_store().await?; - apply_set_directives(&mut config, &directives)?; - let prepared = prepare_server_config(&config, sub_system).await?; - save_server_config_history(&body).await?; - let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) { - commit_prepared_config( - config.clone(), - prepared, - save_server_config_to_store(&config), - publish_prepared_config_snapshots, - ) - .await?; - signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await; - true - } else { - save_server_config_to_store(&config).await?; - publish_server_config(config.clone()); - if let Some(sub_system) = sub_system - && is_dynamic_config_subsystem(sub_system) - { - let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?; - if config_applied { - signal_dynamic_config_reload(sub_system).await; - } - config_applied - } else { - signal_config_snapshot_reload().await; - false - } - }; + let sub_system = config_update_sub_system(&directives)?.map(str::to_owned); + let transaction_sub_system = sub_system.clone(); + let config_store = object_store()?; + let config_applied = supervise_admin_mutation("config mutation", async move { + preflight_notify_config_intent(sub_system.as_deref()).await?; + let (config, storage_class_applied, notify_transition) = + with_admin_server_config_write_lock(config_store, move || async move { + let sub_system = transaction_sub_system.as_deref(); + let mut config = load_server_config_from_store_locked().await?; + apply_set_directives(&mut config, &directives)?; + let prepared = prepare_server_config(&config, sub_system).await?; + save_server_config_history(&body).await?; + if sub_system == Some(STORAGE_CLASS_SUB_SYS) { + commit_prepared_config( + config.clone(), + prepared, + save_server_config_to_store_locked(&config), + publish_prepared_config_snapshots, + ) + .await?; + } else { + save_server_config_to_store_locked(&config).await?; + publish_server_config(config.clone()); + } + let notify_transition = publish_notify_config_intent(&config, sub_system); + Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition)) + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; + + reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await + }) + .await?; success_response(config_applied) } @@ -1660,37 +1797,39 @@ impl Operation for DelConfigKVHandler { } validate_config_directives(&directives)?; - let sub_system = config_update_sub_system(&directives)?; - let mut config = load_server_config_from_store().await?; - apply_delete_directives(&mut config, &directives); - let prepared = prepare_server_config(&config, sub_system).await?; - save_server_config_history(&body).await?; - let config_applied = if sub_system == Some(STORAGE_CLASS_SUB_SYS) { - commit_prepared_config( - config.clone(), - prepared, - save_server_config_to_store(&config), - publish_prepared_config_snapshots, - ) - .await?; - signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await; - true - } else { - save_server_config_to_store(&config).await?; - publish_server_config(config.clone()); - if let Some(sub_system) = sub_system - && is_dynamic_config_subsystem(sub_system) - { - let config_applied = apply_dynamic_config_for_subsystem(&config, sub_system).await?; - if config_applied { - signal_dynamic_config_reload(sub_system).await; - } - config_applied - } else { - signal_config_snapshot_reload().await; - false - } - }; + let sub_system = config_update_sub_system(&directives)?.map(str::to_owned); + let transaction_sub_system = sub_system.clone(); + let config_store = object_store()?; + let config_applied = supervise_admin_mutation("config mutation", async move { + preflight_notify_config_intent(sub_system.as_deref()).await?; + let (config, storage_class_applied, notify_transition) = + with_admin_server_config_write_lock(config_store, move || async move { + let sub_system = transaction_sub_system.as_deref(); + let mut config = load_server_config_from_store_locked().await?; + apply_delete_directives(&mut config, &directives); + let prepared = prepare_server_config(&config, sub_system).await?; + save_server_config_history(&body).await?; + if sub_system == Some(STORAGE_CLASS_SUB_SYS) { + commit_prepared_config( + config.clone(), + prepared, + save_server_config_to_store_locked(&config), + publish_prepared_config_snapshots, + ) + .await?; + } else { + save_server_config_to_store_locked(&config).await?; + publish_server_config(config.clone()); + } + let notify_transition = publish_notify_config_intent(&config, sub_system); + Ok::<_, S3Error>((config, sub_system == Some(STORAGE_CLASS_SUB_SYS), notify_transition)) + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; + + reconcile_targeted_config(config, sub_system, storage_class_applied, notify_transition).await + }) + .await?; success_response(config_applied) } @@ -1776,16 +1915,26 @@ impl Operation for RestoreConfigHistoryKVHandler { let mut config = ServerConfig::new(); apply_set_directives(&mut config, &directives)?; let prepared = prepare_server_config(&config, None).await?; - commit_prepared_config( - config.clone(), - prepared, - save_server_config_to_store(&config), - publish_prepared_config_snapshots, - ) + let config_store = object_store()?; + supervise_admin_mutation("config mutation", async move { + preflight_notify_config_intent(None).await?; + let persisted_config = config.clone(); + let notify_transition = with_admin_server_config_write_lock(config_store, move || async move { + commit_prepared_config( + persisted_config.clone(), + prepared, + save_server_config_to_store_locked(&persisted_config), + publish_prepared_config_snapshots, + ) + .await?; + Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None)) + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config restore: {}", err))??; + + reconcile_full_config(config, notify_transition).await + }) .await?; - signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await; - apply_dynamic_subsystems(&config).await; - signal_config_snapshot_reload().await; success_response(false) } @@ -1820,17 +1969,27 @@ impl Operation for SetConfigHandler { let mut config = ServerConfig::new(); apply_set_directives(&mut config, &directives)?; let prepared = prepare_server_config(&config, None).await?; - save_server_config_history(&body).await?; - commit_prepared_config( - config.clone(), - prepared, - save_server_config_to_store(&config), - publish_prepared_config_snapshots, - ) + let config_store = object_store()?; + supervise_admin_mutation("config mutation", async move { + preflight_notify_config_intent(None).await?; + save_server_config_history(&body).await?; + let persisted_config = config.clone(); + let notify_transition = with_admin_server_config_write_lock(config_store, move || async move { + commit_prepared_config( + persisted_config.clone(), + prepared, + save_server_config_to_store_locked(&persisted_config), + publish_prepared_config_snapshots, + ) + .await?; + Ok::<_, S3Error>(publish_notify_config_intent(&persisted_config, None)) + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock full server config update: {}", err))??; + + reconcile_full_config(config, notify_transition).await + }) .await?; - signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await; - apply_dynamic_subsystems(&config).await; - signal_config_snapshot_reload().await; success_response(false) } @@ -1890,92 +2049,6 @@ mod tests { assert_eq!(*events.lock().expect("result events lock"), ["persist"]); } - #[test] - fn storage_config_write_handlers_persist_before_publishing() { - const SOURCE: &str = include_str!("config_admin.rs"); - - for (handler, next_handler, follow_up) in [ - ( - "SetConfigKVHandler", - "DelConfigKVHandler", - "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await", - ), - ( - "DelConfigKVHandler", - "HelpConfigKVHandler", - "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await", - ), - ( - "RestoreConfigHistoryKVHandler", - "GetConfigHandler", - "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await", - ), - ( - "SetConfigHandler", - "#[cfg(test)]", - "signal_dynamic_config_reload(STORAGE_CLASS_SUB_SYS).await", - ), - ] { - let start_marker = format!("impl Operation for {handler}"); - let start = SOURCE - .find(&start_marker) - .unwrap_or_else(|| panic!("missing {handler} implementation")); - let tail = &SOURCE[start..]; - let end = tail - .find(next_handler) - .unwrap_or_else(|| panic!("missing {next_handler} after {handler}")); - let implementation = &tail[..end]; - let prepared_commit_path = if matches!(handler, "SetConfigKVHandler" | "DelConfigKVHandler") { - let branch_start = implementation - .find("if sub_system == Some(STORAGE_CLASS_SUB_SYS)") - .unwrap_or_else(|| panic!("missing storage-class branch in {handler}")); - let branch = &implementation[branch_start..]; - let branch_end = branch - .find("} else {") - .unwrap_or_else(|| panic!("missing non-storage branch in {handler}")); - &branch[..branch_end] - } else { - implementation - }; - - assert_eq!(prepared_commit_path.matches("commit_prepared_config(").count(), 1, "{handler}"); - let commit_start = prepared_commit_path.find("commit_prepared_config(").expect("commit call"); - let commit_end = prepared_commit_path[commit_start..].find(';').expect("commit terminator") + commit_start; - let commit_statement = &prepared_commit_path[commit_start..=commit_end]; - assert!(commit_statement.contains(".await?;"), "{handler} must propagate commit failure"); - - let follow_up_start = prepared_commit_path - .find(follow_up) - .unwrap_or_else(|| panic!("missing follow-up in {handler}")); - assert!(follow_up_start > commit_end, "{handler} must run follow-up only after commit"); - assert!( - !prepared_commit_path.contains("publish_server_config("), - "{handler} must not publish directly" - ); - assert!( - !prepared_commit_path.contains(".publish_storage_class("), - "{handler} must not publish directly" - ); - - if matches!(handler, "RestoreConfigHistoryKVHandler" | "SetConfigHandler") { - let worker_start = prepared_commit_path - .find("apply_dynamic_subsystems(&config).await") - .unwrap_or_else(|| panic!("missing local worker apply in {handler}")); - let signal_start = prepared_commit_path - .find("signal_config_snapshot_reload().await") - .unwrap_or_else(|| panic!("missing snapshot signal in {handler}")); - assert!( - worker_start > follow_up_start, - "{handler} must converge peer parity before local worker apply" - ); - assert!( - signal_start > worker_start, - "{handler} must signal the full snapshot after local worker apply" - ); - } - } - } - #[test] fn tokenize_config_line_handles_quotes_and_escapes() { let tokens = tokenize_config_line(r#"identity_openid client_id="console app" client_secret="s3cr\"et" enable=on"#) diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index 3e6e2634c..56ae22d8a 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -15,6 +15,7 @@ use crate::admin::{ auth::validate_admin_request, handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot}, + handlers::supervise_admin_mutation, handlers::target_descriptor::{ AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs, build_json_response, collect_runtime_statuses, extract_supported_target_params, @@ -22,6 +23,8 @@ use crate::admin::{ target_mutation_block_reason as shared_target_mutation_block_reason, }, router::{AdminOperation, Operation, S3Router}, + runtime_sources::{AppContext, app_context_from_req}, + service::config::{preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context}, }; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ @@ -43,6 +46,45 @@ use std::sync::LazyLock; use tracing::{Span, error, info, warn}; const LOG_COMPONENT_ADMIN_API: &str = "admin_api"; + +async fn converge_target_mutation_on_cluster( + context: Option<&AppContext>, + target_type: &str, + local_result: Result<(), rustfs_notify::NotificationError>, +) -> S3Result<()> { + let subsystem = notification_target_subsystem(target_type)?; + let peer_result = signal_dynamic_config_reload_checked_for_context(context, subsystem).await; + + match (local_result, peer_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(local), Ok(())) => { + warn!(target_type, error = %local, "Local notification runtime failed to converge"); + Err(s3_error!(InternalError, "local notification runtime failed to converge")) + } + (Ok(()), Err(peer)) => Err(peer), + (Err(local), Err(peer)) => { + warn!(target_type, error = %local, "Local notification runtime failed while peer convergence also failed"); + Err(s3_error!( + InternalError, + "local notification runtime and peer convergence failed: {}", + peer + )) + } + } +} + +fn notification_target_subsystem(target_type: &str) -> S3Result<&'static str> { + notification_target_specs() + .iter() + .find(|spec| spec.subsystem == target_type) + .map(|spec| spec.subsystem) + .ok_or_else(|| s3_error!(InvalidArgument, "unsupported notification target type: {}", target_type)) +} + +async fn preflight_target_mutation_on_cluster(context: Option<&AppContext>, target_type: &str) -> S3Result<()> { + preflight_dynamic_config_reload_for_context(context, notification_target_subsystem(target_type)?).await +} + const LOG_SUBSYSTEM_NOTIFICATION_TARGET: &str = "notification_target"; const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected"; const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed"; @@ -203,6 +245,7 @@ struct NotificationEndpoint { #[derive(Serialize, Debug)] struct NotificationEndpointsResponse { + notify_enabled: bool, notification_endpoints: Vec, } @@ -229,7 +272,7 @@ async fn authorize_notification_admin_request(req: &S3Request, action: Adm validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await } -fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { +fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> S3Result> { shared_target_mutation_block_reason( notification_target_specs(), NOTIFY_ROUTE_PREFIX, @@ -256,16 +299,21 @@ async fn notification_target_operation_block_reason(action: &str) -> Option) -> Vec { - shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses) - .into_iter() - .map(|endpoint| NotificationEndpoint { - account_id: endpoint.account_id, - service: endpoint.service, - status: endpoint.status, - source: endpoint.source, - }) - .collect() +fn merge_notification_endpoints( + config: &Config, + runtime_statuses: HashMap, +) -> S3Result> { + Ok( + shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)? + .into_iter() + .map(|endpoint| NotificationEndpoint { + account_id: endpoint.account_id, + service: endpoint.service, + status: endpoint.status, + source: endpoint.source, + }) + .collect(), + ) } fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec { @@ -284,6 +332,7 @@ impl Operation for NotificationTarget { let span = Span::current(); let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; + let context = app_context_from_req(&req); authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?; if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await { @@ -291,7 +340,7 @@ impl Operation for NotificationTarget { return Err(s3_error!(InvalidRequest, "{reason}")); } let (ns, config_snapshot) = load_notification_config_snapshot().await?; - if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { + if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? { log_notification_target_operation_blocked!("set_target_config", Some(target_type), Some(target_name), &reason); return Err(s3_error!(InvalidRequest, "{reason}")); } @@ -320,7 +369,15 @@ impl Operation for NotificationTarget { ) .await?; - ns.set_target_config(target_type, target_name, kvs).await.map_err(|e| { + let mutation_target_type = target_type.to_owned(); + let mutation_target_name = target_name.to_owned(); + supervise_admin_mutation("notification target mutation", async move { + preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?; + let local_result = ns.set_target_config(&mutation_target_type, &mutation_target_name, kvs).await; + converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await + }) + .await + .map_err(|e| { log_notification_target_request_failed!( "set_target_config", "set_target_config_failed", @@ -328,7 +385,7 @@ impl Operation for NotificationTarget { Some(target_name), e ); - s3_error!(InternalError, "failed to set target config: {}", e) + e })?; log_notification_target_config_updated!("set_target_config", target_type, target_name); @@ -343,11 +400,29 @@ impl Operation for ListNotificationTargets { let span = Span::current(); let _enter = span.enter(); authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?; + refresh_persisted_module_switches_from_store().await.map_err(|err| { + warn!( + event = EVENT_ADMIN_REQUEST_FAILED, + component = LOG_COMPONENT_ADMIN_API, + subsystem = LOG_SUBSYSTEM_NOTIFICATION_TARGET, + operation = "list_targets", + result = "failed", + reason = "module_switch_refresh_failed", + error = %err, + "admin request failed" + ); + s3_error!(InternalError, "failed to refresh notification module state") + })?; + let notify_enabled = refresh_notify_module_enabled(); let (ns, config) = load_notification_config_snapshot().await?; let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await; - let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses); + let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses)?; - let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }).map_err(|e| { + let data = serde_json::to_vec(&NotificationEndpointsResponse { + notify_enabled, + notification_endpoints, + }) + .map_err(|e| { log_notification_target_request_failed!("list_targets", "serialize_targets_failed", None, None, e); s3_error!(InternalError, "failed to serialize targets: {}", e) })?; @@ -409,6 +484,7 @@ impl Operation for RemoveNotificationTarget { let span = Span::current(); let _enter = span.enter(); let (target_type, target_name) = extract_target_params(¶ms)?; + let context = app_context_from_req(&req); authorize_notification_admin_request(&req, AdminAction::SetBucketTargetAction).await?; if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await { @@ -416,12 +492,20 @@ impl Operation for RemoveNotificationTarget { return Err(s3_error!(InvalidRequest, "{reason}")); } let (ns, config_snapshot) = load_notification_config_snapshot().await?; - if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { + if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name)? { log_notification_target_operation_blocked!("remove_target_config", Some(target_type), Some(target_name), &reason); return Err(s3_error!(InvalidRequest, "{reason}")); } - ns.remove_target_config(target_type, target_name).await.map_err(|e| { + let mutation_target_type = target_type.to_owned(); + let mutation_target_name = target_name.to_owned(); + supervise_admin_mutation("notification target mutation", async move { + preflight_target_mutation_on_cluster(context.as_deref(), &mutation_target_type).await?; + let local_result = ns.remove_target_config(&mutation_target_type, &mutation_target_name).await; + converge_target_mutation_on_cluster(context.as_deref(), &mutation_target_type, local_result).await + }) + .await + .map_err(|e| { log_notification_target_request_failed!( "remove_target_config", "remove_target_config_failed", @@ -429,7 +513,7 @@ impl Operation for RemoveNotificationTarget { Some(target_name), e ); - s3_error!(InternalError, "failed to remove target config: {}", e) + e })?; log_notification_target_config_updated!("remove_target_config", target_type, target_name); @@ -464,6 +548,41 @@ mod tests { }]) } + #[test] + fn notification_target_subsystem_resolves_admin_route_type() { + assert_eq!( + notification_target_subsystem(NOTIFY_WEBHOOK_SUB_SYS).expect("webhook subsystem should resolve"), + NOTIFY_WEBHOOK_SUB_SYS + ); + assert!(notification_target_subsystem("webhook").is_err()); + } + + #[test] + fn notification_endpoints_response_includes_required_module_state() { + let response = NotificationEndpointsResponse { + notify_enabled: true, + notification_endpoints: vec![NotificationEndpoint { + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "online".to_string(), + source: TargetEndpointSource::Config, + }], + }; + + assert_eq!( + serde_json::to_value(response).expect("notification target response should serialize"), + serde_json::json!({ + "notify_enabled": true, + "notification_endpoints": [{ + "account_id": "primary", + "service": "webhook", + "status": "online", + "source": "config" + }] + }) + ); + } + #[test] fn merge_notification_endpoints_keeps_configured_targets_after_runtime_loss() { let mut cfg_map = HashMap::new(); @@ -478,7 +597,7 @@ mod tests { let config = Config(cfg_map); let runtime = HashMap::from([(("webhook-a".to_string(), "webhook".to_string()), "online".to_string())]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let mqtt = merged .iter() @@ -507,7 +626,7 @@ mod tests { (("webhook-enabled".to_string(), "webhook".to_string()), "online".to_string()), (("env-only".to_string(), "mqtt".to_string()), "offline".to_string()), ]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let env_only = merged .iter() @@ -549,7 +668,7 @@ mod tests { (("mixed-target".to_string(), "webhook".to_string()), "online".to_string()), (("env-only".to_string(), "webhook".to_string()), "online".to_string()), ]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let mixed = merged .iter() @@ -592,7 +711,7 @@ mod tests { (("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()), (("env-kafka".to_string(), "kafka".to_string()), "online".to_string()), ]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let mixed = merged .iter() @@ -629,7 +748,7 @@ mod tests { (("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()), (("env-amqp".to_string(), "amqp".to_string()), "online".to_string()), ]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let mixed = merged .iter() @@ -656,7 +775,8 @@ mod tests { ], || { let config = Config(HashMap::new()); - let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary"); + let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary") + .expect("target mutation block reason"); assert!(reason.is_some()); assert!(reason.unwrap().contains("managed by environment variables")); }, @@ -696,7 +816,8 @@ mod tests { NOTIFY_WEBHOOK_SUB_SYS.to_string(), HashMap::from([("primary".to_string(), enabled_kvs("on"))]), )])); - let reason = target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary"); + let reason = + target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primary").expect("target mutation block reason"); assert!(reason.is_some()); assert!(reason.unwrap().contains("both persisted config and environment variables")); }); @@ -709,7 +830,11 @@ mod tests { NOTIFY_WEBHOOK_SUB_SYS.to_string(), HashMap::from([(target_name.to_string(), enabled_kvs("on"))]), )])); - assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name).is_none()); + assert!( + target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, target_name) + .expect("target mutation block reason") + .is_none() + ); } #[test] @@ -726,7 +851,7 @@ mod tests { ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MIXED-DISABLED", Some("https://example.com/hook")), ], || { - let merged = merge_notification_endpoints(&config, HashMap::new()); + let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints"); let mixed = merged .iter() .find(|entry| entry.account_id == "mixed-disabled") @@ -748,7 +873,7 @@ mod tests { ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), ], || { - let merged = merge_notification_endpoints(&config, HashMap::new()); + let merged = merge_notification_endpoints(&config, HashMap::new()).expect("merge notification endpoints"); let env_only = merged .iter() .find(|entry| entry.account_id == "env-only") @@ -798,7 +923,7 @@ mod tests { ], || { let runtime = HashMap::from([(("PrimaryCase".to_string(), "webhook".to_string()), "online".to_string())]); - let merged = merge_notification_endpoints(&config, runtime); + let merged = merge_notification_endpoints(&config, runtime).expect("merge notification endpoints"); let mixed = merged .iter() .find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook") @@ -835,7 +960,11 @@ mod tests { ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARYCASE", None::<&str>), ], || { - assert!(target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase").is_none()); + assert!( + target_mutation_block_reason(&config, NOTIFY_WEBHOOK_SUB_SYS, "primarycase") + .expect("target mutation block reason") + .is_none() + ); }, ); } @@ -864,6 +993,22 @@ mod tests { list_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"), "notification target list should require GetBucketTargetAction" ); + let authorize_index = list_block + .find("authorize_notification_admin_request") + .expect("target list should authorize the request"); + let refresh_index = list_block + .find("refresh_persisted_module_switches_from_store().await.map_err") + .expect("target list should fail when persisted module state cannot be refreshed"); + let effective_state_index = list_block + .find("let notify_enabled = refresh_notify_module_enabled();") + .expect("target list should resolve the effective env and persisted module state"); + let load_index = list_block + .find("load_notification_config_snapshot().await?") + .expect("target list should load notification config"); + assert!( + authorize_index < refresh_index && refresh_index < effective_state_index && effective_state_index < load_index, + "target list should authorize, refresh persisted state, resolve effective state, then load targets" + ); assert!( arns_block.contains("authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;"), "notification target arn listing should require GetBucketTargetAction" diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 7ad4723cb..0e8415e78 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -63,6 +63,19 @@ pub mod user_iam; pub mod user_lifecycle; pub mod user_policy_binding; +pub(crate) async fn supervise_admin_mutation( + operation: &'static str, + mutation: impl std::future::Future> + Send + 'static, +) -> s3s::S3Result +where + T: Send + 'static, +{ + tokio::spawn(mutation).await.map_err(|err| { + let outcome = if err.is_cancelled() { "cancelled" } else { "panicked" }; + s3s::s3_error!(InternalError, "{} task {}", operation, outcome) + })? +} + #[cfg(test)] mod tests { use super::*; @@ -114,6 +127,44 @@ mod tests { // Test passes if we reach this point without panicking } + #[tokio::test] + async fn supervised_admin_mutation_survives_waiter_cancellation() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); + + let waiter = tokio::spawn(async move { + supervise_admin_mutation("test mutation", async move { + let _ = started_tx.send(()); + let _ = release_rx.await; + let _ = completed_tx.send(()); + Ok(()) + }) + .await + }); + + started_rx.await.expect("mutation started"); + waiter.abort(); + release_tx.send(()).expect("release mutation"); + tokio::time::timeout(std::time::Duration::from_secs(1), completed_rx) + .await + .expect("detached mutation should complete") + .expect("completion signal"); + } + + #[tokio::test] + async fn supervised_admin_mutation_does_not_expose_panic_payload() { + let error = supervise_admin_mutation::<()>("test mutation", async { + panic!("do-not-expose-payload"); + }) + .await + .expect_err("panicking mutation should fail"); + + let rendered = error.to_string(); + assert!(rendered.contains("panicked")); + assert!(!rendered.contains("do-not-expose-payload")); + } + // Note: Testing the actual async handler implementations requires: // 1. S3Request setup with proper headers, URI, and credentials // 2. Global object store initialization diff --git a/rustfs/src/admin/handlers/module_switch.rs b/rustfs/src/admin/handlers/module_switch.rs index 44fbdb990..b5d8f4d5b 100644 --- a/rustfs/src/admin/handlers/module_switch.rs +++ b/rustfs/src/admin/handlers/module_switch.rs @@ -12,26 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::admin::runtime_sources::default_admin_usecase; +use crate::admin::runtime_sources::{AppContext, app_context_from_req, default_admin_usecase}; +use crate::admin::service::config::{ + preflight_dynamic_config_reload_for_context, signal_dynamic_config_reload_checked_for_context, +}; use crate::admin::{ auth::validate_admin_request, + handlers::supervise_admin_mutation, router::{AdminOperation, Operation, S3Router}, }; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ - ADMIN_PREFIX, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, RemoteAddr, current_module_switch_snapshot, - init_event_notifier, refresh_audit_module_enabled, refresh_notify_module_enabled, - refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, shutdown_event_notifier, - start_audit_system, stop_audit_system, validate_module_switch_update, + ADMIN_PREFIX, MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, + RemoteAddr, apply_audit_module_switch_for_context, current_module_switch_snapshot, mark_event_notifier_reconciled, + mark_event_notifier_unreconciled, refresh_audit_module_enabled, refresh_notify_module_enabled, + refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, save_persisted_module_switches_to, + validate_module_switch_update, }; use http::{HeaderMap, StatusCode}; use hyper::Method; use matchit::Params; -use rustfs_audit::AuditError; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::{Deserialize, Serialize}; +use std::sync::Arc; pub fn register_module_switch_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -139,6 +144,74 @@ async fn refresh_module_switch_snapshot() -> S3Result { Ok(current_module_switch_snapshot()) } +async fn apply_module_switch_update(context: Arc, switches: PersistedModuleSwitches) -> S3Result<()> { + preflight_dynamic_config_reload_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await?; + let store = context.object_store(); + mark_event_notifier_unreconciled(); + let notification_system = rustfs_notify::ensure_live_events(); + if switches.notify_enabled { + notification_system + .reload_persisted_config_from_store(store.clone()) + .await + .map_err(|err| { + tracing::warn!(error = %err, "Failed to load notification config for module switch update"); + s3_error!(InternalError, "failed to load notification config") + })?; + } + + let transition_system = notification_system.clone(); + let notify_transition = save_persisted_module_switches_to(store.clone(), switches, move || { + let enabled = refresh_notify_module_enabled(); + transition_system.publish_targets_enabled(enabled, None) + }) + .await + .map_err(|err| { + tracing::warn!(error = %err, "Failed to save module switches"); + s3_error!(InternalError, "failed to save module switches") + })?; + + let mut failures = Vec::new(); + let mut notify_converged = true; + if let Err(err) = notify_transition.wait().await { + tracing::warn!(error = %err, "Local notification runtime failed to apply module switch update"); + notify_converged = false; + failures.push("local notify"); + } + if !switches.notify_enabled + && let Err(err) = notification_system.reload_persisted_config_from_store(store).await + { + tracing::warn!(error = %err, "Local notification config cache failed to reload after module disable"); + notify_converged = false; + failures.push("local notify config cache"); + } + if notify_converged && notification_system.runtime_lifecycle_is_converged() { + mark_event_notifier_reconciled(); + } else if notify_converged { + failures.push("local notify convergence"); + } + + if apply_audit_module_switch_for_context(Some(context.as_ref())).await.is_err() { + tracing::warn!(reason = "apply_failed", "Local audit runtime failed to apply module switch update"); + failures.push("local audit"); + } + if let Err(err) = + signal_dynamic_config_reload_checked_for_context(Some(context.as_ref()), MODULE_SWITCHES_SIGNAL_SUBSYSTEM).await + { + tracing::warn!(error = %err, "Peer nodes failed to apply module switch update"); + failures.push("peer module switches"); + } + + if failures.is_empty() { + Ok(()) + } else { + Err(s3_error!( + InternalError, + "module switches persisted but runtime convergence failed: {}", + failures.join("; ") + )) + } +} + pub struct GetModuleSwitchesHandler {} #[async_trait::async_trait] @@ -156,7 +229,9 @@ pub struct UpdateModuleSwitchesHandler {} impl Operation for UpdateModuleSwitchesHandler { async fn call(&self, mut req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?; - refresh_persisted_module_switches_from_store() + let context = app_context_from_req(&req).ok_or_else(|| s3_error!(InternalError, "storage layer not initialized"))?; + let store = context.object_store(); + refresh_persisted_module_switches_from(store.clone()) .await .map_err(|e| s3_error!(InternalError, "failed to reload persisted module switches: {}", e))?; @@ -183,28 +258,7 @@ impl Operation for UpdateModuleSwitchesHandler { return Err(s3_error!(InvalidRequest, "{err}")); } - save_persisted_module_switches_to_store(switches) - .await - .map_err(|e| s3_error!(InternalError, "failed to save module switches: {}", e))?; - - // Apply the new effective values immediately on this node so the console - // response reflects the runtime state after to write completes. - if refresh_notify_module_enabled() { - init_event_notifier().await; - } else { - shutdown_event_notifier().await; - } - - if refresh_audit_module_enabled() { - match start_audit_system().await { - Ok(()) | Err(AuditError::AlreadyInitialized) => {} - Err(e) => return Err(s3_error!(InternalError, "failed to apply audit module switch: {}", e)), - } - } else { - stop_audit_system() - .await - .map_err(|e| s3_error!(InternalError, "failed to stop audit module after switch update: {}", e))?; - } + supervise_admin_mutation("module switch update", apply_module_switch_update(context, switches)).await?; let snapshot = current_module_switch_snapshot(); build_response(StatusCode::OK, &ModuleSwitchesResponse::from(snapshot), req.headers.get("x-request-id")) @@ -233,6 +287,12 @@ mod tests { put_block.contains("authorize_module_switch_request(&req, AdminAction::ConfigUpdateAdminAction).await?;"), "module switch PUT should require ConfigUpdateAdminAction" ); + assert!( + put_block.contains( + "supervise_admin_mutation(\"module switch update\", apply_module_switch_update(context, switches)).await?;" + ), + "module switch PUT must delegate the complete mutation to its supervisor" + ); } #[test] diff --git a/rustfs/src/admin/handlers/notify_runtime_access.rs b/rustfs/src/admin/handlers/notify_runtime_access.rs index e76de9a70..cbaf5572d 100644 --- a/rustfs/src/admin/handlers/notify_runtime_access.rs +++ b/rustfs/src/admin/handlers/notify_runtime_access.rs @@ -12,31 +12,33 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::server::init_event_notifier; +use crate::server::{init_event_notifier, is_event_notifier_reconciled}; use rustfs_config::server_config::Config; use s3s::{S3Result, s3_error}; use std::sync::Arc; -use tokio::sync::Mutex; - -static NOTIFICATION_SYSTEM_INIT_LOCK: Mutex<()> = Mutex::const_new(()); pub(crate) async fn get_notification_system() -> S3Result> { - if let Some(system) = rustfs_notify::notification_system() { + if is_event_notifier_reconciled() + && let Some(system) = rustfs_notify::notification_system() + && system.runtime_lifecycle_is_converged() + { return Ok(system); } - let _guard = NOTIFICATION_SYSTEM_INIT_LOCK.lock().await; - if let Some(system) = rustfs_notify::notification_system() { - return Ok(system); + init_event_notifier() + .await + .map_err(|err| s3_error!(InternalError, "failed to reconcile notification runtime: {}", err))?; + let system = + rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))?; + if !system.runtime_lifecycle_is_converged() { + return Err(s3_error!(InternalError, "latest notification lifecycle generation has not converged")); } - - init_event_notifier().await; - rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized")) + Ok(system) } pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc, Config)> { let system = get_notification_system().await?; - let config = system.config.read().await.clone(); + let config = system.config_snapshot().await; Ok((system, config)) } diff --git a/rustfs/src/admin/handlers/oidc.rs b/rustfs/src/admin/handlers/oidc.rs index 9e991223f..b94d796f6 100644 --- a/rustfs/src/admin/handlers/oidc.rs +++ b/rustfs/src/admin/handlers/oidc.rs @@ -19,7 +19,10 @@ use crate::admin::runtime_sources::{ current_server_config_for_context, }; use crate::admin::service::federated_identity::DefaultFederatedSessionBinding; -use crate::admin::storage_api::config::{read_admin_config_without_migrate, save_admin_server_config}; +use crate::admin::storage_api::config::{ + read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config_no_lock, + with_admin_server_config_write_lock, +}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, CONSOLE_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr}; use http::StatusCode; @@ -368,13 +371,20 @@ impl Operation for PutOidcConfigHandler { if is_env_managed_provider(provider_id) { return Err(s3_error!(AccessDenied, "provider is managed by environment variables")); } + let provider_id = provider_id.to_owned(); let request: OidcConfigUpsertRequest = parse_json_body(&mut req).await?; - let mut config = load_server_config_from_store().await?; - let existing_secret = persisted_provider_secret(&config, provider_id); - let provider_config = build_provider_config_from_upsert(provider_id, request, existing_secret)?; - upsert_persisted_provider_config(&mut config, &provider_config); - save_server_config_to_store(&config).await?; + let store = oidc_config_store()?; + let lock_store = store.clone(); + with_admin_server_config_write_lock(lock_store, move || async move { + let mut config = load_server_config_from_store_locked(store.clone()).await?; + let existing_secret = persisted_provider_secret(&config, &provider_id); + let provider_config = build_provider_config_from_upsert(&provider_id, request, existing_secret)?; + upsert_persisted_provider_config(&mut config, &provider_config); + save_server_config_to_store_locked(store, &config).await + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; json_response( StatusCode::OK, @@ -403,10 +413,17 @@ impl Operation for DeleteOidcConfigHandler { if is_env_managed_provider(provider_id) { return Err(s3_error!(AccessDenied, "provider is managed by environment variables")); } + let provider_id = provider_id.to_owned(); - let mut config = load_server_config_from_store().await?; - delete_persisted_provider_config(&mut config, provider_id)?; - save_server_config_to_store(&config).await?; + let store = oidc_config_store()?; + let lock_store = store.clone(); + with_admin_server_config_write_lock(lock_store, move || async move { + let mut config = load_server_config_from_store_locked(store.clone()).await?; + delete_persisted_provider_config(&mut config, &provider_id)?; + save_server_config_to_store_locked(store, &config).await + }) + .await + .map_err(|err| s3_error!(InternalError, "failed to lock server config update: {}", err))??; json_response( StatusCode::OK, @@ -881,23 +898,32 @@ fn json_response(status: StatusCode, payload: &T) -> S3Result S3Result { - let context = current_app_context(); - let Some(store) = current_object_store_handle_for_context(context.as_deref()) else { - return Err(s3_error!(InternalError, "storage layer not initialized")); - }; + let store = oidc_config_store()?; read_admin_config_without_migrate(store) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}"))) } -async fn save_server_config_to_store(config: &ServerConfig) -> S3Result<()> { +fn oidc_config_store() -> S3Result> { let context = current_app_context(); - let Some(store) = current_object_store_handle_for_context(context.as_deref()) else { - return Err(s3_error!(InternalError, "storage layer not initialized")); - }; + current_object_store_handle_for_context(context.as_deref()) + .ok_or_else(|| s3_error!(InternalError, "storage layer not initialized")) +} - save_admin_server_config(store, config) +async fn load_server_config_from_store_locked( + store: std::sync::Arc, +) -> S3Result { + read_admin_config_without_migrate_no_lock(store) + .await + .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to load server config: {e}"))) +} + +async fn save_server_config_to_store_locked( + store: std::sync::Arc, + config: &ServerConfig, +) -> S3Result<()> { + save_admin_server_config_no_lock(store, config) .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("failed to save server config: {e}"))) } diff --git a/rustfs/src/admin/handlers/plugins_instances.rs b/rustfs/src/admin/handlers/plugins_instances.rs index 3e3d9e478..70e634a27 100644 --- a/rustfs/src/admin/handlers/plugins_instances.rs +++ b/rustfs/src/admin/handlers/plugins_instances.rs @@ -605,7 +605,7 @@ fn plugin_instance_mutation_block_reason( target_type: &str, target_name: &str, target_label: &str, -) -> Option { +) -> S3Result> { shared_target_mutation_block_reason(context.specs, context.route_prefix, config, target_type, target_name, target_label) } @@ -667,7 +667,7 @@ async fn collect_domain_instances(context: PluginInstanceDomainContext) -> S3Res let config = plugin_instance_config_snapshot(context).await?; let module_disabled_reason = module_disabled_block_reason(context.domain, "listing plugin instances"); let mut entries = Vec::new(); - for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses) { + for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses)? { entries.push(plugin_instance_list_entry(instance, module_disabled_reason.clone())); } Ok(entries) @@ -688,13 +688,7 @@ async fn find_plugin_instance(instance_id: &str) -> S3Result TargetEndpointSource { - let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); +) -> S3Result { + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?; let service = target_service_name(specs, target_type).unwrap_or_default(); let key = normalized_endpoint_key(target_name, service); - classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key) + Ok(classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)) } pub(crate) fn target_mutation_block_reason( @@ -237,8 +237,8 @@ pub(crate) fn target_mutation_block_reason( target_type: &str, target_name: &str, target_label: &str, -) -> Option { - match endpoint_source(specs, route_prefix, config, target_type, target_name) { +) -> S3Result> { + Ok(match endpoint_source(specs, route_prefix, config, target_type, target_name)? { TargetEndpointSource::Env => Some(format!( "{} '{}' is managed by environment variables and cannot be modified from the console", target_label, target_name @@ -248,7 +248,7 @@ pub(crate) fn target_mutation_block_reason( target_label, target_name )), TargetEndpointSource::Config | TargetEndpointSource::Runtime => None, - } + }) } pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, enabled: bool, action: &str) -> Option { @@ -304,10 +304,10 @@ pub(crate) fn merge_target_endpoints( route_prefix: &str, config: &Config, runtime_statuses: HashMap, -) -> Vec { +) -> S3Result> { let mut endpoints = Vec::new(); let mut seen = HashSet::new(); - let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?; let mut normalized_runtime_statuses: HashMap = HashMap::new(); for ((account_id, service), status) in runtime_statuses { @@ -361,7 +361,7 @@ pub(crate) fn merge_target_endpoints( } endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); - endpoints + Ok(endpoints) } pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String { @@ -373,12 +373,12 @@ pub(crate) fn collect_target_instances( route_prefix: &str, config: &Config, runtime_statuses: HashMap, -) -> Vec { +) -> S3Result> { let mut instances = Vec::new(); let mut seen = HashSet::new(); let mut normalized_runtime_statuses: HashMap = HashMap::new(); let domain = inferred_target_domain(route_prefix); - let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config)?; for ((account_id, service), status) in runtime_statuses { let normalized = normalized_endpoint_key(&account_id, &service); @@ -439,7 +439,7 @@ pub(crate) fn collect_target_instances( } instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); - instances + Ok(instances) } pub(crate) fn find_target_instance( @@ -448,10 +448,10 @@ pub(crate) fn find_target_instance( config: &Config, runtime_statuses: HashMap, canonical_id: &str, -) -> Option { - collect_target_instances(specs, route_prefix, config, runtime_statuses) +) -> S3Result> { + Ok(collect_target_instances(specs, route_prefix, config, runtime_statuses)? .into_iter() - .find(|instance| instance.canonical_id == canonical_id) + .find(|instance| instance.canonical_id == canonical_id)) } pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> { @@ -559,11 +559,11 @@ fn normalized_target_instances( specs: &[AdminTargetSpec], route_prefix: &str, config: &Config, -) -> Vec { - specs - .iter() - .flat_map(|spec| { - normalize_target_plugin_instances( +) -> S3Result> { + let mut instances = Vec::new(); + for spec in specs { + instances.extend( + try_normalize_target_plugin_instances( config, &TargetPluginInstanceCompatDescriptor { domain: inferred_target_domain(route_prefix), @@ -574,8 +574,10 @@ fn normalized_target_instances( valid_fields: spec.valid_keys, }, ) - }) - .collect() + .map_err(|err| s3_error!(InvalidRequest, "invalid {} target environment: {}", spec.service, err))?, + ); + } + Ok(instances) } fn inferred_target_domain(route_prefix: &str) -> TargetDomain { @@ -597,8 +599,8 @@ fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Op specs.iter().find(|spec| spec.service == service) } -fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot { - let normalized_instances = normalized_target_instances(specs, route_prefix, config); +fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> S3Result { + let normalized_instances = normalized_target_instances(specs, route_prefix, config)?; let mut configured_keys = Vec::new(); let mut config_targets = HbHashSet::new(); let mut env_targets = HbHashSet::new(); @@ -618,12 +620,12 @@ fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, conf } } - TargetEndpointSnapshot { + Ok(TargetEndpointSnapshot { normalized_instances, configured_keys, config_targets, env_targets, - } + }) } async fn retry_with_backoff(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index e4c004381..6aa3083af 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -628,7 +628,7 @@ fn resolve_object_lambda_webhook_config_from_server_config( async fn load_current_server_config() -> S3Result { if let Some(system) = notification_system() { - return Ok(system.config.read().await.clone()); + return Ok(system.config_snapshot().await); } if let Some(store) = current_object_store_handle() { diff --git a/rustfs/src/admin/service/config.rs b/rustfs/src/admin/service/config.rs index e2dc44ecc..ab676aa6b 100644 --- a/rustfs/src/admin/service/config.rs +++ b/rustfs/src/admin/service/config.rs @@ -19,18 +19,21 @@ use crate::admin::runtime_sources::{ use crate::admin::storage_api::config::{STORAGE_CLASS_SUB_SYS, read_admin_config_without_migrate, storageclass}; use crate::admin::storage_api::contract::admin::StorageAdminApi; use crate::admin::storage_api::runtime::ECStore; +use crate::server::{ + MODULE_SWITCHES_SIGNAL_SUBSYSTEM, apply_audit_module_switch_for_context, reconcile_event_notifier_from_store, +}; use rustfs_audit::reload_audit_config; +use rustfs_config::AUDIT_DEFAULT_DIR; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS}; -use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_WEBHOOK_SUB_SYS}; +use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_SUB_SYSTEMS}; use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS; use rustfs_config::server_config::{Config as ServerConfig, KVS}; -use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR}; use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState}; use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_iam::oidc::load_oidc_provider_configs_from_server_config; use rustfs_targets::config::{ - validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, validate_nats_config, - validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, + try_collect_target_configs, validate_amqp_config, validate_kafka_config, validate_mqtt_config, validate_mysql_config, + validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config, }; use s3s::{S3Error, S3ErrorCode, S3Result}; use std::future::Future; @@ -38,10 +41,11 @@ use tracing::warn; use url::Url; pub fn is_dynamic_config_subsystem(sub_system: &str) -> bool { - matches!( - sub_system, - STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS - ) + NOTIFY_SUB_SYSTEMS.contains(&sub_system) + || matches!( + sub_system, + STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS + ) } pub(crate) const FULL_CONFIG_WORKER_SUBSYSTEMS: [&str; 2] = [AUDIT_WEBHOOK_SUB_SYS, SCANNER_SUB_SYS]; @@ -179,30 +183,20 @@ fn target_kvs(config: &ServerConfig, sub_system: &str, target: &str) -> KVS { } fn validate_notify_subsystem_config(config: &ServerConfig, sub_system: &str) -> S3Result<()> { - let Some(targets) = config.0.get(sub_system) else { + let Some(descriptor) = rustfs_notify::factory::builtin_target_descriptors() + .into_iter() + .find(|descriptor| descriptor.subsystem() == sub_system) + else { return Ok(()); }; + let plugin = descriptor.plugin(); - for target in targets.keys() { - let kvs = target_kvs(config, sub_system, target); - if !target_enabled(&kvs) { - continue; - } - - let result = match sub_system { - "notify_webhook" => validate_webhook_config(&kvs, EVENT_DEFAULT_DIR), - "notify_amqp" => validate_amqp_config(&kvs, EVENT_DEFAULT_DIR), - "notify_kafka" => validate_kafka_config(&kvs, EVENT_DEFAULT_DIR), - "notify_mqtt" => validate_mqtt_config(&kvs), - "notify_mysql" => validate_mysql_config(&kvs, EVENT_DEFAULT_DIR), - "notify_nats" => validate_nats_config(&kvs, EVENT_DEFAULT_DIR), - "notify_postgres" => validate_postgres_config(&kvs, EVENT_DEFAULT_DIR), - "notify_pulsar" => validate_pulsar_config(&kvs, EVENT_DEFAULT_DIR), - "notify_redis" => validate_redis_config(&kvs, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL), - _ => return Ok(()), - }; - - result.map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?; + for (target, kvs) in try_collect_target_configs(config, NOTIFY_ROUTE_PREFIX, plugin.target_type(), plugin.valid_fields_set()) + .map_err(|err| invalid_request(format!("invalid {sub_system} config: {err}")))? + { + plugin + .validate_config(&kvs) + .map_err(|err| invalid_request(format!("invalid {sub_system} config for target '{target}': {err}")))?; } Ok(()) @@ -314,8 +308,7 @@ pub(crate) async fn prepare_server_config_for_context( Some(STORAGE_CLASS_SUB_SYS) => { prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?); } - Some(NOTIFY_WEBHOOK_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?, - Some(NOTIFY_MQTT_SUB_SYS) => validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?, + Some(sub_system) if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => validate_notify_subsystem_config(config, sub_system)?, Some(AUDIT_WEBHOOK_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?, Some(AUDIT_MQTT_SUB_SYS) => validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?, Some(IDENTITY_OPENID_SUB_SYS) => validate_identity_openid_config(config)?, @@ -324,8 +317,9 @@ pub(crate) async fn prepare_server_config_for_context( Some(_) => {} None => { prepared.storage_class = Some(prepare_storage_class_runtime_config_for_context(context, config).await?); - validate_notify_subsystem_config(config, NOTIFY_WEBHOOK_SUB_SYS)?; - validate_notify_subsystem_config(config, NOTIFY_MQTT_SUB_SYS)?; + for sub_system in NOTIFY_SUB_SYSTEMS { + validate_notify_subsystem_config(config, sub_system)?; + } validate_audit_subsystem_config(config, AUDIT_WEBHOOK_SUB_SYS)?; validate_audit_subsystem_config(config, AUDIT_MQTT_SUB_SYS)?; validate_identity_openid_config(config)?; @@ -371,6 +365,18 @@ pub async fn apply_dynamic_config_for_subsystem_for_context( AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS => reload_audit_config(config.clone()) .await .map_err(|err| internal_error(format!("failed to reload audit config: {err}")))?, + sub_system if NOTIFY_SUB_SYSTEMS.contains(&sub_system) => { + // The notify lifecycle is intentionally process-wide. The explicit + // store keeps the persisted snapshot bound to the request context; + // the ECStore NotificationSys resolved below is the peer-broadcast + // service, not the notify target runtime. + let system = rustfs_notify::ensure_live_events(); + let store = resolve_runtime_config_store_for_context(context)?; + system + .reload_persisted_config_from_store(store) + .await + .map_err(|err| internal_error(format!("failed to reload notification config: {err}")))?; + } SCANNER_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config) .map_err(|err| internal_error(format!("failed to reload scanner config: {err}")))?, HEAL_SUB_SYS => rustfs_scanner::apply_scanner_runtime_config(config) @@ -387,6 +393,18 @@ pub async fn apply_dynamic_config_for_subsystem(config: &ServerConfig, sub_syste } pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { + if sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM { + let store = resolve_runtime_config_store_for_context(context)?; + let notify_result = reconcile_event_notifier_from_store(store).await; + let audit_result = apply_audit_module_switch_for_context(context).await; + return match (notify_result, audit_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(_), Ok(())) => Err(internal_error("failed to reconcile notification module switch")), + (Ok(()), Err(_)) => Err(internal_error("failed to reconcile audit module switch")), + (Err(_), Err(_)) => Err(internal_error("failed to reconcile notification and audit module switches")), + }; + } + if !is_dynamic_config_subsystem(sub_system) { return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}"))); } @@ -398,9 +416,8 @@ pub async fn reload_dynamic_config_runtime_state_for_context(context: Option<&Ap })?; apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) .await - .map_err(|err| { - warn!("peer reload_dynamic_config: failed to apply {sub_system}: {err}"); - err + .inspect_err(|_| { + warn!(config_subsystem = sub_system, reason = "apply_failed", "Peer dynamic config apply failed"); })?; Ok(()) } @@ -428,19 +445,19 @@ where let (config, prepared) = prepare(config).await?; publish(&config, prepared)?; - // Worker reloads mutate live state and have no rollback contract. They are - // therefore best-effort after the validated storage/server snapshots are - // published; a transient worker failure must not leave this peer on stale - // erasure geometry. + // Worker reloads mutate live state and have no rollback contract, so the + // validated snapshots stay published. The RPC still reports convergence + // failure explicitly so the originating Admin request cannot claim success. if let Err(err) = apply_workers(config).await { warn!( event = EVENT_CONFIG_WORKER_RELOAD_FAILED, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_CONFIG, state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - error = ?err, + reason = "apply_failed", "Runtime config snapshot was published but a worker reload failed" ); + return Err(err); } Ok(()) } @@ -468,20 +485,29 @@ pub async fn reload_runtime_config_snapshot_for_context(context: Option<&AppCont Ok(()) }, |config| async move { + let mut failures = Vec::new(); for sub_system in FULL_CONFIG_WORKER_SUBSYSTEMS { - if let Err(err) = apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system).await { + if apply_dynamic_config_for_subsystem_for_context(context, &config, sub_system) + .await + .is_err() + { + failures.push(sub_system); warn!( event = EVENT_CONFIG_WORKER_RELOAD_FAILED, component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_CONFIG, config_subsystem = sub_system, state = CONFIG_WORKER_RELOAD_FAILURE_STATE, - error = ?err, + reason = "apply_failed", "Peer runtime config snapshot was published but a subsystem worker reload failed" ); } } - Ok(()) + if failures.is_empty() { + Ok(()) + } else { + Err(internal_error(format!("runtime worker reload failed: {}", failures.join("; ")))) + } }, ) .await @@ -492,20 +518,77 @@ pub async fn reload_runtime_config_snapshot() -> S3Result<()> { reload_runtime_config_snapshot_for_context(context.as_deref()).await } -pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) { - if !is_dynamic_config_subsystem(sub_system) { - return; +pub async fn preflight_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { + if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) { + return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}"))); } let Some(notification_sys) = current_notification_system_for_context(context) else { - return; + return Ok(()); }; - for failure in notification_sys.reload_dynamic_config(sub_system).await { - if let Some(err) = failure.err { - tracing::warn!("peer {} dynamic config reload for {} failed: {}", failure.host, sub_system, err); + let mut failed = 0usize; + for failure in notification_sys.preflight_dynamic_config(sub_system).await { + if failure.err.is_some() { + failed += 1; + let host = if failure.host.is_empty() { "" } else { &failure.host }; + warn!( + peer = host, + config_subsystem = sub_system, + reason = "unsupported", + "Peer does not support dynamic config convergence" + ); } } + + if failed == 0 { + Ok(()) + } else { + Err(internal_error(format!( + "{failed} peer(s) do not support dynamic config convergence for {sub_system}" + ))) + } +} + +pub async fn preflight_dynamic_config_reload(sub_system: &str) -> S3Result<()> { + let context = current_app_context(); + preflight_dynamic_config_reload_for_context(context.as_deref(), sub_system).await +} + +pub async fn signal_dynamic_config_reload_checked_for_context(context: Option<&AppContext>, sub_system: &str) -> S3Result<()> { + if sub_system != MODULE_SWITCHES_SIGNAL_SUBSYSTEM && !is_dynamic_config_subsystem(sub_system) { + return Err(internal_error(format!("unsupported dynamic config subsystem: {sub_system}"))); + } + + let Some(notification_sys) = current_notification_system_for_context(context) else { + return Ok(()); + }; + + let mut failed = 0usize; + for failure in notification_sys.reload_dynamic_config(sub_system).await { + if failure.err.is_some() { + failed += 1; + let host = if failure.host.is_empty() { "" } else { &failure.host }; + warn!( + peer = host, + config_subsystem = sub_system, + reason = "reload_failed", + "Peer dynamic config convergence failed" + ); + } + } + + if failed == 0 { + Ok(()) + } else { + Err(internal_error(format!("{failed} peer(s) failed dynamic config reload for {sub_system}"))) + } +} + +pub async fn signal_dynamic_config_reload_for_context(context: Option<&AppContext>, sub_system: &str) { + if let Err(err) = signal_dynamic_config_reload_checked_for_context(context, sub_system).await { + tracing::warn!("peer dynamic config reload for {} failed: {}", sub_system, err); + } } pub async fn signal_dynamic_config_reload(sub_system: &str) { @@ -513,16 +596,36 @@ pub async fn signal_dynamic_config_reload(sub_system: &str) { signal_dynamic_config_reload_for_context(context.as_deref(), sub_system).await; } -pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) { +pub async fn signal_dynamic_config_reload_checked(sub_system: &str) -> S3Result<()> { + let context = current_app_context(); + signal_dynamic_config_reload_checked_for_context(context.as_deref(), sub_system).await +} + +pub async fn signal_config_snapshot_reload_checked_for_context(context: Option<&AppContext>) -> S3Result<()> { let Some(notification_sys) = current_notification_system_for_context(context) else { - return; + return Ok(()); }; + let mut failed = 0usize; for failure in notification_sys.refresh_config_snapshot().await { - if let Some(err) = failure.err { - tracing::warn!("peer config snapshot refresh failed for {}: {}", failure.host, err); + if failure.err.is_some() { + failed += 1; + let host = if failure.host.is_empty() { "" } else { &failure.host }; + warn!(peer = host, reason = "reload_failed", "Peer config snapshot refresh failed"); } } + + if failed == 0 { + Ok(()) + } else { + Err(internal_error(format!("{failed} peer(s) failed config snapshot reload"))) + } +} + +pub async fn signal_config_snapshot_reload_for_context(context: Option<&AppContext>) { + if let Err(err) = signal_config_snapshot_reload_checked_for_context(context).await { + tracing::warn!("peer config snapshot refresh failed: {err}"); + } } pub async fn signal_config_snapshot_reload() { @@ -530,30 +633,46 @@ pub async fn signal_config_snapshot_reload() { signal_config_snapshot_reload_for_context(context.as_deref()).await; } +pub async fn signal_config_snapshot_reload_checked() -> S3Result<()> { + let context = current_app_context(); + signal_config_snapshot_reload_checked_for_context(context.as_deref()).await +} + #[cfg(test)] mod tests { use super::*; use crate::admin::runtime_sources::{IamInterface, KmsInterface, ServerConfigInterface, StorageClassInterface}; use crate::admin::storage_api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; - use crate::admin::storage_api::config::save_admin_server_config; + use crate::admin::storage_api::config::{ + read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_server_config, + save_admin_server_config_no_lock, with_admin_server_config_write_lock, + }; + use crate::admin::storage_api::error::StorageError; + use crate::server::{ + ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, is_event_notifier_reconciled, + refresh_persisted_module_switches_from, save_persisted_module_switches_to, + }; use crate::storage_api::cluster::{Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use crate::storage_api::startup::storage::{init_local_disks_with_instance_ctx, new_instance_ctx}; - use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; + use rustfs_config::notify::{ENV_NOTIFY_WEBHOOK_ENABLE, ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_WEBHOOK_SUB_SYS}; use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES}; use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_config::{MQTT_BROKER, MQTT_QUEUE_DIR, MQTT_TOPIC, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_kms::KmsServiceManager; use std::collections::HashMap; + use std::future::{Future, poll_fn}; use std::path::Path; use std::sync::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }; + use std::task::Poll; use tempfile::TempDir; use tokio_util::sync::CancellationToken; const LIFECYCLE_RELOAD_LABEL: &str = "lifecycle"; + const REAL_STORE_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); const REPLICATION_RELOAD_LABEL: &str = "replication"; fn without_storage_class_env(f: impl FnOnce() -> R) -> R { @@ -748,6 +867,139 @@ mod tests { } } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_real_store_config_rmw_preserves_notify_and_oidc_updates() { + let temp_dir = TempDir::new().expect("server config RMW temp dir"); + let store = build_isolated_heterogeneous_store(temp_dir.path()).await; + save_admin_server_config(store.clone(), &ServerConfig::new()) + .await + .expect("persist baseline server config"); + + let notify_entered = Arc::new(tokio::sync::Notify::new()); + let release_notify = Arc::new(tokio::sync::Notify::new()); + let (oidc_polled_tx, oidc_polled_rx) = tokio::sync::oneshot::channel(); + let (oidc_entered_tx, mut oidc_entered_rx) = tokio::sync::oneshot::channel(); + let transaction_order = Arc::new(Mutex::new(Vec::new())); + + let notify_task = { + let store = store.clone(); + let notify_entered = notify_entered.clone(); + let release_notify = release_notify.clone(); + let transaction_order = transaction_order.clone(); + tokio::spawn(async move { + let transaction_store = store.clone(); + with_admin_server_config_write_lock(store, move || async move { + let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?; + transaction_order.lock().expect("transaction order lock").push("notify-read"); + notify_entered.notify_one(); + release_notify.notified().await; + + let mut target = KVS::new(); + target.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + target.insert(WEBHOOK_ENDPOINT.to_string(), "https://notify.example.test/hook".to_string()); + config + .0 + .entry(NOTIFY_WEBHOOK_SUB_SYS.to_string()) + .or_default() + .insert("concurrent-notify".to_string(), target); + save_admin_server_config_no_lock(transaction_store, &config).await?; + transaction_order.lock().expect("transaction order lock").push("notify-save"); + Ok::<(), StorageError>(()) + }) + .await + .expect("notify transaction should acquire config locks") + .expect("notify transaction should persist"); + }) + }; + + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, notify_entered.notified()) + .await + .expect("notify transaction should enter"); + + let oidc_task = { + let store = store.clone(); + let transaction_order = transaction_order.clone(); + tokio::spawn(async move { + let transaction_store = store.clone(); + let transaction = with_admin_server_config_write_lock(store, move || async move { + let _ = oidc_entered_tx.send(()); + let mut config = read_admin_config_without_migrate_no_lock(transaction_store.clone()).await?; + transaction_order.lock().expect("transaction order lock").push("oidc-read"); + + let mut provider = KVS::new(); + provider.insert( + OIDC_CONFIG_URL.to_string(), + "https://identity.example.test/.well-known/openid-configuration".to_string(), + ); + provider.insert(OIDC_CLIENT_ID.to_string(), "console".to_string()); + config + .0 + .entry(IDENTITY_OPENID_SUB_SYS.to_string()) + .or_default() + .insert("concurrent-oidc".to_string(), provider); + save_admin_server_config_no_lock(transaction_store, &config).await?; + transaction_order.lock().expect("transaction order lock").push("oidc-save"); + Ok::<(), StorageError>(()) + }); + tokio::pin!(transaction); + let mut oidc_polled_tx = Some(oidc_polled_tx); + poll_fn(|cx| match transaction.as_mut().poll(cx) { + Poll::Pending => { + if let Some(tx) = oidc_polled_tx.take() { + let _ = tx.send(()); + } + Poll::Ready(()) + } + Poll::Ready(_) => panic!("OIDC transaction entered while notify held the config lock"), + }) + .await; + transaction + .await + .expect("OIDC transaction should acquire config locks") + .expect("OIDC transaction should persist"); + }) + }; + + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, oidc_polled_rx) + .await + .expect("OIDC transaction should be polled") + .expect("OIDC transaction poll signal should be delivered"); + assert!( + matches!(oidc_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "OIDC RMW must not enter while the notify RMW owns the server-config transaction" + ); + + release_notify.notify_one(); + tokio::time::timeout(REAL_STORE_TEST_TIMEOUT, async { + notify_task.await.expect("notify task should not panic"); + oidc_task.await.expect("OIDC task should not panic"); + }) + .await + .expect("both server-config transactions should finish"); + + let persisted = read_admin_config_without_migrate(store) + .await + .expect("read final server config"); + assert_eq!( + persisted + .get_value(NOTIFY_WEBHOOK_SUB_SYS, "concurrent-notify") + .expect("notify update should survive") + .get(WEBHOOK_ENDPOINT), + "https://notify.example.test/hook" + ); + assert_eq!( + persisted + .get_value(IDENTITY_OPENID_SUB_SYS, "concurrent-oidc") + .expect("OIDC update should survive") + .get(OIDC_CLIENT_ID), + "console" + ); + assert_eq!( + *transaction_order.lock().expect("transaction order lock"), + vec!["notify-read", "notify-save", "oidc-read", "oidc-save"] + ); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn peer_dynamic_reload_rejects_later_pool_without_publishing() { @@ -811,6 +1063,67 @@ mod tests { .await; } + #[tokio::test] + #[serial_test::serial] + async fn peer_module_switch_reload_uses_its_explicit_context_store() { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>), + (rustfs_config::ENV_AUDIT_ENABLE, None::<&str>), + ], + async { + let selected = runtime_config_reload_fixture().await; + let fallback = runtime_config_reload_fixture().await; + save_persisted_module_switches_to( + selected.context.object_store(), + PersistedModuleSwitches { + notify_enabled: true, + audit_enabled: true, + }, + || (), + ) + .await + .expect("persist selected module switches"); + + refresh_persisted_module_switches_from(fallback.context.object_store()) + .await + .expect("refresh absent fallback module switches"); + assert!( + !current_module_switch_snapshot().notify_enabled, + "fallback store should resolve the default" + ); + + reload_dynamic_config_runtime_state_for_context(Some(&selected.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM) + .await + .expect("peer module switch reload should converge"); + + let selected_snapshot = current_module_switch_snapshot(); + assert!(selected_snapshot.notify_enabled); + assert!(selected_snapshot.persisted_audit_enabled); + assert_eq!(selected_snapshot.notify_source, ModuleSwitchSource::Console); + assert!(matches!( + rustfs_notify::notification_system() + .expect("notification singleton should exist") + .runtime_lifecycle_state(), + rustfs_notify::NotificationRuntimeState::TargetsEnabled { .. } + )); + assert!(is_event_notifier_reconciled()); + + reload_dynamic_config_runtime_state_for_context(Some(&fallback.context), MODULE_SWITCHES_SIGNAL_SUBSYSTEM) + .await + .expect("restore default module switch state"); + assert!(!current_module_switch_snapshot().notify_enabled); + assert_eq!( + rustfs_notify::notification_system() + .expect("notification singleton should remain stable") + .runtime_lifecycle_state(), + rustfs_notify::NotificationRuntimeState::LiveOnly + ); + }, + ) + .await; + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn peer_full_reload_rejects_later_pool_without_publishing() { @@ -835,12 +1148,12 @@ mod tests { } #[tokio::test] - async fn full_reload_publishes_snapshots_before_best_effort_worker_failure() { + async fn full_reload_publishes_snapshots_before_reporting_worker_failure() { let events = Arc::new(Mutex::new(Vec::new())); let publish_events = events.clone(); let worker_events = events.clone(); - reload_runtime_config_snapshot_with( + let err = reload_runtime_config_snapshot_with( async { Ok(ServerConfig::new()) }, |config| async { Ok((config, PreparedRuntimeConfig::default())) }, move |_config, _prepared| { @@ -855,8 +1168,9 @@ mod tests { }, ) .await - .expect("worker failure must not roll back validated storage/server snapshots"); + .expect_err("worker failure must be reported after publishing validated snapshots"); + assert_eq!(err.message(), Some("injected worker reload failure")); assert_eq!( *events.lock().expect("reload result lock"), ["publish", "worker-1-applied", "worker-2-failed"] @@ -871,7 +1185,9 @@ mod tests { assert!(is_dynamic_config_subsystem(HEAL_SUB_SYS)); assert!(is_dynamic_config_subsystem(STORAGE_CLASS_SUB_SYS)); assert!(!is_dynamic_config_subsystem("identity_openid")); - assert!(!is_dynamic_config_subsystem("notify_webhook")); + for sub_system in NOTIFY_SUB_SYSTEMS { + assert!(is_dynamic_config_subsystem(sub_system)); + } } #[test] @@ -885,6 +1201,7 @@ mod tests { STORAGE_CLASS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, AUDIT_MQTT_SUB_SYS, + NOTIFY_WEBHOOK_SUB_SYS, SCANNER_SUB_SYS, HEAL_SUB_SYS, ] { @@ -1014,17 +1331,52 @@ mod tests { } #[test] + #[serial_test::serial(notify_config_env)] fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() { crate::admin::storage_api::config::init_admin_config_defaults(); let mut config = ServerConfig::new(); let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults"); - let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); + let kvs = targets.entry("primary".to_string()).or_default(); kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); kvs.insert(WEBHOOK_ENDPOINT.to_string(), "not-a-url".to_string()); kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string()); - let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail"); - assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + temp_env::with_vars_unset( + [ + format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"), + format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"), + ], + || { + let err = + validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS).expect_err("invalid endpoint should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + }, + ); + } + + #[test] + #[serial_test::serial(notify_config_env)] + fn validate_notify_subsystem_config_uses_enabled_env_overlay() { + crate::admin::storage_api::config::init_admin_config_defaults(); + let mut config = ServerConfig::new(); + let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults"); + let kvs = targets.entry("primary".to_string()).or_default(); + kvs.insert(ENABLE_KEY.to_string(), EnableState::Off.to_string()); + kvs.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/hook".to_string()); + kvs.insert(WEBHOOK_QUEUE_DIR.to_string(), "/tmp/rustfs-notify".to_string()); + + temp_env::with_vars( + [ + (format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"), Some("on")), + (format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"), Some("not-a-url")), + ], + || { + let err = validate_notify_subsystem_config(&config, NOTIFY_WEBHOOK_SUB_SYS) + .expect_err("invalid environment endpoint should fail"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert!(err.message().is_some_and(|message| message.contains("target 'primary'"))); + }, + ); } #[test] diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 490ee1793..c2ae97208 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -412,6 +412,10 @@ pub(crate) async fn read_admin_config_without_migrate(api: Arc) -> Resu ecstore_config::com::read_config_without_migrate(api).await } +pub(crate) async fn read_admin_config_without_migrate_no_lock(api: Arc) -> Result { + ecstore_config::com::read_config_without_migrate_no_lock(api).await +} + pub(crate) async fn save_admin_config(api: Arc, file: &str, data: Vec) -> Result<()> { ecstore_config::com::save_config(api, file, data).await } @@ -420,10 +424,27 @@ pub(crate) async fn delete_admin_config(api: Arc, file: &str) -> Result ecstore_config::com::delete_config(api, file).await } +#[cfg(test)] pub(crate) async fn save_admin_server_config(api: Arc, cfg: &rustfs_config::server_config::Config) -> Result<()> { ecstore_config::com::save_server_config(api, cfg).await } +pub(crate) async fn save_admin_server_config_no_lock( + api: Arc, + cfg: &rustfs_config::server_config::Config, +) -> Result<()> { + ecstore_config::com::save_server_config_no_lock(api, cfg).await +} + +pub(crate) async fn with_admin_server_config_write_lock(api: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_server_config_write_lock(api, operation).await +} + pub(crate) fn init_admin_config_defaults() { ecstore_config::init(); } @@ -521,10 +542,13 @@ pub(crate) mod cluster { } pub(crate) mod config { + #[cfg(test)] + pub(crate) use super::save_admin_server_config; pub(crate) use super::storageclass; pub(crate) use super::{ RUSTFS_META_BUCKET, STORAGE_CLASS_SUB_SYS, delete_admin_config, init_admin_config_defaults, read_admin_config, - read_admin_config_without_migrate, save_admin_config, save_admin_server_config, + read_admin_config_without_migrate, read_admin_config_without_migrate_no_lock, save_admin_config, + save_admin_server_config_no_lock, with_admin_server_config_write_lock, }; } diff --git a/rustfs/src/embedded.rs b/rustfs/src/embedded.rs index 19b6860c9..5fb67ee08 100644 --- a/rustfs/src/embedded.rs +++ b/rustfs/src/embedded.rs @@ -52,7 +52,10 @@ use crate::server::ShutdownHandle; use crate::startup_embedded::{EmbeddedStartedServer, EmbeddedStartupArgs, EmbeddedStartupError, run_embedded_startup}; use crate::startup_lifecycle::embedded_endpoint_address; use crate::startup_server::find_embedded_available_port; -use crate::startup_shutdown::{run_embedded_server_drop_cleanup, run_embedded_server_shutdown}; +use crate::startup_shutdown::{ + EmbeddedRuntimeOwner, register_embedded_runtime_owner, run_embedded_server_drop_cleanup, run_embedded_server_shutdown, + run_embedded_shutdown_cleanup, +}; use std::net::SocketAddr; use std::path::PathBuf; use tokio_util::sync::CancellationToken; @@ -215,14 +218,21 @@ impl RustFSServerBuilder { } async fn do_build(self) -> Result { - let started = run_embedded_startup(self.startup_args).await?; - - Ok(started.into()) + let mut runtime_owner = register_embedded_runtime_owner().await; + match run_embedded_startup(self.startup_args).await { + Ok(started) => Ok(RustFSServer::from_started(started, runtime_owner)), + Err(err) => { + if let Some(cleanup) = runtime_owner.release() { + run_embedded_shutdown_cleanup(cleanup).await; + } + Err(err.into()) + } + } } } -impl From for RustFSServer { - fn from(started: EmbeddedStartedServer) -> Self { +impl RustFSServer { + fn from_started(started: EmbeddedStartedServer, runtime_owner: EmbeddedRuntimeOwner) -> Self { Self { address: started.bound_addr, access_key: started.access_key, @@ -231,6 +241,7 @@ impl From for RustFSServer { shutdown_handle: Some(started.shutdown_handle), cancel_token: started.cancel_token, temp_dir: started.temp_dir, + runtime_owner: Some(runtime_owner), } } } @@ -250,6 +261,7 @@ pub struct RustFSServer { shutdown_handle: Option, cancel_token: CancellationToken, temp_dir: Option, + runtime_owner: Option, } impl RustFSServer { @@ -290,13 +302,34 @@ impl RustFSServer { } async fn do_shutdown(&mut self) { - run_embedded_server_shutdown(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref()).await; + let Some(runtime_owner) = self.runtime_owner.take() else { + return; + }; + let runtime = runtime_owner.cleanup_runtime_handle(); + run_embedded_server_shutdown( + &runtime, + &self.cancel_token, + &mut self.shutdown_handle, + &mut self.temp_dir, + Some(runtime_owner), + ) + .await; } } impl Drop for RustFSServer { fn drop(&mut self) { - run_embedded_server_drop_cleanup(&self.cancel_token, &mut self.shutdown_handle, self.temp_dir.as_deref()); + let Some(runtime_owner) = self.runtime_owner.take() else { + return; + }; + let runtime = runtime_owner.cleanup_runtime_handle(); + run_embedded_server_drop_cleanup( + &runtime, + &self.cancel_token, + &mut self.shutdown_handle, + &mut self.temp_dir, + Some(runtime_owner), + ); } } diff --git a/rustfs/src/server/audit.rs b/rustfs/src/server/audit.rs index 745a3d9f4..c4581510e 100644 --- a/rustfs/src/server/audit.rs +++ b/rustfs/src/server/audit.rs @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from_store, runtime_sources}; +use super::{ + module_switch::resolve_audit_module_state, refresh_persisted_module_switches_from, + refresh_persisted_module_switches_from_store, runtime_sources, +}; +use crate::runtime_sources::AppContext; use rustfs_audit::{AuditError, AuditResult, audit_system, init_audit_system, system::AuditSystemState}; use std::sync::atomic::{AtomicBool, Ordering}; use tracing::{info, warn}; @@ -23,6 +27,13 @@ fn server_config_from_context() -> Option runtime_sources::current_server_config() } +fn server_config_for_context(context: Option<&AppContext>) -> Option { + match context { + Some(context) => context.server_config().get(), + None => server_config_from_context(), + } +} + pub fn refresh_audit_module_enabled() -> bool { let enabled = resolve_audit_module_state().enabled; AUDIT_MODULE_ENABLED.store(enabled, Ordering::Relaxed); @@ -51,7 +62,15 @@ fn has_any_audit_targets(config: &rustfs_config::server_config::Config) -> bool /// If not configured, it skips the initialization. /// It also handles cases where the audit system is already running or if the global configuration is not loaded. pub async fn start_audit_system() -> AuditResult<()> { - if let Err(err) = refresh_persisted_module_switches_from_store().await { + start_audit_system_for_context(None).await +} + +pub(crate) async fn start_audit_system_for_context(context: Option<&AppContext>) -> AuditResult<()> { + let refresh_result = match context { + Some(context) => refresh_persisted_module_switches_from(context.object_store()).await, + None => refresh_persisted_module_switches_from_store().await, + }; + if let Err(err) = refresh_result { warn!("Failed to refresh persisted audit module switch from store: {}", err); } @@ -70,7 +89,7 @@ pub async fn start_audit_system() -> AuditResult<()> { ); // 1. Get the global configuration loaded by ecstore - let server_config = match server_config_from_context() { + let server_config = match server_config_for_context(context) { Some(config) => config, None => { warn!( @@ -165,3 +184,14 @@ pub async fn stop_audit_system() -> AuditResult<()> { Ok(()) } } + +pub(crate) async fn apply_audit_module_switch_for_context(context: Option<&AppContext>) -> AuditResult<()> { + if refresh_audit_module_enabled() { + match start_audit_system_for_context(context).await { + Ok(()) | Err(AuditError::AlreadyInitialized) => Ok(()), + Err(err) => Err(err), + } + } else { + stop_audit_system().await + } +} diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index d5be15c11..d847543a4 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -12,20 +12,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store, runtime_sources}; -use crate::storage_api::server::event::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook}; +use super::{ + module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from}, + refresh_persisted_module_switches_from_store, runtime_sources, +}; +use crate::storage_api::server::event::{ + EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook, + with_server_config_read_lock, +}; use chrono::{DateTime, Utc}; -use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo}; +use rustfs_notify::{ + EventArgs as NotifyEventArgs, NotificationError, NotificationRuntimeState, NotificationSystem, NotifyObjectInfo, +}; use rustfs_s3_types::EventName; +use std::future::Future; use std::net::SocketAddr; +use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; use tokio::spawn; -use tracing::{error, info, instrument, warn}; +use tokio::task::JoinHandle; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use tracing::{info, instrument, warn}; static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); +static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false); +static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new(); -fn server_config_from_context() -> Option { - runtime_sources::current_server_config() +const EVENT_NOTIFIER_RECONCILE_INTERVAL: Duration = Duration::from_secs(5); +const EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(120); +const EVENT_NOTIFY_RUNTIME_RECONCILE: &str = "notify_runtime_reconcile"; + +pub(crate) fn is_event_notifier_reconciled() -> bool { + NOTIFY_RUNTIME_RECONCILED.load(Ordering::Acquire) +} + +pub(crate) fn mark_event_notifier_reconciled() { + NOTIFY_RUNTIME_RECONCILED.store(true, Ordering::Release); +} + +pub(crate) fn mark_event_notifier_unreconciled() { + NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release); } pub fn refresh_notify_module_enabled() -> bool { @@ -112,79 +140,201 @@ fn parse_host_and_port(host: String) -> (String, u16) { } fn install_ecstore_event_dispatch_hook() { - let installed = register_event_dispatch_hook(|args| { - let Some(notify_args) = convert_ecstore_event_args(args) else { - return; - }; - spawn(async move { - runtime_sources::current_notify_interface().notify(notify_args).await; + ECSTORE_EVENT_DISPATCH_HOOK.get_or_init(|| { + let installed = register_event_dispatch_hook(|args| { + let Some(notify_args) = convert_ecstore_event_args(args) else { + return; + }; + spawn(async move { + runtime_sources::current_notify_interface().notify(notify_args).await; + }); }); + + if !installed { + warn!("ECStore event dispatch hook was already registered"); + } }); +} - if !installed { - warn!("ECStore event dispatch hook was already registered"); +fn ensure_live_events_initialized() -> std::sync::Arc { + let system = rustfs_notify::ensure_live_events(); + install_ecstore_event_dispatch_hook(); + system +} + +fn ensure_event_notifier_converged(system: &NotificationSystem) -> Result<(), NotificationError> { + if system.runtime_lifecycle_is_converged() { + Ok(()) + } else { + Err(NotificationError::Initialization( + "Latest notification lifecycle generation has not converged".to_string(), + )) } } -fn ensure_live_events_initialized() -> bool { - if rustfs_notify::notification_system().is_some() { - return true; - } +pub(crate) async fn reconcile_event_notifier_from_store( + store: std::sync::Arc, +) -> Result<(), NotificationError> { + let result = async { + validate_notify_module_env().map_err(NotificationError::Initialization)?; + let system = ensure_live_events_initialized(); + let transition_system = system.clone(); + let transition_store = store.clone(); + let transition = with_refreshed_notify_module_state_from(store, move |resolution| async move { + NOTIFY_MODULE_ENABLED.store(resolution.enabled, Ordering::Relaxed); + let read_store = transition_store.clone(); + let config_system = transition_system.clone(); + with_server_config_read_lock(transition_store, move || async move { + let config = read_existing_server_config_no_lock(read_store) + .await + .map_err(|err| NotificationError::ReadConfig(err.to_string()))?; + let mode_matches = match config_system.runtime_lifecycle_state() { + NotificationRuntimeState::LiveOnly => !resolution.enabled, + NotificationRuntimeState::TargetsEnabled { .. } => resolution.enabled, + NotificationRuntimeState::Terminated => false, + }; + if config_system.config_snapshot().await == config + && mode_matches + && config_system.runtime_lifecycle_is_converged() + { + Ok::<_, NotificationError>(None) + } else { + Ok(Some(config_system.publish_targets_enabled(resolution.enabled, Some(config)))) + } + }) + .await + .map_err(|err| NotificationError::StorageNotAvailable(err.to_string()))? + }) + .await + .map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??; - match rustfs_notify::initialize_live_events() { - Ok(()) => { - install_ecstore_event_dispatch_hook(); - true + if let Some(transition) = transition { + transition.wait().await?; } - Err(e) => { - error!("Failed to initialize live event stream support: {}", e); - false + + ensure_event_notifier_converged(&system) + } + .await; + + if result.is_ok() { + mark_event_notifier_reconciled(); + } else { + mark_event_notifier_unreconciled(); + } + result +} + +pub(crate) fn start_persisted_event_notifier_reconciler( + store: std::sync::Arc, + cancellation: CancellationToken, +) -> JoinHandle<()> { + spawn(run_persisted_event_notifier_reconciler( + cancellation, + EVENT_NOTIFIER_RECONCILE_INTERVAL, + move || { + let store = store.clone(); + async move { reconcile_event_notifier_from_store(store).await } + }, + )) +} + +async fn run_persisted_event_notifier_reconciler( + cancellation: CancellationToken, + reconcile_interval: Duration, + mut reconcile: Reconcile, +) where + Reconcile: FnMut() -> ReconcileFuture, + ReconcileFuture: Future>, +{ + let first_tick = Instant::now() + reconcile_interval; + let mut ticker = tokio::time::interval_at(first_tick, reconcile_interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut failure_reported = false; + + loop { + tokio::select! { + biased; + _ = cancellation.cancelled() => break, + _ = ticker.tick() => {} + } + + let result = tokio::select! { + biased; + _ = cancellation.cancelled() => break, + result = tokio::time::timeout(EVENT_NOTIFIER_RECONCILE_ATTEMPT_TIMEOUT, reconcile()) => result, + }; + + match result { + Ok(Ok(())) => { + if failure_reported { + info!( + event = EVENT_NOTIFY_RUNTIME_RECONCILE, + component = "notify", + subsystem = "lifecycle", + state = "recovered", + "Persisted notification runtime reconciliation recovered" + ); + } + failure_reported = false; + } + Ok(Err(_)) | Err(_) => { + if !failure_reported { + warn!( + event = EVENT_NOTIFY_RUNTIME_RECONCILE, + component = "notify", + subsystem = "lifecycle", + state = "degraded", + reason = "reconcile_failed_or_timed_out", + "Persisted notification runtime reconciliation failed" + ); + } + failure_reported = true; + } } } } -/// Shuts down the event notifier system gracefully -pub async fn shutdown_event_notifier() { +/// Irreversibly shuts down the event notifier target runtime for process exit. +pub async fn shutdown_event_notifier() -> Result<(), NotificationError> { info!("Shutting down event notifier system..."); - - if !rustfs_notify::is_notification_system_initialized() { + let Some(system) = rustfs_notify::notification_system() else { info!("Event notifier system is not initialized, nothing to shut down."); - return; - } - - let system = match rustfs_notify::notification_system() { - Some(sys) => sys, - None => { - info!("Event notifier system is not initialized."); - return; - } + return Ok(()); }; - // Call the shutdown function from the rustfs_notify module - system.shutdown().await; + system.shutdown_checked().await?; info!("Event notifier system shut down successfully."); + Ok(()) } #[instrument] -pub async fn init_event_notifier() { - if let Err(err) = refresh_persisted_module_switches_from_store().await { - warn!("Failed to refresh persisted notify module switch from store: {}", err); - } +pub async fn init_event_notifier() -> Result<(), NotificationError> { + mark_event_notifier_unreconciled(); + validate_notify_module_env().map_err(NotificationError::Initialization)?; + let system = ensure_live_events_initialized(); + refresh_persisted_module_switches_from_store() + .await + .map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))?; let enabled = refresh_notify_module_enabled(); + if !enabled { info!( target: "rustfs::main::init_event_notifier", "Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.", rustfs_config::ENV_NOTIFY_ENABLE ); - if ensure_live_events_initialized() { - info!( - target: "rustfs::main::init_event_notifier", - "Live event stream support initialized successfully." - ); + if system.runtime_lifecycle_state() != NotificationRuntimeState::LiveOnly { + system.set_targets_enabled(false, None).await?; } - return; + system.reload_persisted_config().await?; + info!( + target: "rustfs::main::init_event_notifier", + "Live event stream support initialized successfully." + ); + ensure_event_notifier_converged(&system)?; + mark_event_notifier_reconciled(); + return Ok(()); } info!( @@ -192,53 +342,44 @@ pub async fn init_event_notifier() { "Initializing event notifier..." ); - // 1. Get the global configuration loaded by ecstore - let server_config = match server_config_from_context() { - Some(config) => config, - None => { - warn!("Event notifier initialization failed: Global server config not loaded."); - return; - } - }; - info!( target: "rustfs::main::init_event_notifier", "Event notifier configuration found, proceeding with initialization." ); - if let Some(system) = rustfs_notify::notification_system() { - // Reuse the existing global system on re-enable so bucket rules, metrics, - // and stream lifecycle stay aligned with the current process singleton. - if let Err(e) = system.reload_config(server_config).await { - error!("Failed to reload event notifier system: {}", e); - } else { - info!( - target: "rustfs::main::init_event_notifier", - "Event notifier system reloaded successfully." - ); - } - } else { - match rustfs_notify::initialize(server_config).await { - Ok(()) => { - install_ecstore_event_dispatch_hook(); - info!( - target: "rustfs::main::init_event_notifier", - "Event notifier system initialized successfully." - ); - } - Err(e) => error!("Failed to initialize event notifier system: {}", e), - } + system.reload_persisted_config().await?; + let runtime_state = system.runtime_lifecycle_state(); + if !matches!(runtime_state, NotificationRuntimeState::TargetsEnabled { .. }) { + system.set_targets_enabled(true, None).await?; } + info!( + target: "rustfs::main::init_event_notifier", + "Event notifier system initialized successfully." + ); + ensure_event_notifier_converged(&system)?; + mark_event_notifier_reconciled(); + Ok(()) } #[cfg(test)] mod tests { - use super::{convert_ecstore_object_info, parse_host_and_port}; + use super::{convert_ecstore_object_info, parse_host_and_port, run_persisted_event_notifier_reconciler}; use crate::storage_api::server::event::StorageObjectInfo; use crate::storage_api::server::event::contract::lifecycle::TransitionedObject; use chrono::{DateTime, Utc}; - use std::{collections::HashMap, sync::Arc}; + use rustfs_notify::NotificationError; + use std::{ + collections::HashMap, + future::pending, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration as StdDuration, + }; use time::{Duration, OffsetDateTime}; + use tokio::sync::Notify; + use tokio_util::sync::CancellationToken; #[test] fn parse_host_and_port_with_ipv4_and_port() { @@ -303,4 +444,88 @@ mod tests { assert_eq!(converted.storage_class.as_deref(), Some("GLACIER")); assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE")); } + + #[tokio::test(start_paused = true)] + async fn persisted_reconciler_converges_after_one_injected_tick() { + let persisted_generation = Arc::new(AtomicUsize::new(1)); + let runtime_generation = Arc::new(AtomicUsize::new(1)); + let runtime_converged = Arc::new(AtomicBool::new(true)); + let reconcile_calls = Arc::new(AtomicUsize::new(0)); + let reconciled = Arc::new(Notify::new()); + let cancellation = CancellationToken::new(); + + let task = tokio::spawn(run_persisted_event_notifier_reconciler( + cancellation.clone(), + StdDuration::from_secs(5), + { + let persisted_generation = persisted_generation.clone(); + let runtime_generation = runtime_generation.clone(); + let runtime_converged = runtime_converged.clone(); + let reconcile_calls = reconcile_calls.clone(); + let reconciled = reconciled.clone(); + move || { + let persisted_generation = persisted_generation.clone(); + let runtime_generation = runtime_generation.clone(); + let runtime_converged = runtime_converged.clone(); + let reconcile_calls = reconcile_calls.clone(); + let reconciled = reconciled.clone(); + async move { + runtime_generation.store(persisted_generation.load(Ordering::SeqCst), Ordering::SeqCst); + runtime_converged.store(true, Ordering::SeqCst); + reconcile_calls.fetch_add(1, Ordering::SeqCst); + reconciled.notify_one(); + Ok(()) + } + } + }, + )); + tokio::task::yield_now().await; + + persisted_generation.store(2, Ordering::SeqCst); + runtime_converged.store(false, Ordering::SeqCst); + assert_eq!( + reconcile_calls.load(Ordering::SeqCst), + 0, + "the first tick must wait for the configured interval" + ); + + tokio::time::advance(StdDuration::from_secs(5)).await; + reconciled.notified().await; + + assert_eq!(reconcile_calls.load(Ordering::SeqCst), 1); + assert_eq!(runtime_generation.load(Ordering::SeqCst), 2); + assert!(runtime_converged.load(Ordering::SeqCst)); + + cancellation.cancel(); + task.await.expect("persisted reconciler should stop after cancellation"); + } + + #[tokio::test(start_paused = true)] + async fn persisted_reconciler_cancellation_interrupts_an_inflight_attempt() { + let entered = Arc::new(Notify::new()); + let cancellation = CancellationToken::new(); + let task = tokio::spawn(run_persisted_event_notifier_reconciler( + cancellation.clone(), + StdDuration::from_secs(5), + { + let entered = entered.clone(); + move || { + let entered = entered.clone(); + async move { + entered.notify_one(); + pending::>().await + } + } + }, + )); + tokio::task::yield_now().await; + tokio::time::advance(StdDuration::from_secs(5)).await; + entered.notified().await; + + cancellation.cancel(); + tokio::time::timeout(StdDuration::from_secs(1), task) + .await + .expect("cancellation should stop an in-flight reconciliation attempt") + .expect("persisted reconciler should not panic"); + } } diff --git a/rustfs/src/server/mod.rs b/rustfs/src/server/mod.rs index af30acc33..220f1da98 100644 --- a/rustfs/src/server/mod.rs +++ b/rustfs/src/server/mod.rs @@ -32,6 +32,7 @@ pub mod tls_material; use tracing::warn; // Items used by main.rs (binary crate) and/or embedded.rs — must be fully pub. +pub(crate) use audit::apply_audit_module_switch_for_context; pub use audit::{is_audit_module_enabled, refresh_audit_module_enabled, start_audit_system, stop_audit_system}; pub use event::{init_event_notifier, is_notify_module_enabled, refresh_notify_module_enabled, shutdown_event_notifier}; pub use http::start_http_server; @@ -45,6 +46,10 @@ pub use service_state::wait_for_shutdown; // Items only used within the library crate (admin handlers, server/http.rs, etc.). pub(crate) use event::convert_ecstore_object_info; +pub(crate) use event::{ + is_event_notifier_reconciled, mark_event_notifier_reconciled, mark_event_notifier_unreconciled, + reconcile_event_notifier_from_store, start_persisted_event_notifier_reconciler, +}; #[cfg(test)] pub(crate) use health::{ HealthPayloadContext, HealthReadinessSource, build_component_details, build_health_payload, health_check_state, @@ -55,8 +60,9 @@ pub(crate) use http::HeaderMapCarrier; pub(crate) use http::active_http_requests; pub(crate) use layer::RequestContextLayer; pub(crate) use module_switch::{ - ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot, - refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update, + MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, + current_module_switch_snapshot, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store, + save_persisted_module_switches_to, validate_module_switch_update, }; pub(crate) use prefix::{ ADMIN_PREFIX, CONSOLE_PREFIX, FAVICON_PATH, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, diff --git a/rustfs/src/server/module_switch.rs b/rustfs/src/server/module_switch.rs index 423bb56be..5e7cbe509 100644 --- a/rustfs/src/server/module_switch.rs +++ b/rustfs/src/server/module_switch.rs @@ -13,17 +13,35 @@ // limitations under the License. use super::runtime_sources; -use crate::storage_api::server::module_switch::{Error as StorageError, read_config, save_config}; +use crate::storage_api::server::module_switch::{ + Error as StorageError, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, + with_config_object_write_lock, +}; +use crate::storage_api::server::runtime_sources::ECStore; use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::sync::LazyLock; use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::sync::Mutex; const MODULE_SWITCH_CONFIG_PATH: &str = "config/module_switches.json"; +pub(crate) const MODULE_SWITCHES_SIGNAL_SUBSYSTEM: &str = "module_switches"; // Keep a cheap in-process snapshot so hot-path checks do not need to read // cluster metadata after startup or console-triggered refresh. static PERSISTED_NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE); static PERSISTED_AUDIT_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_AUDIT_ENABLE); static PERSISTED_MODULE_SWITCH_CONFIGURED: AtomicBool = AtomicBool::new(false); +static MODULE_SWITCH_RMW_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + +async fn serialize_module_switch_rmw(operation: F) -> T +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + let _rmw_guard = MODULE_SWITCH_RMW_LOCK.lock().await; + operation().await +} #[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] pub(crate) struct PersistedModuleSwitches { @@ -80,6 +98,14 @@ fn env_override_value(key: &str) -> Option { rustfs_utils::get_env_opt_bool(key) } +pub(crate) fn validate_notify_module_env() -> Result<(), String> { + let key = rustfs_config::ENV_NOTIFY_ENABLE; + if env_override_exists(key) && env_override_value(key).is_none() { + return Err(format!("{key} is not a valid boolean")); + } + Ok(()) +} + fn effective_module_switch_state(env_key: &str, persisted_enabled: bool, default_enabled: bool) -> ModuleSwitchResolution { // Explicit env remains the highest-priority source so process-level bootstrap // cannot be silently overridden by a later console write. @@ -162,42 +188,157 @@ pub(crate) async fn refresh_persisted_module_switches_from_store() -> Result ( +pub(crate) async fn refresh_persisted_module_switches_from(store: Arc) -> Result { + refresh_persisted_module_switches_with(|| async move { + match read_config(store, MODULE_SWITCH_CONFIG_PATH).await { + Ok(data) => Ok(Some(data)), + Err(StorageError::ConfigNotFound) => Ok(None), + Err(err) => Err(format!("failed to load module switch config: {err}")), + } + }) + .await +} + +pub(crate) async fn with_refreshed_notify_module_state_from( + store: Arc, + publish: Publish, +) -> Result +where + Publish: FnOnce(ModuleSwitchResolution) -> PublishFuture + Send + 'static, + PublishFuture: std::future::Future + Send + 'static, + T: Send + 'static, +{ + // Lock order: local module RMW -> distributed module-object read -> the + // callback's server-config read lock. Server-config writers never acquire + // the module-object lock, and module writers release any server-config + // read before taking the module-object write lock. + serialize_module_switch_rmw(|| async move { + let read_store = store.clone(); + with_config_object_read_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move { + let persisted = match read_config_no_lock(read_store, MODULE_SWITCH_CONFIG_PATH).await { + Ok(data) => Some(data), + Err(StorageError::ConfigNotFound) => None, + Err(err) => return Err(format!("failed to load module switch config: {err}")), + }; + let config = decode_persisted_module_switches(persisted)?; + set_persisted_module_switches(config.0, config.1); + Ok(publish(resolve_notify_module_state()).await) + }) + .await + .map_err(|err| format!("failed to lock module switch refresh: {err}"))? + }) + .await +} + +fn decode_persisted_module_switches(data: Option>) -> Result<(PersistedModuleSwitches, bool), String> { + match data { + Some(data) => Ok(( serde_json::from_slice::(&data) .map_err(|e| format!("failed to deserialize module switch config: {e}"))?, true, - ), - Err(StorageError::ConfigNotFound) => (PersistedModuleSwitches::default(), false), - Err(err) => return Err(format!("failed to load module switch config: {err}")), - }; - - // Track whether the persisted file exists so the effective state can - // distinguish "console configured false" from "never configured, use default". - set_persisted_module_switches(config, configured); - Ok(config) + )), + None => Ok((PersistedModuleSwitches::default(), false)), + } } -pub(crate) async fn save_persisted_module_switches_to_store(config: PersistedModuleSwitches) -> Result<(), String> { - let Some(store) = runtime_sources::current_object_store_handle() else { - return Err("storage layer not initialized".to_string()); - }; +async fn refresh_persisted_module_switches_with(read: F) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, String>>, +{ + serialize_module_switch_rmw(|| async move { + let (config, configured) = decode_persisted_module_switches(read().await?)?; - let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?; - save_config(store, MODULE_SWITCH_CONFIG_PATH, data) + // Track whether the persisted file exists so the effective state can + // distinguish "console configured false" from "never configured, use default". + set_persisted_module_switches(config, configured); + Ok(config) + }) + .await +} + +pub(crate) async fn save_persisted_module_switches_to( + store: Arc, + config: PersistedModuleSwitches, + publish: F, +) -> Result +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + // Lock order matches refresh_persisted_module_switches_from: local RMW + // guard before the namespace lock. Taking these in the opposite order lets + // a reader hold the local guard while waiting for a writer that is itself + // waiting for the local guard. + serialize_module_switch_rmw(|| async move { + let save_store = store.clone(); + with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move { + save_persisted_module_switches_inner( + config, + move |data| async move { + save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data) + .await + .map_err(|e| format!("failed to save module switch config: {e}")) + }, + publish, + ) + .await + }) .await - .map_err(|e| format!("failed to save module switch config: {e}"))?; + .map_err(|err| format!("failed to lock module switch update: {err}"))? + }) + .await +} + +#[cfg(test)] +async fn save_persisted_module_switches_with( + config: PersistedModuleSwitches, + save: Save, + publish: F, +) -> Result +where + F: FnOnce() -> T, + Save: FnOnce(Vec) -> SaveFuture, + SaveFuture: std::future::Future>, +{ + serialize_module_switch_rmw(|| save_persisted_module_switches_inner(config, save, publish)).await +} + +async fn save_persisted_module_switches_inner( + config: PersistedModuleSwitches, + save: Save, + publish: F, +) -> Result +where + F: FnOnce() -> T, + Save: FnOnce(Vec) -> SaveFuture, + SaveFuture: std::future::Future>, +{ + let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?; + save(data).await?; set_persisted_module_switches(config, true); - Ok(()) + Ok(publish()) } #[cfg(test)] mod tests { use super::*; + use crate::storage_api::startup::storage::{EndpointServerPools, init_local_disks_with_instance_ctx, new_instance_ctx}; use serial_test::serial; + use std::future::{Future, poll_fn}; + use std::sync::{ + Arc, Mutex as StdMutex, + atomic::{AtomicBool, Ordering as AtomicOrdering}, + }; + use std::task::Poll; use temp_env::{with_var, with_vars}; + use tempfile::TempDir; + use tokio::sync::{Notify, oneshot}; + use tokio_util::sync::CancellationToken; #[test] #[serial] @@ -299,6 +440,15 @@ mod tests { }); } + #[test] + #[serial] + fn invalid_notify_env_is_rejected_before_runtime_reconcile() { + with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("invalid"), || { + let err = validate_notify_module_env().expect_err("invalid notify env must fail closed"); + assert!(err.contains(rustfs_config::ENV_NOTIFY_ENABLE)); + }); + } + #[test] #[serial] fn validate_module_switch_update_allows_matching_env_override() { @@ -330,4 +480,200 @@ mod tests { assert!(err.contains("not a valid boolean")); }); } + + #[tokio::test] + #[serial] + async fn save_publish_and_refresh_share_the_rmw_lock() { + let previous = current_persisted_module_switches(); + let previous_configured = persisted_module_switches_configured(); + + save_persisted_module_switches_with( + PersistedModuleSwitches { + notify_enabled: false, + audit_enabled: true, + }, + |_| async { Ok(()) }, + || { + assert!(MODULE_SWITCH_RMW_LOCK.try_lock().is_err(), "publish must run while the RMW lock is held"); + }, + ) + .await + .expect("save and publish should succeed"); + + let events = Arc::new(StdMutex::new(Vec::new())); + let release_save = Arc::new(Notify::new()); + let (save_started_tx, save_started_rx) = oneshot::channel(); + + let save_events = events.clone(); + let publish_events = events.clone(); + let save_release = release_save.clone(); + let save_task = tokio::spawn(async move { + save_persisted_module_switches_with( + PersistedModuleSwitches { + notify_enabled: true, + audit_enabled: false, + }, + move |_| async move { + save_events.lock().expect("lock event log").push("save"); + save_started_tx.send(()).expect("signal save start"); + save_release.notified().await; + Ok(()) + }, + move || { + publish_events.lock().expect("lock event log").push("publish"); + }, + ) + .await + }); + + save_started_rx.await.expect("save should reach its persistence step"); + + let refresh_events = events.clone(); + let (refresh_waiting_tx, refresh_waiting_rx) = oneshot::channel(); + let refresh_task = tokio::spawn(async move { + let mut refresh = Box::pin(refresh_persisted_module_switches_with(move || async move { + refresh_events.lock().expect("lock event log").push("refresh"); + Ok(None) + })); + + poll_fn(|cx| match refresh.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(_) => panic!("refresh must wait for an in-flight save"), + }) + .await; + refresh_waiting_tx.send(()).expect("signal refresh wait"); + refresh.await + }); + + refresh_waiting_rx.await.expect("refresh should be waiting on the RMW lock"); + release_save.notify_one(); + + save_task.await.expect("join save task").expect("save should succeed"); + refresh_task + .await + .expect("join refresh task") + .expect("refresh should succeed"); + + assert_eq!(*events.lock().expect("lock event log"), ["save", "publish", "refresh"]); + set_persisted_module_switches(previous, previous_configured); + } + + #[tokio::test] + #[serial] + async fn distributed_refresh_orders_old_true_before_new_false_publish() { + temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, None::<&str>)], async { + let previous = current_persisted_module_switches(); + let previous_configured = persisted_module_switches_configured(); + let temp_dir = TempDir::new().expect("module switch ordering temp dir"); + let volume = temp_dir.path().join("disk"); + tokio::fs::create_dir_all(&volume) + .await + .expect("create module switch test disk"); + let (endpoint_pools, _) = + EndpointServerPools::from_volumes("127.0.0.1:29131", vec![volume.to_string_lossy().into_owned()]) + .await + .expect("create module switch test endpoints"); + let instance_ctx = new_instance_ctx(); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("initialize module switch test disk"); + let shutdown = CancellationToken::new(); + let store = ECStore::new_with_instance_ctx( + "127.0.0.1:29131".parse().expect("module switch test address"), + endpoint_pools, + shutdown.clone(), + instance_ctx, + ) + .await + .expect("create module switch test store"); + + save_persisted_module_switches_to( + store.clone(), + PersistedModuleSwitches { + notify_enabled: true, + audit_enabled: false, + }, + || (), + ) + .await + .expect("persist initial enabled module state"); + + let publish_order = Arc::new(StdMutex::new(Vec::new())); + let runtime_enabled = Arc::new(AtomicBool::new(false)); + let reader_entered = Arc::new(Notify::new()); + let release_reader = Arc::new(Notify::new()); + let reader = tokio::spawn({ + let store = store.clone(); + let publish_order = publish_order.clone(); + let runtime_enabled = runtime_enabled.clone(); + let reader_entered = reader_entered.clone(); + let release_reader = release_reader.clone(); + async move { + with_refreshed_notify_module_state_from(store, move |resolution| async move { + assert!(resolution.enabled, "the old persisted snapshot should be enabled"); + reader_entered.notify_one(); + release_reader.notified().await; + publish_order.lock().expect("module publish order lock").push("old-true"); + runtime_enabled.store(true, AtomicOrdering::SeqCst); + }) + .await + } + }); + reader_entered.notified().await; + + let writer_entered = Arc::new(AtomicBool::new(false)); + let (writer_started_tx, writer_started_rx) = oneshot::channel(); + let writer = tokio::spawn({ + let store = store.clone(); + let publish_order = publish_order.clone(); + let runtime_enabled = runtime_enabled.clone(); + let writer_entered = writer_entered.clone(); + async move { + writer_started_tx.send(()).expect("signal remote writer start"); + let save_store = store.clone(); + with_config_object_write_lock(store, MODULE_SWITCH_CONFIG_PATH.to_string(), move || async move { + writer_entered.store(true, AtomicOrdering::SeqCst); + let data = serde_json::to_vec(&PersistedModuleSwitches { + notify_enabled: false, + audit_enabled: false, + }) + .expect("serialize disabled module state"); + save_config_no_lock(save_store, MODULE_SWITCH_CONFIG_PATH, data) + .await + .map_err(|err| err.to_string())?; + publish_order.lock().expect("module publish order lock").push("new-false"); + runtime_enabled.store(false, AtomicOrdering::SeqCst); + Ok::<(), String>(()) + }) + .await + .expect("remote writer should acquire module object lock") + .expect("remote writer should persist disabled state"); + } + }); + writer_started_rx.await.expect("remote writer should start"); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + !writer_entered.load(AtomicOrdering::SeqCst), + "the remote writer must wait until the old read has synchronously published" + ); + + release_reader.notify_one(); + reader + .await + .expect("join module reader") + .expect("module reader should succeed"); + writer.await.expect("join remote module writer"); + + assert_eq!( + *publish_order.lock().expect("module publish order result lock"), + ["old-true", "new-false"] + ); + assert!(!runtime_enabled.load(AtomicOrdering::SeqCst), "the latest persisted false state must win"); + shutdown.cancel(); + set_persisted_module_switches(previous, previous_configured); + }) + .await; + } } diff --git a/rustfs/src/startup_audit.rs b/rustfs/src/startup_audit.rs index c4d5de035..99449e8ee 100644 --- a/rustfs/src/startup_audit.rs +++ b/rustfs/src/startup_audit.rs @@ -20,6 +20,7 @@ use tracing::{error, info}; const LOG_COMPONENT_MAIN: &str = "main"; const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_AUDIT_SYSTEM_STATE: &str = "audit_system_state"; +const EVENT_NOTIFY_SYSTEM_STATE: &str = "notify_system_state"; pub(crate) async fn init_audit_runtime() { match init_event_notifier_and_audit().await { @@ -47,17 +48,28 @@ pub(crate) async fn init_event_notifier_and_audit() -> AuditResult<()> { init_event_notifier_and_audit_with(init_event_notifier, start_audit_system).await } -async fn init_event_notifier_and_audit_with( +async fn init_event_notifier_and_audit_with( notify: NotifyFn, start_audit: AuditFn, ) -> AuditResult<()> where NotifyFn: FnOnce() -> NotifyFuture, - NotifyFuture: Future, + NotifyFuture: Future>, + NotifyError: std::fmt::Display, AuditFn: FnOnce() -> AuditFuture, AuditFuture: Future>, { - notify().await; + if let Err(err) = notify().await { + error!( + target: "rustfs::main::run", + event = EVENT_NOTIFY_SYSTEM_STATE, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "degraded", + error = %err, + "Notification runtime failed to start; continuing in degraded mode" + ); + } start_audit().await } @@ -75,6 +87,7 @@ mod tests { let result = init_event_notifier_and_audit_with( move || async move { notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify"); + Ok::<(), &'static str>(()) }, move || async move { audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit"); @@ -97,6 +110,7 @@ mod tests { let result = init_event_notifier_and_audit_with( move || async move { notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify"); + Ok::<(), &'static str>(()) }, move || async move { audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit"); @@ -109,4 +123,26 @@ mod tests { let events = events.lock().unwrap_or_else(|err| err.into_inner()).clone(); assert_eq!(events, ["notify", "audit"]); } + + #[tokio::test] + async fn notify_failure_still_starts_audit() { + let events = Arc::new(Mutex::new(Vec::new())); + let notify_events = events.clone(); + let audit_events = events.clone(); + + let result = init_event_notifier_and_audit_with( + move || async move { + notify_events.lock().unwrap_or_else(|err| err.into_inner()).push("notify"); + Err::<(), &'static str>("notify failed") + }, + move || async move { + audit_events.lock().unwrap_or_else(|err| err.into_inner()).push("audit"); + Ok(()) + }, + ) + .await; + + assert!(result.is_ok()); + assert_eq!(*events.lock().unwrap_or_else(|err| err.into_inner()), ["notify", "audit"]); + } } diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index 1a7f4e04b..47abbf248 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -14,7 +14,7 @@ use crate::storage_api::startup::lifecycle::ECStore; use crate::{ - server::{ServiceStateManager, ShutdownHandle, wait_for_shutdown}, + server::{ServiceStateManager, ShutdownHandle, start_persisted_event_notifier_reconciler, wait_for_shutdown}, startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap}, startup_runtime_sources, startup_services::StartupServiceRuntime, @@ -142,6 +142,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec ); publish_ready_for_iam_bootstrap(iam_bootstrap, readiness.as_ref(), Some(state_manager.as_ref())).await?; startup_runtime_sources::publish_init_time_now().await; + let event_notifier_reconciler = start_persisted_event_notifier_reconciler(store.clone(), shutdown_token.clone()); if enable_scanner { init_data_scanner(shutdown_token.clone(), store).await; @@ -157,6 +158,17 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec shutdown_token, ) .await; + if let Err(err) = event_notifier_reconciler.await { + tracing::warn!( + target: "rustfs::main::run", + event = "notify_runtime_reconcile", + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "join_failed", + reason = if err.is_cancelled() { "task_cancelled" } else { "task_panicked" }, + "Persisted notification runtime reconciler task failed to join" + ); + } info!( target: "rustfs::main::run", diff --git a/rustfs/src/startup_shutdown.rs b/rustfs/src/startup_shutdown.rs index 8b89f0f3b..2830502e5 100644 --- a/rustfs/src/startup_shutdown.rs +++ b/rustfs/src/startup_shutdown.rs @@ -21,8 +21,11 @@ use crate::{ startup_runtime_sources, }; use rustfs_heal::shutdown_ahm_services; +use rustfs_notify::NotificationLifecycleTransition; use rustfs_utils::get_env_bool_with_aliases; -use std::path::Path; +use std::future::Future; +use std::path::PathBuf; +use std::sync::Mutex; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; @@ -43,6 +46,137 @@ const EVENT_EVENT_NOTIFIER_SHUTDOWN: &str = "event_notifier_shutdown"; const EVENT_PROFILING_SHUTDOWN: &str = "profiling_shutdown"; const EVENT_SERVER_SHUTDOWN_STATE: &str = "server_shutdown_state"; +fn join_failure_reason(error: &tokio::task::JoinError) -> &'static str { + if error.is_cancelled() { + "join_cancelled" + } else { + "join_panicked" + } +} + +struct EmbeddedRuntimeOwnerState { + owners: usize, + pending_cleanup: Option, +} + +struct EmbeddedRuntimeOwners { + state: Mutex, +} + +impl EmbeddedRuntimeOwners { + const fn new() -> Self { + Self { + state: Mutex::new(EmbeddedRuntimeOwnerState { + owners: 0, + pending_cleanup: None, + }), + } + } + + fn register(&self) -> Option { + let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner()); + state.owners += 1; + state + .pending_cleanup + .as_ref() + .filter(|cleanup| !cleanup.is_cancelled()) + .cloned() + } + + fn release_with(&self, prepare_cleanup: impl FnOnce(CancellationToken) -> T) -> Option { + let mut state = self.state.lock().unwrap_or_else(|err| err.into_inner()); + if state.owners == 0 { + return None; + } + state.owners -= 1; + if state.owners != 0 { + return None; + } + if state.pending_cleanup.as_ref().is_some_and(|cleanup| !cleanup.is_cancelled()) { + return None; + } + + let completion = CancellationToken::new(); + // The disable generation must be accepted while registration is excluded; + // otherwise a concurrently starting owner can enable targets first and be + // overwritten by this last-owner release. + let cleanup = prepare_cleanup(completion.clone()); + state.pending_cleanup = Some(completion); + Some(cleanup) + } + + #[cfg(test)] + fn owner_count(&self) -> usize { + self.state.lock().unwrap_or_else(|err| err.into_inner()).owners + } +} + +static EMBEDDED_RUNTIME_OWNERS: EmbeddedRuntimeOwners = EmbeddedRuntimeOwners::new(); + +pub(crate) struct EmbeddedRuntimeOwner { + active: bool, + runtime: tokio::runtime::Handle, +} + +pub(crate) struct EmbeddedRuntimeCleanup { + completion: CancellationToken, + notification: Option, + runtime: tokio::runtime::Handle, +} + +impl EmbeddedRuntimeCleanup { + fn prepare(completion: CancellationToken, runtime: tokio::runtime::Handle) -> Self { + let _runtime_guard = runtime.enter(); + let system = rustfs_notify::ensure_live_events(); + Self { + completion, + notification: Some(system.publish_targets_enabled(false, None)), + runtime, + } + } +} + +impl Drop for EmbeddedRuntimeCleanup { + fn drop(&mut self) { + self.completion.cancel(); + } +} + +impl EmbeddedRuntimeOwner { + pub(crate) fn cleanup_runtime_handle(&self) -> tokio::runtime::Handle { + tokio::runtime::Handle::try_current().unwrap_or_else(|_| self.runtime.clone()) + } + + pub(crate) fn release(&mut self) -> Option { + if !self.active { + return None; + } + self.active = false; + let runtime = self.cleanup_runtime_handle(); + EMBEDDED_RUNTIME_OWNERS.release_with(move |completion| EmbeddedRuntimeCleanup::prepare(completion, runtime)) + } +} + +impl Drop for EmbeddedRuntimeOwner { + fn drop(&mut self) { + if let Some(cleanup) = self.release() { + schedule_embedded_runtime_cleanup(cleanup); + } + } +} + +pub(crate) async fn register_embedded_runtime_owner() -> EmbeddedRuntimeOwner { + let runtime = tokio::runtime::Handle::current(); + let pending_cleanup = EMBEDDED_RUNTIME_OWNERS.register(); + let owner = EmbeddedRuntimeOwner { active: true, runtime }; + // Reserve ownership before waiting so another release cannot start a second + // process-runtime cleanup while this startup is queued behind the first. + if let Some(cleanup) = pending_cleanup { + cleanup.cancelled().await; + } + owner +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BackgroundShutdownStep { DataScanner, @@ -153,7 +287,17 @@ pub(crate) async fn run_startup_shutdown_sequence( state = "stopping", "Event notifier shutdown started" ); - shutdown_event_notifier().await; + if let Err(err) = shutdown_event_notifier().await { + error!( + target: "rustfs::main::handle_shutdown", + event = EVENT_EVENT_NOTIFIER_SHUTDOWN, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + state = "stop_failed", + error = %err, + "Event notifier shutdown failed" + ); + } info!( target: "rustfs::main::handle_shutdown", @@ -222,9 +366,19 @@ pub(crate) async fn run_startup_shutdown_sequence( ); } -pub(crate) async fn run_embedded_shutdown_cleanup() { - shutdown_event_notifier().await; - +async fn run_embedded_runtime_cleanup(mut cleanup: EmbeddedRuntimeCleanup) { + if let Some(notification) = cleanup.notification.take() + && let Err(err) = notification.wait().await + { + warn!( + component = LOG_COMPONENT_EMBEDDED, + subsystem = LOG_SUBSYSTEM_EMBEDDED, + event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, + service = "notification", + error = %err, + "Embedded shutdown cleanup failed" + ); + } if let Err(err) = stop_audit_system().await { warn!( component = LOG_COMPONENT_EMBEDDED, @@ -235,6 +389,38 @@ pub(crate) async fn run_embedded_shutdown_cleanup() { "Embedded shutdown cleanup failed" ); } + if let Err(err) = startup_runtime_sources::shutdown_observability_guard() { + warn!( + component = LOG_COMPONENT_EMBEDDED, + subsystem = LOG_SUBSYSTEM_EMBEDDED, + event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, + service = "observability", + error = %err, + "Embedded shutdown cleanup failed" + ); + } + cleanup.completion.cancel(); +} + +pub(crate) async fn run_embedded_shutdown_cleanup(cleanup: EmbeddedRuntimeCleanup) { + let completion = cleanup.completion.clone(); + let runtime = cleanup.runtime.clone(); + if let Err(err) = runtime.spawn(run_embedded_runtime_cleanup(cleanup)).await { + completion.cancel(); + warn!( + component = LOG_COMPONENT_EMBEDDED, + subsystem = LOG_SUBSYSTEM_EMBEDDED, + event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, + service = "process_runtime", + reason = join_failure_reason(&err), + "Embedded shutdown cleanup failed" + ); + } +} + +fn schedule_embedded_runtime_cleanup(cleanup: EmbeddedRuntimeCleanup) { + let runtime = cleanup.runtime.clone(); + runtime.spawn(run_embedded_shutdown_cleanup(cleanup)); } pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle, ctx: &CancellationToken) { @@ -242,24 +428,66 @@ pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle, ctx.cancel(); } +async fn release_embedded_runtime_after_drain(mut runtime_owner: Option) { + if let Some(runtime_owner) = runtime_owner.as_mut() + && let Some(cleanup) = runtime_owner.release() + { + run_embedded_shutdown_cleanup(cleanup).await; + } +} + pub(crate) fn run_embedded_server_drop_cleanup( + runtime: &tokio::runtime::Handle, ctx: &CancellationToken, shutdown_handle: &mut Option, - temp_dir: Option<&Path>, + temp_dir: &mut Option, + runtime_owner: Option, ) { ctx.cancel(); - if let Some(shutdown_handle) = shutdown_handle.take() { + if let Some(shutdown_handle) = shutdown_handle.as_ref() { shutdown_handle.signal(); } - if let Some(dir) = temp_dir { - let _ = std::fs::remove_dir_all(dir); + + let shutdown_handle = shutdown_handle.take(); + let temp_dir = temp_dir.take(); + runtime.spawn(finish_embedded_server_cleanup( + shutdown_handle, + temp_dir, + release_embedded_runtime_after_drain(runtime_owner), + )); +} + +async fn finish_embedded_server_cleanup(shutdown_handle: Option, temp_dir: Option, process_cleanup: F) +where + F: Future, +{ + if let Some(shutdown_handle) = shutdown_handle { + shutdown_handle.shutdown().await; + } + + process_cleanup.await; + + if let Some(dir) = temp_dir + && let Err(err) = tokio::fs::remove_dir_all(&dir).await + { + warn!( + component = LOG_COMPONENT_EMBEDDED, + subsystem = LOG_SUBSYSTEM_EMBEDDED, + event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, + service = "temp_dir", + path = %dir.display(), + error = %err, + "Embedded shutdown cleanup failed" + ); } } pub(crate) async fn run_embedded_server_shutdown( + runtime: &tokio::runtime::Handle, ctx: &CancellationToken, shutdown_handle: &mut Option, - temp_dir: Option<&Path>, + temp_dir: &mut Option, + runtime_owner: Option, ) { info!( target: "rustfs::embedded", @@ -272,22 +500,22 @@ pub(crate) async fn run_embedded_server_shutdown( ctx.cancel(); - run_embedded_shutdown_cleanup().await; - - if let Some(shutdown_handle) = shutdown_handle.take() { - shutdown_handle.shutdown().await; + if let Some(shutdown_handle) = shutdown_handle.as_ref() { + shutdown_handle.signal(); } - if let Some(dir) = temp_dir - && let Err(err) = tokio::fs::remove_dir_all(dir).await - { + let cleanup_task = runtime.spawn(finish_embedded_server_cleanup( + shutdown_handle.take(), + temp_dir.take(), + release_embedded_runtime_after_drain(runtime_owner), + )); + if let Err(err) = cleanup_task.await { warn!( component = LOG_COMPONENT_EMBEDDED, subsystem = LOG_SUBSYSTEM_EMBEDDED, event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, - service = "temp_dir", - path = %dir.display(), - error = %err, + service = "server_cleanup", + reason = join_failure_reason(&err), "Embedded shutdown cleanup failed" ); } @@ -300,25 +528,16 @@ pub(crate) async fn run_embedded_server_shutdown( state = "stopped", "Embedded server state changed" ); - - if let Err(err) = startup_runtime_sources::shutdown_observability_guard() { - warn!( - component = LOG_COMPONENT_EMBEDDED, - subsystem = LOG_SUBSYSTEM_EMBEDDED, - event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, - service = "observability", - error = %err, - "Embedded shutdown cleanup failed" - ); - } } #[cfg(test)] mod tests { use super::{ - BackgroundShutdownStep, background_shutdown_steps, run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown, + BackgroundShutdownStep, EmbeddedRuntimeOwners, background_shutdown_steps, finish_embedded_server_cleanup, + run_embedded_server_drop_cleanup, signal_embedded_startup_shutdown, }; use crate::server::ShutdownHandle; + use std::sync::{Arc, mpsc}; use std::time::Duration; use tokio::sync::broadcast; use tokio_util::sync::CancellationToken; @@ -367,15 +586,152 @@ mod tests { let cancel_token = CancellationToken::new(); let temp_dir = tempfile::tempdir().expect("temp dir should create"); let temp_path = temp_dir.path().to_path_buf(); + let mut owned_temp_path = Some(temp_path.clone()); - run_embedded_server_drop_cleanup(&cancel_token, &mut shutdown_handle, Some(temp_dir.path())); + run_embedded_server_drop_cleanup( + &tokio::runtime::Handle::current(), + &cancel_token, + &mut shutdown_handle, + &mut owned_temp_path, + None, + ); tokio::time::timeout(Duration::from_secs(1), observed_rx) .await .expect("drop cleanup should signal shutdown") .expect("shutdown signal should be delivered"); + tokio::time::timeout(Duration::from_secs(1), async { + while temp_path.exists() { + tokio::task::yield_now().await; + } + }) + .await + .expect("drop cleanup should remove the temporary directory"); assert!(cancel_token.is_cancelled()); assert!(shutdown_handle.is_none()); + assert!(owned_temp_path.is_none()); assert!(!temp_path.exists()); } + + #[test] + fn only_the_last_embedded_owner_cleans_process_runtime() { + let owners = EmbeddedRuntimeOwners::new(); + assert!(owners.register().is_none()); + assert!(owners.register().is_none()); + assert!(owners.release_with(|_| ()).is_none()); + assert!(owners.release_with(|_| ()).is_some()); + assert!(owners.release_with(|_| ()).is_none()); + } + + #[test] + fn last_owner_cleanup_intent_is_serialized_before_new_registration() { + let owners = Arc::new(EmbeddedRuntimeOwners::new()); + assert!(owners.register().is_none()); + + let (cleanup_entered_tx, cleanup_entered_rx) = mpsc::channel(); + let (allow_cleanup_tx, allow_cleanup_rx) = mpsc::channel(); + let release_owners = owners.clone(); + let release_task = std::thread::spawn(move || { + release_owners.release_with(|completion| { + cleanup_entered_tx.send(()).expect("cleanup preparation should be observed"); + allow_cleanup_rx.recv().expect("cleanup preparation should be released"); + completion.cancel(); + }) + }); + cleanup_entered_rx + .recv_timeout(Duration::from_secs(1)) + .expect("last-owner cleanup should enter preparation"); + + let (register_started_tx, register_started_rx) = mpsc::channel(); + let (register_done_tx, register_done_rx) = mpsc::channel(); + let register_owners = owners.clone(); + let register_task = std::thread::spawn(move || { + register_started_tx.send(()).expect("registration should start"); + let pending_cleanup = register_owners.register(); + register_done_tx + .send(pending_cleanup) + .expect("registration result should be observed"); + }); + register_started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("registration should start"); + let early_registration = match register_done_rx.try_recv() { + Ok(result) => Some(result), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => panic!("registration thread disconnected"), + }; + let registered_before_intent_published = early_registration.is_some(); + + allow_cleanup_tx.send(()).expect("cleanup preparation should finish"); + assert!(release_task.join().expect("release thread should not panic").is_some()); + let registration = match early_registration { + Some(result) => result, + None => register_done_rx + .recv_timeout(Duration::from_secs(1)) + .expect("registration should complete"), + }; + register_task.join().expect("register thread should not panic"); + assert!(!registered_before_intent_published); + assert!(registration.is_none()); + assert_eq!(owners.owner_count(), 1); + } + + #[tokio::test] + async fn new_owner_waits_for_prior_last_owner_cleanup() { + let owners = EmbeddedRuntimeOwners::new(); + assert!(owners.register().is_none()); + let cleanup_finished = owners + .release_with(|completion| completion) + .expect("last owner should publish a cleanup barrier"); + let pending_cleanup = owners.register().expect("new owner should observe unfinished cleanup"); + + let wait_task = tokio::spawn(async move { + pending_cleanup.cancelled().await; + }); + tokio::task::yield_now().await; + assert!(!wait_task.is_finished()); + + cleanup_finished.cancel(); + wait_task.await.expect("cleanup waiter should not panic"); + } + + #[tokio::test] + async fn server_drain_and_process_runtime_cleanup_finish_before_temp_dir_removal() { + let temp_dir = tempfile::tempdir().expect("temp dir should create"); + let temp_path = temp_dir.path().to_path_buf(); + let (shutdown_tx, mut shutdown_rx) = broadcast::channel(1); + let (shutdown_entered_tx, shutdown_entered_rx) = tokio::sync::oneshot::channel(); + let (allow_shutdown_tx, allow_shutdown_rx) = tokio::sync::oneshot::channel(); + let shutdown_task = tokio::spawn(async move { + let _ = shutdown_rx.recv().await; + shutdown_entered_tx.send(()).expect("shutdown should be observed"); + allow_shutdown_rx.await.expect("shutdown should be released"); + }); + let shutdown_handle = ShutdownHandle::new(shutdown_tx, shutdown_task); + let (cleanup_entered_tx, mut cleanup_entered_rx) = tokio::sync::oneshot::channel(); + let (allow_cleanup_tx, allow_cleanup_rx) = tokio::sync::oneshot::channel(); + + let cleanup_task = tokio::spawn(finish_embedded_server_cleanup( + Some(shutdown_handle), + Some(temp_path.clone()), + async move { + cleanup_entered_tx.send(()).expect("cleanup should be observed"); + allow_cleanup_rx.await.expect("cleanup should be released"); + }, + )); + shutdown_entered_rx.await.expect("shutdown should start"); + assert!( + matches!(cleanup_entered_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)), + "process runtime cleanup must wait for the server to drain" + ); + assert!(temp_path.exists(), "temporary data must remain available while the server drains"); + + allow_shutdown_tx.send(()).expect("shutdown should finish"); + cleanup_entered_rx.await.expect("cleanup should start"); + assert!(temp_path.exists(), "temporary data must remain available during runtime cleanup"); + + allow_cleanup_tx.send(()).expect("cleanup should finish"); + cleanup_task.await.expect("cleanup task should not panic"); + assert!(!temp_path.exists(), "temporary data should be removed only after runtime cleanup"); + } } diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index b8b550949..8b68f3aeb 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -70,9 +70,9 @@ pub(crate) use storage_api::{ init_bucket_metadata_sys, init_ecstore_config, init_local_disks_with_instance_ctx, init_lock_clients, is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, load_bucket_metadata, options_consumer, prewarm_local_disk_id_map_with_instance_ctx, read_config, record_replication_proxy, - rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize, set_bucket_metadata, - table_catalog_path_hash, to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, - try_migrate_iam_config, try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader, + rpc_consumer, runtime_sources_consumer, s3_api_consumer, serialize, set_bucket_metadata, table_catalog_path_hash, + to_s3s_etag, topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, + try_migrate_server_config, update_bucket_metadata_config, verify_rpc_signature, wrap_reader, }; #[cfg(test)] diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 864fff455..70e6c2f4e 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -16,15 +16,15 @@ use crate::admin::service::{ config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot}, site_replication::reload_site_replication_runtime_state, }; +use crate::server::MODULE_SWITCHES_SIGNAL_SUBSYSTEM; use crate::storage::storage_api::ecstore_tier::tier_mutation_peer::{self, TierMutationPeerState as EcTierMutationPeerState}; -#[cfg(test)] use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS; #[cfg(test)] use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType}; use crate::storage::storage_api::rpc_consumer::node_service::{ - DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref, - reload_transition_tier_config, + DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, + find_local_disk_by_ref, reload_transition_tier_config, }; use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources}; use crate::storage::storage_api::{sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest}; @@ -32,6 +32,9 @@ use bytes::Bytes; use futures::Stream; use futures_util::future::join_all; use rmp_serde::Deserializer; +use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; +use rustfs_config::notify::NOTIFY_SUB_SYSTEMS; +use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_filemeta::MetacacheReader; use rustfs_iam::store::UserType; use rustfs_lock::LockClient; @@ -75,6 +78,14 @@ const TIER_MUTATION_PEER_STATE_PREPARED_WIRE: i32 = 1; const TIER_MUTATION_PEER_STATE_COMMITTED_WIRE: i32 = 2; const TIER_MUTATION_PEER_STATE_ABORTED_WIRE: i32 = 3; +fn supports_dynamic_config_rpc(sub_system: &str) -> bool { + NOTIFY_SUB_SYSTEMS.contains(&sub_system) + || matches!( + sub_system, + STORAGE_CLASS_SUB_SYS | AUDIT_WEBHOOK_SUB_SYS | AUDIT_MQTT_SUB_SYS | SCANNER_SUB_SYS | HEAL_SUB_SYS + ) +} + #[derive(Debug)] struct HealControlReplayEntry { command_digest: [u8; 32], @@ -1517,6 +1528,18 @@ impl Node for NodeService { let raw_signal = vars.get(PEER_RESTSIGNAL).map(String::as_str); let signal = raw_signal.and_then(|value| value.parse::().ok()); let sub_system = vars.get(PEER_RESTSUB_SYS).map(String::as_str).unwrap_or_default(); + let dry_run = match vars.get(PEER_RESTDRY_RUN).map(String::as_str) { + None => false, + Some(value) => match value.parse::() { + Ok(value) => value, + Err(_) => { + return Ok(Response::new(SignalServiceResponse { + success: false, + error_info: Some(format!("invalid dry-run value: {value}")), + })); + } + }, + }; match signal { Some(SERVICE_SIGNAL_REFRESH_CONFIG) => match reload_runtime_config_snapshot().await { @@ -1524,21 +1547,36 @@ impl Node for NodeService { success: true, error_info: None, })), - Err(err) => Ok(Response::new(SignalServiceResponse { + Err(_) => Ok(Response::new(SignalServiceResponse { success: false, - error_info: Some(err.to_string()), - })), - }, - Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => match reload_dynamic_config_runtime_state(sub_system).await { - Ok(()) => Ok(Response::new(SignalServiceResponse { - success: true, - error_info: None, - })), - Err(err) => Ok(Response::new(SignalServiceResponse { - success: false, - error_info: Some(err.to_string()), + error_info: Some("runtime config snapshot reload failed".to_string()), })), }, + Some(SERVICE_SIGNAL_RELOAD_DYNAMIC) => { + let supported = sub_system == MODULE_SWITCHES_SIGNAL_SUBSYSTEM || supports_dynamic_config_rpc(sub_system); + if !supported { + return Ok(Response::new(SignalServiceResponse { + success: false, + error_info: Some(format!("unsupported dynamic config subsystem: {sub_system}")), + })); + } + if dry_run { + return Ok(Response::new(SignalServiceResponse { + success: true, + error_info: None, + })); + } + match reload_dynamic_config_runtime_state(sub_system).await { + Ok(()) => Ok(Response::new(SignalServiceResponse { + success: true, + error_info: None, + })), + Err(_) => Ok(Response::new(SignalServiceResponse { + success: false, + error_info: Some(format!("dynamic config reload failed for {sub_system}")), + })), + } + } Some(other) => Ok(Response::new(SignalServiceResponse { success: false, error_info: Some(format!("unsupported service signal: {other}")), @@ -1817,12 +1855,12 @@ impl Node for NodeService { #[allow(unused_imports)] mod tests { use super::{ - CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService, PEER_RESTSIGNAL, - PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, - admit_heal_control_replay, background_rebalance_start_error_message, execute_heal_control_envelope_with_manager, - initialize_heal_topology_fingerprint, make_heal_control_server, make_heal_control_server_with_cache, make_server, - make_tier_mutation_control_server_for_context, remove_heal_control_replay, scanner_activity_response, - stop_rebalance_response, + CollectMetricsOpts, DiskStore, Error, HEAL_CONTROL_PAYLOAD_MAX_SIZE, MetricType, Node as _, NodeService, + PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, + STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message, + execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, make_heal_control_server, + make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server_for_context, + remove_heal_control_replay, scanner_activity_response, stop_rebalance_response, }; use crate::storage::rpc::node_service::heal::heal_topology_fingerprint; use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint}; @@ -4185,6 +4223,44 @@ mod tests { assert!(error_info.contains("unsupported dynamic config subsystem: identity_openid")); } + #[test] + fn dynamic_config_rpc_allowlist_matches_supported_subsystems() { + for sub_system in rustfs_config::notify::NOTIFY_SUB_SYSTEMS { + assert!(super::supports_dynamic_config_rpc(sub_system)); + } + for sub_system in [ + STORAGE_CLASS_SUB_SYS, + rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, + rustfs_config::audit::AUDIT_MQTT_SUB_SYS, + rustfs_config::SCANNER_SUB_SYS, + rustfs_config::HEAL_SUB_SYS, + ] { + assert!(super::supports_dynamic_config_rpc(sub_system)); + } + assert!(!super::supports_dynamic_config_rpc("identity_openid")); + } + + #[tokio::test] + async fn test_signal_service_dry_run_accepts_notify_without_runtime_mutation() { + let service = create_test_node_service(); + + let mut vars = HashMap::new(); + vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string()); + vars.insert(PEER_RESTSUB_SYS.to_string(), rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_string()); + vars.insert(PEER_RESTDRY_RUN.to_string(), true.to_string()); + + let response = service + .signal_service(Request::new(SignalServiceRequest { + vars: Some(Mss { value: vars }), + })) + .await + .expect("notify capability probe should return a response") + .into_inner(); + + assert!(response.success, "new nodes must advertise notify lifecycle reload support"); + assert!(response.error_info.is_none()); + } + #[tokio::test] #[ignore = "requires isolated global object layer state"] #[serial_test::serial] @@ -4204,7 +4280,7 @@ mod tests { let signal_response = response.unwrap().into_inner(); assert!(!signal_response.success); let error_info = signal_response.error_info.expect("expected error info"); - assert!(error_info.contains("storage layer not initialized")); + assert_eq!(error_info, "runtime config snapshot reload failed"); } #[tokio::test] @@ -4227,7 +4303,7 @@ mod tests { let signal_response = response.unwrap().into_inner(); assert!(!signal_response.success); let error_info = signal_response.error_info.expect("expected error info"); - assert!(error_info.contains("storage layer not initialized")); + assert_eq!(error_info, format!("dynamic config reload failed for {STORAGE_CLASS_SUB_SYS}")); } fn assert_unimplemented_status(response: Result, Status>, method: &str) { diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 2e16c52b2..6df459857 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -220,11 +220,11 @@ pub(crate) mod rpc_consumer { pub(crate) mod node_service { pub(crate) use super::super::{ BatchReadVersionReq, BatchReadVersionResp, CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, - ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, - ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, - StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, - get_local_server_property, load_bucket_metadata, reload_transition_tier_config, remove_bucket_metadata, - set_bucket_metadata, validate_batch_read_version_item_count, + ECStore, Error, FileInfoVersions, LocalPeerS3Client, MetricType, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, + ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, + StorageDiskRpcExt, StoragePeerS3ClientExt, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, + find_local_disk_by_ref, get_local_server_property, load_bucket_metadata, reload_transition_tier_config, + remove_bucket_metadata, set_bucket_metadata, validate_batch_read_version_item_count, }; pub(crate) type StorageResult = super::super::Result; @@ -233,7 +233,6 @@ pub(crate) mod rpc_consumer { #[cfg(test)] pub(crate) type HealBucketInfo = super::super::contract::bucket::BucketInfo; - #[cfg(test)] pub(crate) const STORAGE_CLASS_SUB_SYS: &str = super::super::STORAGE_CLASS_SUB_SYS; pub(crate) mod contract { @@ -479,9 +478,9 @@ pub(crate) mod ecstore_rio { pub(crate) mod ecstore_rpc { pub(crate) use rustfs_ecstore::api::rpc::{ - LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, - SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, sign_tonic_rpc_response_proof, - verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature, + LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, + sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_rpc_signature, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::{ @@ -533,6 +532,7 @@ pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCK pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG; pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE; pub(crate) const OBJECT_LOCK_CONFIG: &str = ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; +pub(crate) const PEER_RESTDRY_RUN: &str = ecstore_rpc::PEER_RESTDRY_RUN; pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL; pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS; pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG; @@ -554,7 +554,6 @@ pub(crate) use ecstore_rpc::sign_tonic_rpc_response_proof; #[cfg(test)] pub(crate) use ecstore_rpc::verify_tonic_rpc_response_proof; -#[cfg(test)] pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS; pub(crate) type BucketMetadata = ecstore_bucket::metadata::BucketMetadata; @@ -873,6 +872,14 @@ pub(crate) async fn read_config(api: Arc, file: &str) -> Result ecstore_config::com::read_config(api, file).await } +pub(crate) async fn read_config_no_lock(api: Arc, file: &str) -> Result> { + ecstore_config::com::read_config_no_lock(api, file).await +} + +pub(crate) async fn read_existing_server_config_no_lock(api: Arc) -> Result { + ecstore_config::com::read_existing_server_config_no_lock(api).await +} + pub(crate) async fn prewarm_local_disk_id_map_with_instance_ctx(instance_ctx: &Arc) { ecstore_storage::prewarm_local_disk_id_map_with_instance_ctx(instance_ctx).await; } @@ -881,8 +888,35 @@ pub(crate) fn replication_queue_current_count() -> Option { get_global_replication_stats().and_then(|stats| stats.queue_current_count()) } -pub(crate) async fn save_config(api: Arc, file: &str, data: Vec) -> Result<()> { - ecstore_config::com::save_config(api, file, data).await +pub(crate) async fn save_config_no_lock(api: Arc, file: &str, data: Vec) -> Result<()> { + ecstore_config::com::save_config_no_lock(api, file, data).await +} + +pub(crate) async fn with_config_object_write_lock(api: Arc, object: String, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_config_object_write_lock(api, object, operation).await +} + +pub(crate) async fn with_config_object_read_lock(api: Arc, object: String, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_config_object_read_lock(api, object, operation).await +} + +pub(crate) async fn with_server_config_read_lock(api: Arc, operation: F) -> Result +where + F: FnOnce() -> Fut + Send + 'static, + Fut: std::future::Future + Send + 'static, + T: Send + 'static, +{ + ecstore_config::com::with_server_config_read_lock(api, operation).await } pub(crate) fn shutdown_background_services() { diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index b691899d1..539a4d582 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -89,7 +89,10 @@ pub(crate) mod server { } } - pub(crate) use crate::storage::storage_api::{EventArgs, StorageObjectInfo, register_event_dispatch_hook}; + pub(crate) use crate::storage::storage_api::{ + EventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook, + with_server_config_read_lock, + }; } pub(crate) mod http { @@ -151,7 +154,10 @@ pub(crate) mod server { } pub(crate) mod module_switch { - pub(crate) use crate::storage::storage_api::{Error, read_config, save_config}; + pub(crate) use crate::storage::storage_api::{ + Error, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, + with_config_object_write_lock, + }; } pub(crate) mod readiness { diff --git a/rustfs/tests/embedded_notification_lifecycle_test.rs b/rustfs/tests/embedded_notification_lifecycle_test.rs new file mode 100644 index 000000000..1f33e5fcd --- /dev/null +++ b/rustfs/tests/embedded_notification_lifecycle_test.rs @@ -0,0 +1,102 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use aws_sdk_s3::config::{Credentials, Region}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::{Client, Config}; +use rustfs::embedded::{RustFSServerBuilder, find_available_port}; +use rustfs_notify::{NotificationRuntimeState, notification_system}; + +fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client { + let credentials = Credentials::new(access_key, secret_key, None, None, "test"); + let config = Config::builder() + .credentials_provider(credentials) + .region(Region::new("us-east-1")) + .endpoint_url(endpoint) + .force_path_style(true) + .behavior_version_latest() + .build(); + Client::from_conf(config) +} + +#[tokio::test] +async fn notification_runtime_stays_enabled_until_the_last_embedded_owner_drains() { + temp_env::async_with_vars([(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"))], async { + let port_a = match find_available_port() { + Ok(port) => port, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("find free port for server A: {err}"), + }; + let server_a = RustFSServerBuilder::new() + .address(format!("127.0.0.1:{port_a}")) + .access_key("shared-access") + .secret_key("shared-secret") + .build() + .await + .expect("start embedded server A"); + + let port_b = match find_available_port() { + Ok(port) => port, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + server_a.shutdown().await; + return; + } + Err(err) => { + server_a.shutdown().await; + panic!("find free port for server B: {err}"); + } + }; + let server_b = RustFSServerBuilder::new() + .address(format!("127.0.0.1:{port_b}")) + .access_key("shared-access") + .secret_key("shared-secret") + .build() + .await + .expect("start embedded server B"); + + let notification = notification_system().expect("embedded startup should initialize notification runtime"); + let active_state = notification.runtime_lifecycle_state(); + assert!(matches!(active_state, NotificationRuntimeState::TargetsEnabled { .. })); + assert!(notification.runtime_lifecycle_is_converged()); + + server_b.shutdown().await; + assert_eq!( + notification.runtime_lifecycle_state(), + active_state, + "one owner must not suspend the shared notification runtime" + ); + assert!(notification.runtime_lifecycle_is_converged()); + + let client_a = s3_client(&server_a.endpoint(), server_a.access_key(), server_a.secret_key()); + client_a + .create_bucket() + .bucket("survives-notify-owner-shutdown") + .send() + .await + .expect("server A should remain usable after server B shuts down"); + client_a + .put_object() + .bucket("survives-notify-owner-shutdown") + .key("marker.txt") + .body(ByteStream::from_static(b"still here")) + .send() + .await + .expect("server A should still write after server B shuts down"); + + server_a.shutdown().await; + assert_eq!(notification.runtime_lifecycle_state(), NotificationRuntimeState::LiveOnly); + assert!(notification.runtime_lifecycle_is_converged()); + }) + .await; +}