diff --git a/Cargo.lock b/Cargo.lock index 529146aaf..2f64559aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9381,6 +9381,7 @@ dependencies = [ name = "rustfs-audit" version = "1.0.0-beta.3" dependencies = [ + "async-trait", "chrono", "const-str", "futures", @@ -9836,6 +9837,7 @@ dependencies = [ "chrono", "form_urlencoded", "hashbrown 0.17.1", + "metrics", "percent-encoding", "quick-xml 0.39.4", "rayon", diff --git a/crates/audit/AGENTS.md b/crates/audit/AGENTS.md new file mode 100644 index 000000000..39870cd65 --- /dev/null +++ b/crates/audit/AGENTS.md @@ -0,0 +1,53 @@ +# Audit Crate Instructions + +Applies to `crates/audit/`. + +`rustfs-audit` is the domain layer for audit event fan-out and observability. +It composes shared plugin/runtime abstractions from `rustfs-targets` and keeps +audit-specific dispatch semantics, state transitions, and metrics in this +crate. + +## Domain Boundaries + +- Keep audit-specific behavior here: + - audit event shaping and fan-out pipeline + - audit system lifecycle/state transitions + - audit metrics and reporting +- Keep shared plugin/runtime mechanics in `rustfs-targets`: + - no duplicated replay worker orchestration + - no duplicated runtime manager primitives + - no plugin install/control-plane modeling in this crate + +## Runtime Layering Rules + +- `pipeline.rs` hosts: + - `AuditPipeline` (dispatch and snapshot access) + - `AuditRuntimeFacade` (runtime mutation path) + - `AuditRuntimeView` (runtime read path) +- `registry.rs` should remain the single owner of runtime target container and + plugin registry composition for audit. +- `system.rs` should coordinate lifecycle by calling facade/view/registry + boundaries rather than embedding low-level runtime logic. + +## Change Style + +- Preserve audit delivery semantics and error handling behavior unless the task + explicitly changes them. +- Prefer extending shared abstractions in `rustfs-targets` over patching + one-off audit-only runtime flows. +- Keep logging and observability machine-meaningful; avoid noisy churn in hot + dispatch paths. + +## Testing + +- Keep unit tests close to changed modules. +- Keep pipeline-layer regressions in `tests/pipeline_layer_test.rs`. +- Add regression tests for: + - runtime facade activation/replace/stop/shutdown behavior + - runtime view target/snapshot access + - system reload and runtime commit/clear boundaries +- Suggested validation: + - `cargo test -p rustfs-audit` + - Focused: `cargo test -p rustfs-audit --test pipeline_layer_test` + - Focused: `cargo test -p rustfs-audit pipeline` +- Full gate before commit: `make pre-commit` diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index 77cea6b84..04ce7cda6 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -42,6 +42,7 @@ tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", " tracing = { workspace = true, features = ["std", "attributes"] } [dev-dependencies] +async-trait = { workspace = true } temp-env = { workspace = true } url = { workspace = true } @@ -49,5 +50,4 @@ url = { workspace = true } workspace = true [lib] -test = false doctest = false diff --git a/crates/audit/src/factory.rs b/crates/audit/src/factory.rs index 0b51fcc07..83ae1ff7f 100644 --- a/crates/audit/src/factory.rs +++ b/crates/audit/src/factory.rs @@ -13,138 +13,11 @@ // limitations under the License. use crate::AuditEntry; -use rustfs_config::AUDIT_DEFAULT_DIR; -use rustfs_config::audit::{ - AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, - AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS, -}; -use rustfs_targets::config::{ - build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, - build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, - validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, - validate_webhook_config, -}; -use rustfs_targets::target::{ChannelTargetType, TargetType}; -use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target}; +use rustfs_targets::catalog::builtin::builtin_audit_target_descriptors; +use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor}; pub fn builtin_target_descriptors() -> Vec> { - vec![ - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_AMQP_SUB_SYS, - TargetRequestValidator::Amqp(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::Amqp.as_str(), - AUDIT_AMQP_KEYS, - |config| validate_amqp_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS, - TargetRequestValidator::Webhook, - TargetPluginDescriptor::new( - ChannelTargetType::Webhook.as_str(), - AUDIT_WEBHOOK_KEYS, - |config| validate_webhook_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_MQTT_SUB_SYS, - TargetRequestValidator::Mqtt, - TargetPluginDescriptor::new(ChannelTargetType::Mqtt.as_str(), AUDIT_MQTT_KEYS, validate_mqtt_config, |id, config| { - let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?)) - }), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_NATS_SUB_SYS, - TargetRequestValidator::Nats(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::Nats.as_str(), - AUDIT_NATS_KEYS, - |config| validate_nats_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_PULSAR_SUB_SYS, - TargetRequestValidator::Pulsar(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::Pulsar.as_str(), - AUDIT_PULSAR_KEYS, - |config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_KAFKA_SUB_SYS, - TargetRequestValidator::Kafka(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::Kafka.as_str(), - AUDIT_KAFKA_KEYS, - |config| validate_kafka_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_REDIS_SUB_SYS, - TargetRequestValidator::Redis { - default_channel: AUDIT_REDIS_DEFAULT_CHANNEL, - target_type: TargetType::AuditLog, - }, - TargetPluginDescriptor::new( - ChannelTargetType::Redis.as_str(), - AUDIT_REDIS_KEYS, - |config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL), - |id, config| { - let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_MYSQL_SUB_SYS, - TargetRequestValidator::MySql(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::MySql.as_str(), - AUDIT_MYSQL_KEYS, - |config| validate_mysql_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - rustfs_config::audit::AUDIT_POSTGRES_SUB_SYS, - TargetRequestValidator::Postgres(TargetType::AuditLog), - TargetPluginDescriptor::new( - ChannelTargetType::Postgres.as_str(), - AUDIT_POSTGRES_KEYS, - |config| validate_postgres_config(config, AUDIT_DEFAULT_DIR), - |id, config| { - let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; - Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?)) - }, - ), - ), - ] + builtin_audit_target_descriptors::() } pub fn builtin_target_plugins() -> Vec> { diff --git a/crates/audit/src/lib.rs b/crates/audit/src/lib.rs index 2efd99cd7..c7c3d8849 100644 --- a/crates/audit/src/lib.rs +++ b/crates/audit/src/lib.rs @@ -23,6 +23,7 @@ pub mod error; pub mod factory; pub mod global; pub mod observability; +pub mod pipeline; pub mod registry; pub mod system; @@ -30,5 +31,6 @@ pub use entity::{ApiDetails, AuditEntry, ObjectVersion}; pub use error::{AuditError, AuditResult}; pub use global::*; pub use observability::{AuditMetrics, AuditMetricsReport, PerformanceValidation}; +pub use pipeline::{AuditPipeline, AuditRuntimeFacade, AuditRuntimeView}; pub use registry::AuditRegistry; pub use system::{AuditSystem, AuditTargetMetricSnapshot}; diff --git a/crates/audit/src/pipeline.rs b/crates/audit/src/pipeline.rs new file mode 100644 index 000000000..0457cfeb2 --- /dev/null +++ b/crates/audit/src/pipeline.rs @@ -0,0 +1,355 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{AuditEntry, AuditResult, observability, system::AuditTargetMetricSnapshot}; +use rustfs_targets::{ + BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, SharedTarget, Target, + target::EntityTarget, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Mutex, RwLock}; +use tracing::{error, info, warn}; + +#[derive(Clone)] +pub struct AuditPipeline { + registry: Arc>, +} + +impl AuditPipeline { + pub fn new(registry: Arc>) -> Self { + Self { registry } + } + + pub async fn dispatch(&self, entry: Arc) -> AuditResult<()> { + let start_time = std::time::Instant::now(); + + let targets: Vec> = { + let registry = self.registry.lock().await; + let targets = registry.list_target_values(); + + if targets.is_empty() { + warn!("No audit targets configured for dispatch"); + return Ok(()); + } + + targets + }; + + let mut tasks = Vec::new(); + + for target in targets { + let entity_target = EntityTarget { + object_name: entry.api.name.clone().unwrap_or_default(), + bucket_name: entry.api.bucket.clone().unwrap_or_default(), + event_name: entry.event, + data: (*entry).clone(), + }; + + let task = async move { + let result = target.save(Arc::new(entity_target)).await; + (target.id().to_string(), result) + }; + + tasks.push(task); + } + + let results = futures::future::join_all(tasks).await; + + let mut errors = Vec::new(); + let mut success_count = 0; + + for (target_key, result) in results { + match result { + Ok(_) => { + success_count += 1; + observability::record_target_success(); + } + Err(e) => { + error!(target_id = %target_key, error = %e, "Failed to dispatch audit log to target"); + errors.push(e); + observability::record_target_failure(); + } + } + } + + let dispatch_time = start_time.elapsed(); + + if errors.is_empty() { + observability::record_audit_success(dispatch_time); + } else { + observability::record_audit_failure(dispatch_time); + warn!( + error_count = errors.len(), + success_count = success_count, + "Some audit targets failed to receive log entry" + ); + } + + Ok(()) + } + + pub async fn dispatch_batch(&self, entries: Vec>) -> AuditResult<()> { + let start_time = std::time::Instant::now(); + + let targets: Vec> = { + let registry = self.registry.lock().await; + let targets = registry.list_target_values(); + + if targets.is_empty() { + warn!("No audit targets configured for batch dispatch"); + return Ok(()); + } + + targets + }; + + let mut tasks = Vec::new(); + for target in targets { + let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect(); + + let task = async move { + let mut success_count = 0; + let mut errors = Vec::new(); + for entry in entries_clone { + let entity_target = EntityTarget { + object_name: entry.api.name.clone().unwrap_or_default(), + bucket_name: entry.api.bucket.clone().unwrap_or_default(), + event_name: entry.event, + data: (*entry).clone(), + }; + match target.save(Arc::new(entity_target)).await { + Ok(_) => success_count += 1, + Err(e) => errors.push(e), + } + } + (target.id().to_string(), success_count, errors) + }; + tasks.push(task); + } + + let results = futures::future::join_all(tasks).await; + let mut total_success = 0; + let mut total_errors = 0; + for (_target_id, success_count, errors) in results { + total_success += success_count; + total_errors += errors.len(); + for e in errors { + error!("Batch dispatch error: {:?}", e); + } + } + + let dispatch_time = start_time.elapsed(); + info!( + "Batch dispatched {} entries, success: {}, errors: {}, time: {:?}", + entries.len(), + total_success, + total_errors, + dispatch_time + ); + + Ok(()) + } + + pub async fn snapshot_target_metrics(&self) -> Vec { + let registry = self.registry.lock().await; + registry + .list_target_values() + .into_iter() + .map(|target| { + let delivery = target.delivery_snapshot(); + AuditTargetMetricSnapshot { + failed_messages: delivery.failed_messages, + queue_length: delivery.queue_length, + target_id: target.id().to_string(), + total_messages: delivery.total_messages, + } + }) + .collect() + } + + pub async fn snapshot_target_health(&self) -> Vec { + let registry = self.registry.lock().await; + registry.runtime_manager().health_snapshots().await + } +} + +#[derive(Clone)] +pub struct AuditRuntimeView { + registry: Arc>, +} + +impl AuditRuntimeView { + pub fn new(registry: Arc>) -> Self { + Self { registry } + } + + pub async fn list_targets(&self) -> Vec { + let registry = self.registry.lock().await; + registry.list_targets() + } + + pub async fn get_target_values(&self) -> Vec> { + let registry = self.registry.lock().await; + registry.list_target_values() + } + + pub async fn get_target(&self, target_id: &str) -> Option { + let registry = self.registry.lock().await; + registry.get_target(target_id).map(|target| target.id().to_string()) + } + + pub async fn enable_target(&self, target_id: &str) -> AuditResult<()> { + let registry = self.registry.lock().await; + if registry.get_target(target_id).is_some() { + info!(target_id = %target_id, "Target enabled"); + Ok(()) + } else { + Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None)) + } + } + + pub async fn disable_target(&self, target_id: &str) -> AuditResult<()> { + let registry = self.registry.lock().await; + if registry.get_target(target_id).is_some() { + info!(target_id = %target_id, "Target disabled"); + Ok(()) + } else { + Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None)) + } + } + + pub async fn remove_target(&self, target_id: &str) -> AuditResult<()> { + let mut registry = self.registry.lock().await; + if registry.remove_target(target_id).await.is_some() { + info!(target_id = %target_id, "Target removed"); + Ok(()) + } else { + Err(crate::AuditError::Configuration(format!("Target not found: {target_id}"), None)) + } + } + + pub async fn upsert_target(&self, target_id: String, target: Box + Send + Sync>) -> AuditResult<()> { + if let Err(err) = target.init().await { + return Err(crate::AuditError::Target(err)); + } + + let shared_target: SharedTarget = Arc::from(target); + let mut registry = self.registry.lock().await; + let _ = registry.remove_target(&target_id).await; + registry.add_shared_target(target_id.clone(), shared_target); + info!(target_id = %target_id, "Target upserted"); + Ok(()) + } +} + +#[derive(Clone)] +pub struct AuditRuntimeFacade { + registry: Arc>, + replay_workers: Arc>, + runtime_adapter: Arc>, +} + +impl AuditRuntimeFacade { + pub fn new(registry: Arc>, replay_workers: Arc>) -> Self { + let runtime_adapter = BuiltinPluginRuntimeAdapter::new( + Arc::new(move |event: ReplayEvent| { + Box::pin(async move { + match event { + ReplayEvent::Delivered { key, target } => { + info!("Successfully sent audit entry, target: {}, key: {}", target.id(), key.to_string()); + observability::record_target_success(); + } + ReplayEvent::RetryableError { error, target, .. } => match error { + rustfs_targets::TargetError::NotConnected => { + warn!("Target {} not connected, retrying...", target.id()); + } + rustfs_targets::TargetError::Timeout(_) => { + warn!("Timeout sending to target {}, retrying...", target.id()); + } + _ => {} + }, + ReplayEvent::Dropped { reason, target, .. } => { + warn!("Dropped queued payload for target {}: {}", target.id(), reason); + observability::record_target_failure(); + } + ReplayEvent::PermanentFailure { error, target, .. } => { + error!("Permanent error for target {}: {}", target.id(), error); + target.record_final_failure(); + observability::record_target_failure(); + } + ReplayEvent::RetryExhausted { key, target } => { + warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id()); + target.record_final_failure(); + observability::record_target_failure(); + } + ReplayEvent::UnreadableEntry { key, error, target } => { + warn!("Skipping unreadable audit store entry {} for target {}: {}", key, target.id(), error); + } + } + }) + }), + Arc::new(|target_id, has_replay| { + if has_replay { + info!(target_id = %target_id, "Audit stream processing started"); + } else { + info!(target_id = %target_id, "No store configured, skip audit stream processing"); + } + }), + None, + Duration::from_millis(500), + Duration::from_millis(500), + "Stopping audit stream", + ); + + Self { + registry, + replay_workers, + runtime_adapter: Arc::new(runtime_adapter), + } + } + + pub async fn replace_targets(&self, activation: RuntimeActivation) -> AuditResult<()> { + let mut registry = self.registry.lock().await; + let mut replay_workers = self.replay_workers.write().await; + self.runtime_adapter + .replace_runtime_targets(registry.runtime_manager_mut(), &mut replay_workers, activation) + .await + .map_err(crate::AuditError::Target)?; + Ok(()) + } + + pub async fn shutdown_runtime( + &self, + registry: &mut crate::AuditRegistry, + replay_workers: &mut ReplayWorkerManager, + ) -> AuditResult<()> { + self.runtime_adapter + .shutdown(registry.runtime_manager_mut(), replay_workers) + .await + .map_err(crate::AuditError::Target) + } + + pub async fn activate_targets_with_replay( + &self, + targets: Vec + Send + Sync>>, + ) -> RuntimeActivation { + self.runtime_adapter.activate_with_replay(targets).await + } + + pub async fn stop_replay_workers(&self) { + let mut replay_workers = self.replay_workers.write().await; + self.runtime_adapter.stop_replay_workers(&mut replay_workers).await; + } +} diff --git a/crates/audit/src/registry.rs b/crates/audit/src/registry.rs index b038eb783..2270c3194 100644 --- a/crates/audit/src/registry.rs +++ b/crates/audit/src/registry.rs @@ -13,17 +13,16 @@ // limitations under the License. use crate::{AuditEntry, AuditError, AuditResult, factory::builtin_target_plugins}; -use hashbrown::HashMap; use rustfs_config::audit::AUDIT_ROUTE_PREFIX; use rustfs_ecstore::config::{Config, KVS}; use rustfs_targets::arn::TargetID; -use rustfs_targets::{Target, TargetError, TargetPluginRegistry}; -use tracing::{error, info}; +use rustfs_targets::{SharedTarget, Target, TargetError, TargetPluginRegistry, TargetRuntimeManager}; +use tracing::info; /// Registry for managing audit targets pub struct AuditRegistry { /// Storage for created targets - targets: HashMap + Send + Sync>>, + targets: TargetRuntimeManager, /// Registered plugins for creating targets plugins: TargetPluginRegistry, } @@ -41,7 +40,7 @@ impl AuditRegistry { plugins.register_all(builtin_target_plugins()); AuditRegistry { - targets: HashMap::new(), + targets: TargetRuntimeManager::new(), plugins, } } @@ -92,8 +91,14 @@ impl AuditRegistry { /// # Arguments /// * `id` - The identifier for the target. /// * `target` - The target instance to be added. - pub fn add_target(&mut self, id: String, target: Box + Send + Sync>) { - self.targets.insert(id, target); + pub fn add_target(&mut self, _id: String, target: Box + Send + Sync>) { + debug_assert_eq!(_id, target.id().to_string()); + self.targets.add_boxed(target); + } + + pub fn add_shared_target(&mut self, _id: String, target: SharedTarget) { + debug_assert_eq!(_id, target.id().to_string()); + self.targets.add_arc(target); } /// Removes a target from the registry @@ -103,8 +108,8 @@ impl AuditRegistry { /// /// # Returns /// * `Option + Send + Sync>>` - The removed target if it existed. - pub fn remove_target(&mut self, id: &str) -> Option + Send + Sync>> { - self.targets.remove(id) + pub async fn remove_target(&mut self, id: &str) -> Option> { + self.targets.remove_and_close(id).await } /// Gets a target from the registry @@ -114,13 +119,21 @@ impl AuditRegistry { /// /// # Returns /// * `Option<&(dyn Target + Send + Sync)>` - The target if it exists. - pub fn get_target(&self, id: &str) -> Option<&(dyn Target + Send + Sync)> { - self.targets.get(id).map(|t| t.as_ref()) + pub fn get_target(&self, id: &str) -> Option> { + self.targets.get(id) } /// Lists cloned target values for runtime inspection without exposing mutable registry access. - pub fn list_target_values(&self) -> Vec + Send + Sync>> { - self.targets.values().map(|target| target.clone_dyn()).collect() + pub fn list_target_values(&self) -> Vec> { + self.targets.values() + } + + pub fn runtime_manager(&self) -> &TargetRuntimeManager { + &self.targets + } + + pub fn runtime_manager_mut(&mut self) -> &mut TargetRuntimeManager { + &mut self.targets } /// Lists all target IDs @@ -128,7 +141,7 @@ impl AuditRegistry { /// # Returns /// * `Vec` - A vector of all target IDs in the registry. pub fn list_targets(&self) -> Vec { - self.targets.keys().cloned().collect() + self.targets.keys() } /// Closes all targets and clears the registry @@ -136,20 +149,23 @@ impl AuditRegistry { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure. pub async fn close_all(&mut self) -> AuditResult<()> { - let mut errors = Vec::new(); + let mut first_error = None; - for (id, target) in self.targets.drain() { - if let Err(e) = target.close().await { - error!(target_id = %id, error = %e, "Failed to close audit target"); - errors.push(e); + for target_id in self.targets.keys() { + if let Some(target) = self.targets.remove(&target_id) + && let Err(err) = target.close().await + { + tracing::error!(target_id = %target_id, error = %err, "Failed to close target during shutdown"); + if first_error.is_none() { + first_error = Some(err); + } } } - if let Some(error) = errors.into_iter().next() { - return Err(AuditError::Target(error)); + match first_error { + Some(err) => Err(AuditError::Target(err)), + None => Ok(()), } - - Ok(()) } /// Creates a unique key for a target based on its type and ID @@ -224,7 +240,8 @@ impl AuditRegistry { target: Box + Send + Sync>, ) -> AuditResult<()> { let key = self.create_key(target_type, target_id); - self.targets.insert(key, target); + debug_assert_eq!(key, target.id().to_string()); + self.targets.add_boxed(target); Ok(()) } } @@ -232,7 +249,70 @@ impl AuditRegistry { #[cfg(test)] mod tests { use super::AuditRegistry; - use rustfs_targets::target::ChannelTargetType; + use crate::{AuditEntry, AuditError}; + use rustfs_targets::arn::TargetID; + use rustfs_targets::store::{Key, Store}; + use rustfs_targets::target::{ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use rustfs_targets::{StoreError, Target, TargetError}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Clone)] + struct CloseTestTarget { + id: TargetID, + close_calls: Arc, + fail_on_close: bool, + } + + impl CloseTestTarget { + fn new(id: TargetID, close_calls: Arc, fail_on_close: bool) -> Self { + Self { + id, + close_calls, + fail_on_close, + } + } + } + + #[async_trait::async_trait] + impl Target for CloseTestTarget { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_on_close { + Err(TargetError::Unknown("close failed".to_string())) + } else { + Ok(()) + } + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + fn is_enabled(&self) -> bool { + true + } + } #[test] fn registry_registers_amqp_factory() { @@ -240,4 +320,27 @@ mod tests { assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str())); } + + #[tokio::test] + async fn close_all_returns_first_error_and_clears_targets() { + let mut registry = AuditRegistry::new(); + let ok_calls = Arc::new(AtomicUsize::new(0)); + let fail_calls = Arc::new(AtomicUsize::new(0)); + + let ok_id = TargetID::new("ok".to_string(), "webhook".to_string()); + let fail_id = TargetID::new("fail".to_string(), "webhook".to_string()); + + registry.add_target(ok_id.to_string(), Box::new(CloseTestTarget::new(ok_id, Arc::clone(&ok_calls), false))); + registry.add_target( + fail_id.to_string(), + Box::new(CloseTestTarget::new(fail_id, Arc::clone(&fail_calls), true)), + ); + + let result = registry.close_all().await; + + assert!(matches!(result, Err(AuditError::Target(TargetError::Unknown(_))))); + assert_eq!(ok_calls.load(Ordering::SeqCst), 1); + assert_eq!(fail_calls.load(Ordering::SeqCst), 1); + assert!(registry.list_targets().is_empty()); + } } diff --git a/crates/audit/src/system.rs b/crates/audit/src/system.rs index 2c050a682..87bb50fc7 100644 --- a/crates/audit/src/system.rs +++ b/crates/audit/src/system.rs @@ -12,16 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability}; -use hashbrown::HashMap; -use rustfs_ecstore::config::Config; -use rustfs_targets::{ - StoreError, Target, TargetError, - store::{Key, Store}, - target::{EntityTarget, QueuedPayload}, +use crate::{ + AuditEntry, AuditError, AuditRegistry, AuditResult, observability, + pipeline::{AuditPipeline, AuditRuntimeFacade, AuditRuntimeView}, }; +use rustfs_ecstore::config::Config; +use rustfs_targets::{ReplayWorkerManager, Target}; use std::sync::Arc; -use tokio::sync::{Mutex, RwLock, mpsc}; +use tokio::sync::{Mutex, RwLock}; use tracing::{error, info, warn}; #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -49,7 +47,7 @@ pub struct AuditSystem { state: Arc>, config: Arc>>, /// Cancellation senders for active audit stream tasks (target_id -> cancel tx) - stream_cancellers: Arc>>>, + stream_cancellers: Arc>, } impl Default for AuditSystem { @@ -59,16 +57,68 @@ impl Default for AuditSystem { } impl AuditSystem { + fn pipeline(&self) -> AuditPipeline { + AuditPipeline::new(self.registry.clone()) + } + + fn runtime_view(&self) -> AuditRuntimeView { + AuditRuntimeView::new(self.registry.clone()) + } + + fn runtime_facade(&self) -> AuditRuntimeFacade { + AuditRuntimeFacade::new(self.registry.clone(), self.stream_cancellers.clone()) + } + /// Creates a new audit system pub fn new() -> Self { Self { registry: Arc::new(Mutex::new(AuditRegistry::new())), state: Arc::new(RwLock::new(AuditSystemState::Stopped)), config: Arc::new(RwLock::new(None)), - stream_cancellers: Arc::new(RwLock::new(HashMap::new())), + stream_cancellers: Arc::new(RwLock::new(ReplayWorkerManager::new())), } } + async fn create_targets_from_config(&self, config: &Config) -> AuditResult + Send + Sync>>> { + let registry = self.registry.lock().await; + registry.create_audit_targets_from_config(config).await + } + + async fn clear_runtime_targets(&self) -> AuditResult<()> { + { + let mut registry = self.registry.lock().await; + let mut replay_workers = self.stream_cancellers.write().await; + self.runtime_facade() + .shutdown_runtime(&mut registry, &mut replay_workers) + .await?; + } + + let mut state = self.state.write().await; + *state = AuditSystemState::Stopped; + Ok(()) + } + + async fn commit_runtime_targets( + &self, + targets: Vec + Send + Sync>>, + final_state: AuditSystemState, + ) -> AuditResult<()> { + if targets.is_empty() { + info!("No enabled audit targets found, keeping audit system stopped"); + self.clear_runtime_targets().await?; + return Ok(()); + } + + info!(target_count = targets.len(), "Created audit targets successfully"); + + let activation = self.runtime_facade().activate_targets_with_replay(targets).await; + self.runtime_facade().replace_targets(activation).await?; + + let mut state = self.state.write().await; + *state = final_state; + Ok(()) + } + /// Starts the audit system with the given configuration /// /// # Arguments @@ -103,31 +153,14 @@ impl AuditSystem { *config_guard = Some(config.clone()); } - // Create targets from configuration - let mut registry = self.registry.lock().await; - match registry.create_audit_targets_from_config(&config).await { + match self.create_targets_from_config(&config).await { Ok(targets) => { - if targets.is_empty() { - info!("No enabled audit targets found, keeping audit system stopped"); - drop(registry); - return Ok(()); - } - { let mut state = self.state.write().await; *state = AuditSystemState::Starting; } - info!(target_count = targets.len(), "Created audit targets successfully"); - - // Initialize all targets - for target in targets { - self.init_and_register_target(target, &mut registry).await; - } - - // Update state to running - let mut state = self.state.write().await; - *state = AuditSystemState::Running; + self.commit_runtime_targets(targets, AuditSystemState::Running).await?; info!("Audit system started successfully"); Ok(()) } @@ -207,18 +240,10 @@ impl AuditSystem { info!("Stopping audit system"); // Stop all stream tasks first - self.stop_all_streams().await; - - // Close all targets - let mut registry = self.registry.lock().await; - if let Err(e) = registry.close_all().await { + if let Err(e) = self.clear_runtime_targets().await { error!(error = %e, "Failed to close some audit targets"); } - // Update state to stopped - let mut state = self.state.write().await; - *state = AuditSystemState::Stopped; - // Clear configuration let mut config_guard = self.config.write().await; *config_guard = None; @@ -248,8 +273,6 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn dispatch(&self, entry: Arc) -> AuditResult<()> { - let start_time = std::time::Instant::now(); - let state = self.state.read().await; match *state { @@ -262,77 +285,7 @@ impl AuditSystem { } } drop(state); - - // Collect cloned targets under lock, then dispatch without holding it - let targets: Vec<(String, Box + Send + Sync>)> = { - let registry = self.registry.lock().await; - let target_keys = registry.list_targets(); - - if target_keys.is_empty() { - warn!("No audit targets configured for dispatch"); - return Ok(()); - } - - target_keys - .into_iter() - .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn()))) - .collect() - }; - - // Dispatch to all targets concurrently (no lock held) - let mut tasks = Vec::new(); - - for (target_key, target) in targets { - let entity_target = EntityTarget { - object_name: entry.api.name.clone().unwrap_or_default(), - bucket_name: entry.api.bucket.clone().unwrap_or_default(), - event_name: entry.event, - data: (*entry).clone(), - }; - - let task = async move { - let result = target.save(Arc::new(entity_target)).await; - (target_key, result) - }; - - tasks.push(task); - } - - // Execute all dispatch tasks - let results = futures::future::join_all(tasks).await; - - let mut errors = Vec::new(); - let mut success_count = 0; - - for (target_key, result) in results { - match result { - Ok(_) => { - success_count += 1; - observability::record_target_success(); - } - Err(e) => { - error!(target_id = %target_key, error = %e, "Failed to dispatch audit log to target"); - errors.push(e); - observability::record_target_failure(); - } - } - } - - let dispatch_time = start_time.elapsed(); - - if errors.is_empty() { - observability::record_audit_success(dispatch_time); - } else { - observability::record_audit_failure(dispatch_time); - // Log errors but don't fail the entire dispatch - warn!( - error_count = errors.len(), - success_count = success_count, - "Some audit targets failed to receive log entry" - ); - } - - Ok(()) + self.pipeline().dispatch(entry).await } /// Dispatches a batch of audit log entries to all active targets @@ -343,239 +296,12 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn dispatch_batch(&self, entries: Vec>) -> AuditResult<()> { - let start_time = std::time::Instant::now(); - let state = self.state.read().await; if *state != AuditSystemState::Running { return Err(AuditError::NotInitialized("Audit system is not running".to_string())); } drop(state); - - // Collect targets under lock, then dispatch without holding it - let targets: Vec<(String, Box + Send + Sync>)> = { - let registry = self.registry.lock().await; - let target_keys = registry.list_targets(); - - if target_keys.is_empty() { - warn!("No audit targets configured for batch dispatch"); - return Ok(()); - } - - target_keys - .into_iter() - .filter_map(|key| registry.get_target(&key).map(|t| (key, t.clone_dyn()))) - .collect() - }; - - let mut tasks = Vec::new(); - for (target_key, target) in targets { - let entries_clone: Vec<_> = entries.iter().map(Arc::clone).collect(); - let target_key_clone = target_key.clone(); - - let task = async move { - let mut success_count = 0; - let mut errors = Vec::new(); - for entry in entries_clone { - let entity_target = EntityTarget { - object_name: entry.api.name.clone().unwrap_or_default(), - bucket_name: entry.api.bucket.clone().unwrap_or_default(), - event_name: entry.event, - data: (*entry).clone(), - }; - match target.save(Arc::new(entity_target)).await { - Ok(_) => success_count += 1, - Err(e) => errors.push(e), - } - } - (target_key_clone, success_count, errors) - }; - tasks.push(task); - } - - let results = futures::future::join_all(tasks).await; - let mut total_success = 0; - let mut total_errors = 0; - for (_target_id, success_count, errors) in results { - total_success += success_count; - total_errors += errors.len(); - for e in errors { - error!("Batch dispatch error: {:?}", e); - } - } - - let dispatch_time = start_time.elapsed(); - info!( - "Batch dispatched {} entries, success: {}, errors: {}, time: {:?}", - entries.len(), - total_success, - total_errors, - dispatch_time - ); - - Ok(()) - } - - /// Stops all active audit stream tasks by sending cancellation signals. - async fn stop_all_streams(&self) { - let mut cancellers = self.stream_cancellers.write().await; - for (target_id, cancel_tx) in cancellers.drain() { - info!(target_id = %target_id, "Stopping audit stream"); - let _ = cancel_tx.send(()).await; - } - } - - /// Initializes a single target: runs init(), starts stream if store is present, - /// and adds it to the registry. For store-backed targets, registration and stream - /// startup proceed even if init() fails so queued entries can be drained later. - async fn init_and_register_target( - &self, - target: Box + Send + Sync>, - registry: &mut AuditRegistry, - ) -> Option { - let target_id = target.id().to_string(); - let has_store = target.store().is_some(); - - if let Err(e) = target.init().await { - error!(target_id = %target_id, error = %e, "Failed to initialize audit target"); - // Non-store targets: init failure is fatal. - if !has_store { - return None; - } - // Store-backed targets: still register and start the stream so queued - // entries can be drained when connectivity recovers. - warn!( - target_id = %target_id, - "Proceeding with store-backed audit target despite init failure" - ); - } - - if target.is_enabled() { - if let Some(store) = target.store() { - info!(target_id = %target_id, "Start audit stream processing for target"); - let store_clone: Box + Send> = store.boxed_clone(); - let target_arc: Arc + Send + Sync> = Arc::from(target.clone_dyn()); - let cancel_tx = self.start_audit_stream_with_batching(store_clone, target_arc); - - self.stream_cancellers.write().await.insert(target_id.clone(), cancel_tx); - info!(target_id = %target_id, "Audit stream processing started"); - } else { - info!(target_id = %target_id, "No store configured, skip audit stream processing"); - } - } else { - info!(target_id = %target_id, "Target disabled, skip audit stream processing"); - } - - registry.add_target(target_id.clone(), target); - Some(target_id) - } - - /// Starts the audit stream processing for a target with batching and retry logic - /// - /// # Arguments - /// * `store` - The store from which to read audit entries - /// * `target` - The target to which audit entries will be sent - /// - /// This function spawns a background task that continuously reads audit entries from the provided store - /// and attempts to send them to the specified target. It implements retry logic with exponential backoff - fn start_audit_stream_with_batching( - &self, - store: Box + Send>, - target: Arc + Send + Sync>, - ) -> mpsc::Sender<()> { - let (cancel_tx, mut cancel_rx) = mpsc::channel(1); - let state = self.state.clone(); - - tokio::spawn(async move { - use std::time::Duration; - use tokio::time::sleep; - - info!("Starting audit stream for target: {}", target.id()); - - const MAX_RETRIES: usize = 5; - const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); - - loop { - // Check for cancellation signal - if cancel_rx.try_recv().is_ok() { - info!("Audit stream cancelled for target: {}", target.id()); - break; - } - - match *state.read().await { - AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {} - _ => { - info!("Audit stream stopped for target: {}", target.id()); - break; - } - } - - let keys: Vec = store.list(); - if keys.is_empty() { - tokio::select! { - _ = sleep(Duration::from_millis(500)) => {}, - _ = cancel_rx.recv() => { - info!("Audit stream cancelled during idle for target: {}", target.id()); - return; - } - } - continue; - } - - for key in keys { - if cancel_rx.try_recv().is_ok() { - info!("Audit stream cancelled during processing for target: {}", target.id()); - return; - } - - let mut retries = 0usize; - let mut success = false; - - while retries < MAX_RETRIES && !success { - match target.send_from_store(key.clone()).await { - Ok(_) => { - info!("Successfully sent audit entry, target: {}, key: {}", target.id(), key.to_string()); - observability::record_target_success(); - success = true; - } - Err(e) => { - match &e { - TargetError::NotConnected => { - warn!("Target {} not connected, retrying...", target.id()); - } - TargetError::Timeout(_) => { - warn!("Timeout sending to target {}, retrying...", target.id()); - } - TargetError::Dropped(reason) => { - warn!("Dropped queued payload for target {}: {}", target.id(), reason); - observability::record_target_failure(); - break; - } - _ => { - error!("Permanent error for target {}: {}", target.id(), e); - target.record_final_failure(); - observability::record_target_failure(); - break; - } - } - retries += 1; - let backoff = BASE_RETRY_DELAY * (1 << retries); - sleep(backoff).await; - } - } - } - - if retries >= MAX_RETRIES && !success { - warn!("Max retries exceeded for key {}, target: {}, skipping", key.to_string(), target.id()); - target.record_final_failure(); - observability::record_target_failure(); - } - } - - sleep(Duration::from_millis(100)).await; - } - }); - - cancel_tx + self.pipeline().dispatch_batch(entries).await } /// Enables a specific target @@ -586,15 +312,7 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn enable_target(&self, target_id: &str) -> AuditResult<()> { - // This would require storing enabled/disabled state per target - // For now, just check if target exists - let registry = self.registry.lock().await; - if registry.get_target(target_id).is_some() { - info!(target_id = %target_id, "Target enabled"); - Ok(()) - } else { - Err(AuditError::Configuration(format!("Target not found: {target_id}"), None)) - } + self.runtime_view().enable_target(target_id).await } /// Disables a specific target @@ -605,15 +323,7 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn disable_target(&self, target_id: &str) -> AuditResult<()> { - // This would require storing enabled/disabled state per target - // For now, just check if target exists - let registry = self.registry.lock().await; - if registry.get_target(target_id).is_some() { - info!(target_id = %target_id, "Target disabled"); - Ok(()) - } else { - Err(AuditError::Configuration(format!("Target not found: {target_id}"), None)) - } + self.runtime_view().disable_target(target_id).await } /// Removes a target from the system @@ -624,16 +334,7 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn remove_target(&self, target_id: &str) -> AuditResult<()> { - let mut registry = self.registry.lock().await; - if let Some(target) = registry.remove_target(target_id) { - if let Err(e) = target.close().await { - error!(target_id = %target_id, error = %e, "Failed to close removed target"); - } - info!(target_id = %target_id, "Target removed"); - Ok(()) - } else { - Err(AuditError::Configuration(format!("Target not found: {target_id}"), None)) - } + self.runtime_view().remove_target(target_id).await } /// Updates or inserts a target @@ -645,23 +346,7 @@ impl AuditSystem { /// # Returns /// * `AuditResult<()>` - Result indicating success or failure pub async fn upsert_target(&self, target_id: String, target: Box + Send + Sync>) -> AuditResult<()> { - let mut registry = self.registry.lock().await; - - // Initialize the target - if let Err(e) = target.init().await { - return Err(AuditError::Target(e)); - } - - // Remove existing target if present - if let Some(old_target) = registry.remove_target(&target_id) - && let Err(e) = old_target.close().await - { - error!(target_id = %target_id, error = %e, "Failed to close old target during upsert"); - } - - registry.add_target(target_id.clone(), target); - info!(target_id = %target_id, "Target upserted"); - Ok(()) + self.runtime_view().upsert_target(target_id, target).await } /// Lists all targets @@ -669,33 +354,27 @@ impl AuditSystem { /// # Returns /// * `Vec` - List of target IDs pub async fn list_targets(&self) -> Vec { - let registry = self.registry.lock().await; - registry.list_targets() + self.runtime_view().list_targets().await } /// Returns cloned target values for read-only runtime inspection. - pub async fn get_target_values(&self) -> Vec + Send + Sync>> { - let registry = self.registry.lock().await; - registry.list_target_values() + pub async fn get_target_values(&self) -> Vec> { + self.runtime_view().get_target_values().await } /// Returns per-target delivery metrics for Prometheus collection. pub async fn snapshot_target_metrics(&self) -> Vec { - let targets = self.get_target_values().await; - let mut snapshots = Vec::with_capacity(targets.len()); + self.pipeline().snapshot_target_metrics().await + } - for target in targets { - let delivery = target.delivery_snapshot(); - snapshots.push(AuditTargetMetricSnapshot { - failed_messages: delivery.failed_messages, - queue_length: delivery.queue_length, - target_id: target.id().to_string(), - total_messages: delivery.total_messages, - }); - } + pub async fn snapshot_target_health(&self) -> Vec { + self.pipeline().snapshot_target_health().await + } - snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id)); - snapshots + pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot { + let replay_workers = self.stream_cancellers.read().await; + let registry = self.registry.lock().await; + registry.runtime_manager().status_snapshot(&replay_workers) } /// Gets information about a specific target @@ -706,8 +385,7 @@ impl AuditSystem { /// # Returns /// * `Option` - Target ID if found pub async fn get_target(&self, target_id: &str) -> Option { - let registry = self.registry.lock().await; - registry.get_target(target_id).map(|target| target.id().to_string()) + self.runtime_view().get_target(target_id).await } /// Reloads configuration and updates targets @@ -722,32 +400,20 @@ impl AuditSystem { observability::record_config_reload(); - // Stop all existing stream tasks first - self.stop_all_streams().await; - // Store new configuration { let mut config_guard = self.config.write().await; *config_guard = Some(new_config.clone()); } - // Close all existing targets - let mut registry = self.registry.lock().await; - if let Err(e) = registry.close_all().await { - error!(error = %e, "Failed to close existing targets during reload"); - } + let final_state = match self.get_state().await { + AuditSystemState::Paused => AuditSystemState::Paused, + _ => AuditSystemState::Running, + }; - // Create new targets from updated configuration - match registry.create_audit_targets_from_config(&new_config).await { + match self.create_targets_from_config(&new_config).await { Ok(targets) => { - info!(target_count = targets.len(), "Reloaded audit targets successfully"); - - for target in targets { - if let Some(target_id) = self.init_and_register_target(target, &mut registry).await { - info!(target_id = %target_id, "Target initialized (reload)"); - } - } - + self.commit_runtime_targets(targets, final_state).await?; info!("Audit configuration reloaded successfully"); Ok(()) } @@ -779,3 +445,106 @@ impl AuditSystem { observability::reset_metrics().await; } } + +#[cfg(test)] +mod tests { + use super::{AuditSystem, AuditSystemState}; + use async_trait::async_trait; + use rustfs_targets::ReplayWorkerManager; + use rustfs_targets::arn::TargetID; + use rustfs_targets::store::{Key, Store}; + use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use rustfs_targets::{StoreError, Target, TargetError}; + use serde::{Serialize, de::DeserializeOwned}; + use std::collections::HashMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::mpsc; + + #[derive(Clone)] + struct TestTarget { + close_calls: Arc, + id: TargetID, + } + + impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + close_calls: Arc::new(AtomicUsize::new(0)), + id: TargetID::new(id.to_string(), name.to_string()), + } + } + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + #[tokio::test] + async fn reload_with_empty_config_stops_existing_runtime() { + let system = AuditSystem::new(); + let target = TestTarget::new("primary", "webhook"); + let close_calls = Arc::clone(&target.close_calls); + + { + let mut registry = system.registry.lock().await; + registry.add_target("primary:webhook".to_string(), Box::new(target)); + } + { + let mut state = system.state.write().await; + *state = AuditSystemState::Running; + } + { + let mut replay_workers = system.stream_cancellers.write().await; + let (cancel_tx, _cancel_rx) = mpsc::channel(1); + replay_workers.insert("primary:webhook".to_string(), cancel_tx); + assert_eq!(replay_workers.len(), 1); + } + + system + .reload_config(rustfs_ecstore::config::Config(HashMap::new())) + .await + .expect("reload with empty config should succeed"); + + assert_eq!(system.get_state().await, AuditSystemState::Stopped); + assert!(system.list_targets().await.is_empty()); + assert_eq!(system.runtime_status_snapshot().await, ReplayWorkerManager::new().snapshot(0)); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + assert_eq!(*system.config.read().await, Some(rustfs_ecstore::config::Config(HashMap::new()))); + } +} diff --git a/crates/audit/tests/pipeline_layer_test.rs b/crates/audit/tests/pipeline_layer_test.rs new file mode 100644 index 000000000..8e76b1081 --- /dev/null +++ b/crates/audit/tests/pipeline_layer_test.rs @@ -0,0 +1,170 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use rustfs_audit::{AuditPipeline, AuditRegistry, AuditRuntimeFacade, AuditRuntimeView}; +use rustfs_targets::arn::TargetID; +use rustfs_targets::store::{Key, Store}; +use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; +use rustfs_targets::{StoreError, Target, TargetError}; +use serde::{Serialize, de::DeserializeOwned}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokio::sync::{Mutex, RwLock}; + +#[derive(Clone)] +struct TestTarget { + close_calls: Arc, + id: TargetID, + init_calls: Arc, +} + +impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + close_calls: Arc::new(AtomicUsize::new(0)), + id: TargetID::new(id.to_string(), name.to_string()), + init_calls: Arc::new(AtomicUsize::new(0)), + } + } +} + +#[async_trait] +impl Target for TestTarget +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + self.init_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } +} + +#[tokio::test] +async fn audit_runtime_view_lists_empty_targets() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let runtime_view = AuditRuntimeView::new(registry); + + assert!(runtime_view.list_targets().await.is_empty()); + assert!(runtime_view.get_target_values().await.is_empty()); + assert!(runtime_view.get_target("missing").await.is_none()); +} + +#[tokio::test] +async fn audit_pipeline_reports_empty_runtime_snapshots() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let pipeline = AuditPipeline::new(registry); + + assert!(pipeline.snapshot_target_metrics().await.is_empty()); + assert!(pipeline.snapshot_target_health().await.is_empty()); +} + +#[tokio::test] +async fn audit_runtime_facade_stops_empty_replay_workers() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new())); + let facade = AuditRuntimeFacade::new(registry, replay_workers); + + facade.stop_replay_workers().await; +} + +#[tokio::test] +async fn audit_runtime_facade_activates_empty_target_list() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new())); + let facade = AuditRuntimeFacade::new(registry, replay_workers); + + let activation = facade.activate_targets_with_replay(Vec::new()).await; + assert!(activation.targets.is_empty()); + assert_eq!(activation.replay_workers.len(), 0); +} + +#[tokio::test] +async fn audit_runtime_view_upsert_and_remove_target() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let runtime_view = AuditRuntimeView::new(registry.clone()); + let target = TestTarget::new("primary", "webhook"); + let init_calls = Arc::clone(&target.init_calls); + let close_calls = Arc::clone(&target.close_calls); + + runtime_view + .upsert_target("primary:webhook".to_string(), Box::new(target)) + .await + .expect("upsert should succeed"); + + assert_eq!(runtime_view.list_targets().await, vec!["primary:webhook".to_string()]); + assert_eq!(init_calls.load(Ordering::SeqCst), 1); + + runtime_view + .remove_target("primary:webhook") + .await + .expect("remove should succeed"); + + assert!(runtime_view.list_targets().await.is_empty()); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn audit_runtime_facade_replace_targets_commits_runtime_state() { + let registry = Arc::new(Mutex::new(AuditRegistry::new())); + let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new())); + let facade = AuditRuntimeFacade::new(registry.clone(), replay_workers.clone()); + let target = TestTarget::new("primary", "webhook"); + let activation = rustfs_targets::RuntimeActivation { + replay_workers: rustfs_targets::ReplayWorkerManager::new(), + targets: vec![Arc::new(target) as rustfs_targets::SharedTarget], + }; + + facade + .replace_targets(activation) + .await + .expect("replace_targets should succeed"); + + let runtime_view = AuditRuntimeView::new(registry); + assert_eq!(runtime_view.list_targets().await, vec!["primary:webhook".to_string()]); + assert_eq!(replay_workers.read().await.len(), 0); +} diff --git a/crates/notify/AGENTS.md b/crates/notify/AGENTS.md new file mode 100644 index 000000000..f253eb024 --- /dev/null +++ b/crates/notify/AGENTS.md @@ -0,0 +1,51 @@ +# Notify Crate Instructions + +Applies to `crates/notify/`. + +`rustfs-notify` is the domain layer for bucket notification semantics. It +builds rules, event dispatch flow, and config/runtime orchestration on top of +shared plugin/runtime primitives from `rustfs-targets`. + +## Domain Boundaries + +- Keep notify-specific business logic here: + - bucket/rule evaluation + - event bridge and pipeline dispatch + - notify config reload orchestration +- Keep shared runtime/plugin mechanics in `rustfs-targets`: + - do not duplicate replay worker lifecycle logic + - do not reimplement plugin descriptor/registry/catalog semantics + - do not move install/control-plane state into this crate + +## Runtime Layering Rules + +- `runtime_facade.rs` is the mutation/orchestration boundary: + activation, replace, stop workers, shutdown. +- `runtime_view.rs` is read-only runtime observation: + active targets, metrics/health snapshots, runtime status snapshots. +- `config_manager.rs` should map config to runtime updates through facade/view + and `runtime_target_id_for_subsystem`; avoid bypassing these boundaries. +- `stream.rs` is a compatibility shim; new replay/runtime work should prefer + shared helpers in `rustfs-targets::runtime`. + +## Change Style + +- Preserve best-effort dispatch semantics and observability signals unless the + task explicitly requests behavior changes. +- Reuse existing notify constants and subsystem mappings from `rustfs_config`. +- Keep changes local and avoid cross-crate refactors from this crate unless + required by the task. + +## Testing + +- Keep unit tests close to changed modules. +- Add regression tests for: + - rules to runtime target resolution + - runtime facade replace/shutdown behavior + - runtime view health/status/metrics snapshots +- Suggested validation: + - `cargo test -p rustfs-notify` + - Focused: `cargo test -p rustfs-notify runtime_facade` + - Focused: `cargo test -p rustfs-notify runtime_view` + - Focused: `cargo test -p rustfs-notify config_manager` +- Full gate before commit: `make pre-commit` diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index 07ae434d5..131593804 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -46,6 +46,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] } tracing = { workspace = true } url = { workspace = true } wildmatch = { workspace = true, features = ["serde"] } +metrics = { workspace = true } # quick-xml dependencies for custom S3KeyFilter XML deserialization # Custom deserializer implemented for S3KeyFilter to handle both XML structures: diff --git a/crates/notify/src/bucket_config_manager.rs b/crates/notify/src/bucket_config_manager.rs new file mode 100644 index 000000000..8e041b52d --- /dev/null +++ b/crates/notify/src/bucket_config_manager.rs @@ -0,0 +1,125 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + BucketNotificationConfig, NotificationError, config_manager::notify_configuration_hint, + notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine, + rules::ParseConfigError, +}; +use rustfs_s3_common::EventName; +use std::sync::Arc; +use tracing::{debug, info, warn}; + +#[derive(Clone)] +pub struct NotifyBucketConfigManager { + notifier: Arc, + rule_engine: NotifyRuleEngine, + subscriber_view: Arc, +} + +impl NotifyBucketConfigManager { + pub fn new( + notifier: Arc, + rule_engine: NotifyRuleEngine, + subscriber_view: Arc, + ) -> Self { + Self { + notifier, + rule_engine, + subscriber_view, + } + } + + pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool { + if !self.subscriber_view.has_subscriber(bucket, event) { + return false; + } + self.rule_engine.has_subscriber(bucket, event).await + } + + pub async fn load_bucket_notification_config( + &self, + bucket: &str, + cfg: &BucketNotificationConfig, + ) -> Result<(), NotificationError> { + let arn_list = self.notifier.get_arn_list(&cfg.region).await; + if arn_list.is_empty() { + return Err(NotificationError::Configuration(notify_configuration_hint())); + } + info!("Available ARNs: {:?}", arn_list); + + if let Err(e) = cfg.validate(&cfg.region, &arn_list) { + debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e); + if !matches!(e, ParseConfigError::ArnNotFound(_)) { + return Err(NotificationError::BucketNotification(e.to_string())); + } + warn!( + bucket = %bucket, + region = %cfg.region, + error = %e, + "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules" + ); + } + + self.subscriber_view.apply_bucket_config(bucket, cfg); + self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await; + info!("Loaded notification config for bucket: {}", bucket); + Ok(()) + } + + pub async fn remove_bucket_notification_config(&self, bucket: &str) { + self.subscriber_view.clear_bucket(bucket); + self.rule_engine.clear_bucket_rules(bucket).await; + } +} + +#[cfg(test)] +mod tests { + use super::NotifyBucketConfigManager; + use crate::{ + BucketNotificationConfig, integration::NotificationMetrics, + notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine, + }; + use rustfs_s3_common::EventName; + use rustfs_targets::arn::TargetID; + use std::sync::Arc; + + fn build_manager() -> NotifyBucketConfigManager { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics, rule_engine.clone())); + let subscriber_view = Arc::new(NotificationSystemSubscriberView::new()); + NotifyBucketConfigManager::new(notifier, rule_engine, subscriber_view) + } + + #[tokio::test] + async fn bucket_config_manager_reports_no_subscriber_for_empty_state() { + let manager = build_manager(); + assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await); + } + + #[tokio::test] + async fn bucket_config_manager_clears_bucket_snapshot() { + let manager = build_manager(); + let target_id = TargetID::new("primary".to_string(), "webhook".to_string()); + let mut cfg = BucketNotificationConfig::new("us-east-1"); + cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id); + + manager.subscriber_view.apply_bucket_config("bucket", &cfg); + assert!(manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut)); + + manager.remove_bucket_notification_config("bucket").await; + assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut)); + } +} diff --git a/crates/notify/src/config_manager.rs b/crates/notify/src/config_manager.rs new file mode 100644 index 000000000..956b77ee7 --- /dev/null +++ b/crates/notify/src/config_manager.rs @@ -0,0 +1,338 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + Event, NotificationError, registry::TargetRegistry, rule_engine::NotifyRuleEngine, runtime_facade::NotifyRuntimeFacade, +}; +use rustfs_config::notify::{ + NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, +}; +use rustfs_ecstore::config::{Config, KVS}; +use rustfs_targets::{Target, arn::TargetID}; +use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{debug, info, warn}; + +pub(crate) fn notify_configuration_hint() -> String { + let webhook_enable_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENABLE); + let webhook_endpoint_primary = format!("{}_PRIMARY", rustfs_config::notify::ENV_NOTIFY_WEBHOOK_ENDPOINT); + format!( + "No notify targets configured. Check {}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {} is writable.", + rustfs_config::ENV_NOTIFY_ENABLE, + rustfs_config::EVENT_DEFAULT_DIR, + ) +} + +fn subsystem_target_type(target_type: &str) -> &str { + match target_type { + NOTIFY_AMQP_SUB_SYS => "amqp", + NOTIFY_WEBHOOK_SUB_SYS => "webhook", + NOTIFY_KAFKA_SUB_SYS => "kafka", + NOTIFY_MQTT_SUB_SYS => "mqtt", + NOTIFY_MYSQL_SUB_SYS => "mysql", + NOTIFY_NATS_SUB_SYS => "nats", + NOTIFY_POSTGRES_SUB_SYS => "postgres", + NOTIFY_PULSAR_SUB_SYS => "pulsar", + NOTIFY_REDIS_SUB_SYS => "redis", + _ => target_type, + } +} + +pub fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID { + TargetID { + id: target_name.to_lowercase(), + name: subsystem_target_type(target_type).to_string(), + } +} + +#[derive(Clone)] +pub struct NotifyConfigManager { + config: Arc>, + registry: Arc, + rule_engine: NotifyRuleEngine, + runtime_facade: NotifyRuntimeFacade, +} + +impl NotifyConfigManager { + pub fn new( + config: Arc>, + registry: Arc, + rule_engine: NotifyRuleEngine, + runtime_facade: NotifyRuntimeFacade, + ) -> Self { + Self { + config, + registry, + rule_engine, + runtime_facade, + } + } + + pub async fn init(&self) -> Result<(), NotificationError> { + info!("Initialize notification system..."); + + let config = { + let guard = self.config.read().await; + debug!( + subsystem_count = guard.0.len(), + "Initializing notification system with configuration summary" + ); + guard.clone() + }; + + let targets: Vec + Send + Sync>> = self.registry.create_targets_from_config(&config).await?; + + info!("{} notification targets were created", targets.len()); + if targets.is_empty() { + warn!("{}", notify_configuration_hint()); + } + + let activation = self.runtime_facade.activate_targets_with_replay(targets).await; + self.runtime_facade.replace_targets(activation).await?; + info!("Notification system initialized"); + Ok(()) + } + + pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> { + info!("Attempting to remove target: {}", target_id); + + let ttype = target_type.to_lowercase(); + let tname = target_id.id.to_lowercase(); + + self.update_config_and_reload(|config| { + let mut changed = false; + if let Some(targets_of_type) = config.0.get_mut(&ttype) { + if targets_of_type.remove(&tname).is_some() { + info!("Removed target {} from configuration", target_id); + changed = true; + } + if targets_of_type.is_empty() { + config.0.remove(&ttype); + } + } + if !changed { + warn!("Target {} not found in configuration", target_id); + } + changed + }) + .await + } + + pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> { + info!("Setting config for target {} of type {}", target_name, target_type); + let ttype = target_type.to_lowercase(); + let tname = target_name.to_lowercase(); + self.update_config_and_reload(|config| { + config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone()); + true + }) + .await + } + + pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> { + info!("Removing config for target {} of type {}", target_name, target_type); + + let ttype = target_type.to_lowercase(); + let tname = target_name.to_lowercase(); + let target_id = runtime_target_id_for_subsystem(&ttype, &tname); + + if self.rule_engine.is_target_bound_to_any_bucket(&target_id).await { + return Err(NotificationError::Configuration(format!( + "Target is still bound to bucket rules and deletion is prohibited: type={} name={}", + ttype, tname + ))); + } + + self.update_config_and_reload(|config| { + let mut changed = false; + if let Some(targets) = config.0.get_mut(&ttype) { + if targets.remove(&tname).is_some() { + changed = true; + } + if targets.is_empty() { + config.0.remove(&ttype); + } + } + if !changed { + info!("Target {} of type {} not found, no changes made.", target_name, target_type); + } + debug!( + subsystem_count = config.0.len(), + "Target config removal processed and configuration summary updated" + ); + changed + }) + .await + } + + pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> { + info!("Reload notification configuration starts"); + + self.update_config(new_config.clone()).await; + + let targets: Vec + Send + Sync>> = self + .registry + .create_targets_from_config(&new_config) + .await + .map_err(NotificationError::Target)?; + + info!("{} notification targets were created from the new configuration", targets.len()); + if targets.is_empty() { + warn!("{}", notify_configuration_hint()); + } + + let activation = self.runtime_facade.activate_targets_with_replay(targets).await; + self.runtime_facade.replace_targets(activation).await?; + info!("Configuration reloaded end"); + Ok(()) + } + + async fn update_config(&self, new_config: Config) { + let mut config = self.config.write().await; + *config = new_config; + } + + async fn update_config_and_reload(&self, mut modifier: F) -> Result<(), NotificationError> + where + F: FnMut(&mut Config) -> bool, + { + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { + return Err(NotificationError::StorageNotAvailable( + "Failed to save target configuration: server storage not initialized".to_string(), + )); + }; + + let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| NotificationError::ReadConfig(e.to_string()))?; + + if !modifier(&mut new_config) { + info!("Configuration not changed, skipping save and reload."); + return Ok(()); + } + + rustfs_ecstore::config::com::save_server_config(store, &new_config) + .await + .map_err(|e| NotificationError::SaveConfig(e.to_string()))?; + + info!("Configuration updated. Reloading system..."); + self.reload_config(new_config).await + } +} + +#[cfg(test)] +mod tests { + use super::{NotifyConfigManager, runtime_target_id_for_subsystem}; + use crate::{ + integration::NotificationMetrics, notifier::EventNotifier, registry::TargetRegistry, rule_engine::NotifyRuleEngine, + runtime_facade::NotifyRuntimeFacade, + }; + use rustfs_config::notify::{ + NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, + NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, + }; + use rustfs_ecstore::config::Config; + use rustfs_targets::ReplayWorkerManager; + use std::sync::Arc; + use tokio::sync::{RwLock, Semaphore}; + + fn build_manager() -> NotifyConfigManager { + let config = Arc::new(RwLock::new(Config::default())); + let registry = Arc::new(TargetRegistry::new()); + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let target_list = notifier.target_list(); + let runtime_facade = NotifyRuntimeFacade::new( + target_list, + Arc::new(RwLock::new(ReplayWorkerManager::new())), + Arc::new(Semaphore::new(4)), + metrics, + ); + + NotifyConfigManager::new(config, registry, rule_engine, runtime_facade) + } + + #[tokio::test] + async fn config_manager_init_accepts_empty_target_set() { + let manager = build_manager(); + manager.init().await.expect("init should succeed for empty targets"); + } + + #[tokio::test] + async fn config_manager_reload_accepts_empty_target_set() { + let manager = build_manager(); + manager + .reload_config(Config::default()) + .await + .expect("reload_config should succeed for empty targets"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "webhook"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "amqp"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics"); + assert_eq!(target_id.id, "analytics"); + assert_eq!(target_id.name, "mqtt"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus"); + assert_eq!(target_id.id, "eventbus"); + assert_eq!(target_id.name, "kafka"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus"); + assert_eq!(target_id.id, "bus"); + assert_eq!(target_id.name, "nats"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger"); + assert_eq!(target_id.id, "ledger"); + assert_eq!(target_id.name, "pulsar"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary"); + assert_eq!(target_id.id, "primary"); + assert_eq!(target_id.name, "redis"); + } + + #[test] + fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() { + let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail"); + assert_eq!(target_id.id, "audittrail"); + assert_eq!(target_id.name, "postgres"); + } +} diff --git a/crates/notify/src/event_bridge.rs b/crates/notify/src/event_bridge.rs new file mode 100644 index 000000000..125f1af23 --- /dev/null +++ b/crates/notify/src/event_bridge.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub use crate::pipeline::{LiveEventHistory, NotifyEventBridge}; diff --git a/crates/notify/src/factory.rs b/crates/notify/src/factory.rs index a948cb344..08998b334 100644 --- a/crates/notify/src/factory.rs +++ b/crates/notify/src/factory.rs @@ -13,146 +13,11 @@ // limitations under the License. use crate::Event; -use rustfs_config::EVENT_DEFAULT_DIR; -use rustfs_config::notify::{ - NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, - NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, - NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, - NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, -}; -use rustfs_targets::config::{ - build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, - build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, - validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, - validate_webhook_config, -}; -use rustfs_targets::target::{ChannelTargetType, TargetType}; -use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target}; +use rustfs_targets::catalog::builtin::builtin_notify_target_descriptors; +use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor}; pub fn builtin_target_descriptors() -> Vec> { - vec![ - BuiltinTargetDescriptor::new( - NOTIFY_WEBHOOK_SUB_SYS, - TargetRequestValidator::Webhook, - TargetPluginDescriptor::new( - ChannelTargetType::Webhook.as_str(), - NOTIFY_WEBHOOK_KEYS, - |config| validate_webhook_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_AMQP_SUB_SYS, - TargetRequestValidator::Amqp(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::Amqp.as_str(), - NOTIFY_AMQP_KEYS, - |config| validate_amqp_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_KAFKA_SUB_SYS, - TargetRequestValidator::Kafka(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::Kafka.as_str(), - NOTIFY_KAFKA_KEYS, - |config| validate_kafka_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_MQTT_SUB_SYS, - TargetRequestValidator::Mqtt, - TargetPluginDescriptor::new( - ChannelTargetType::Mqtt.as_str(), - NOTIFY_MQTT_KEYS, - validate_mqtt_config, - |id, config| { - let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_MYSQL_SUB_SYS, - TargetRequestValidator::MySql(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::MySql.as_str(), - NOTIFY_MYSQL_KEYS, - |config| validate_mysql_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_NATS_SUB_SYS, - TargetRequestValidator::Nats(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::Nats.as_str(), - NOTIFY_NATS_KEYS, - |config| validate_nats_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_POSTGRES_SUB_SYS, - TargetRequestValidator::Postgres(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::Postgres.as_str(), - NOTIFY_POSTGRES_KEYS, - |config| validate_postgres_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_REDIS_SUB_SYS, - TargetRequestValidator::Redis { - default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL, - target_type: TargetType::NotifyEvent, - }, - TargetPluginDescriptor::new( - ChannelTargetType::Redis.as_str(), - NOTIFY_REDIS_KEYS, - |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL), - |id, config| { - let args = - build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?)) - }, - ), - ), - BuiltinTargetDescriptor::new( - NOTIFY_PULSAR_SUB_SYS, - TargetRequestValidator::Pulsar(TargetType::NotifyEvent), - TargetPluginDescriptor::new( - ChannelTargetType::Pulsar.as_str(), - NOTIFY_PULSAR_KEYS, - |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR), - |id, config| { - let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; - Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?)) - }, - ), - ), - ] + builtin_notify_target_descriptors::() } pub fn builtin_target_plugins() -> Vec> { diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 04b8fe557..dd8f5dc16 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -13,66 +13,29 @@ // limitations under the License. use crate::notification_system_subscriber::NotificationSystemSubscriberView; -use crate::notifier::TargetList; +use crate::notifier::{EventNotifier, TargetList}; +use crate::services::NotifyServices; use crate::{ - Event, - error::NotificationError, - notifier::EventNotifier, - registry::TargetRegistry, - rules::{BucketNotificationConfig, ParseConfigError}, - stream, + Event, error::NotificationError, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine, + rules::BucketNotificationConfig, }; use hashbrown::HashMap; -use rustfs_config::notify::{ - DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE, - ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, - NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, -}; -use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR}; +use metrics::{counter, gauge}; +use rustfs_config::notify::{DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY}; use rustfs_ecstore::config::{Config, KVS}; use rustfs_s3_common::EventName; use rustfs_targets::arn::TargetID; -use rustfs_targets::store::{Key, Store}; -use rustfs_targets::target::QueuedPayload; -use rustfs_targets::{StoreError, Target}; -use std::collections::VecDeque; +use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; -use tokio::sync::{RwLock, Semaphore, broadcast, mpsc}; -use tracing::{debug, info, warn}; +use tokio::sync::{RwLock, Semaphore, broadcast}; +use tracing::info; -const MAX_RECENT_LIVE_EVENTS: usize = 1024; - -fn notify_configuration_hint() -> String { - let webhook_enable_primary = format!("{ENV_NOTIFY_WEBHOOK_ENABLE}_PRIMARY"); - let webhook_endpoint_primary = format!("{ENV_NOTIFY_WEBHOOK_ENDPOINT}_PRIMARY"); - format!( - "No notify targets configured. Check {ENV_NOTIFY_ENABLE}=true and instance-scoped target env vars (for example {webhook_enable_primary} + {webhook_endpoint_primary} for arn:rustfs:sqs::primary:webhook). If using default queue_dir, ensure {EVENT_DEFAULT_DIR} is writable." - ) -} - -fn subsystem_target_type(target_type: &str) -> &str { - match target_type { - NOTIFY_AMQP_SUB_SYS => "amqp", - NOTIFY_WEBHOOK_SUB_SYS => "webhook", - NOTIFY_KAFKA_SUB_SYS => "kafka", - NOTIFY_MQTT_SUB_SYS => "mqtt", - NOTIFY_MYSQL_SUB_SYS => "mysql", - NOTIFY_NATS_SUB_SYS => "nats", - NOTIFY_POSTGRES_SUB_SYS => "postgres", - NOTIFY_PULSAR_SUB_SYS => "pulsar", - NOTIFY_REDIS_SUB_SYS => "redis", - _ => target_type, - } -} - -fn runtime_target_id_for_subsystem(target_type: &str, target_name: &str) -> TargetID { - TargetID { - id: target_name.to_lowercase(), - name: subsystem_target_type(target_type).to_string(), - } -} +const METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS: &str = "rustfs_notification_current_send_in_progress"; +const METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL: &str = "rustfs_notification_events_errors_total"; +const METRIC_NOTIFICATION_EVENTS_SENT_TOTAL: &str = "rustfs_notification_events_sent_total"; +const METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL: &str = "rustfs_notification_events_skipped_total"; #[derive(Clone)] pub struct LiveEventBatch { @@ -81,46 +44,6 @@ pub struct LiveEventBatch { pub truncated: bool, } -#[derive(Default)] -struct LiveEventHistory { - next_sequence: u64, - events: VecDeque<(u64, Arc)>, -} - -impl LiveEventHistory { - fn record(&mut self, event: Arc) { - self.next_sequence = self.next_sequence.saturating_add(1); - self.events.push_back((self.next_sequence, event)); - while self.events.len() > MAX_RECENT_LIVE_EVENTS { - self.events.pop_front(); - } - } - - fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { - let mut events = Vec::new(); - let mut next_sequence = after_sequence; - let mut truncated = false; - - for (sequence, event) in self.events.iter() { - if *sequence <= after_sequence { - continue; - } - if events.len() >= limit { - truncated = true; - break; - } - next_sequence = *sequence; - events.push(event.clone()); - } - - LiveEventBatch { - events, - next_sequence, - truncated, - } - } -} - /// Notify the system of monitoring indicators pub struct NotificationMetrics { /// The number of events currently being processed @@ -231,18 +154,7 @@ pub struct NotificationSystem { pub registry: Arc, /// The current configuration pub config: Arc>, - /// Cancel sender for managing stream processing tasks - stream_cancellers: Arc>>>, - /// Concurrent control signal quantity - concurrency_limiter: Arc, - /// Monitoring indicators - metrics: Arc, - /// Subscriber view - subscriber_view: NotificationSystemSubscriberView, - /// Live event fan-out for in-process streaming consumers. - live_event_sender: broadcast::Sender>, - /// Recent live event history for peer fan-in consumers. - live_event_history: Arc>, + services: NotifyServices, } impl NotificationSystem { @@ -252,107 +164,40 @@ impl NotificationSystem { rustfs_utils::get_env_usize(ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY); let (live_event_sender, _) = broadcast::channel(1024); let metrics = Arc::new(NotificationMetrics::new()); - NotificationSystem { - subscriber_view: NotificationSystemSubscriberView::new(), - notifier: Arc::new(EventNotifier::new(metrics.clone())), - registry: Arc::new(TargetRegistry::new()), - config: Arc::new(RwLock::new(config)), - stream_cancellers: Arc::new(RwLock::new(HashMap::new())), - concurrency_limiter: Arc::new(Semaphore::new(concurrency_limiter)), // Limit the maximum number of concurrent processing events to 20 + let subscriber_view = Arc::new(NotificationSystemSubscriberView::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let target_list = notifier.target_list(); + let registry = Arc::new(TargetRegistry::new()); + let config = Arc::new(RwLock::new(config)); + let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new())); + let concurrency_limiter = Arc::new(Semaphore::new(concurrency_limiter)); // Limit the maximum number of concurrent processing events to 20 + let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default())); + let services = NotifyServices::new( + notifier.clone(), + rule_engine, + target_list, + registry.clone(), + config.clone(), + stream_cancellers, + concurrency_limiter, metrics, + subscriber_view, live_event_sender, - live_event_history: Arc::new(RwLock::new(LiveEventHistory::default())), + live_event_history, + ); + + NotificationSystem { + notifier, + registry, + config, + services, } } - /// Initializes targets and starts event streams for those with stores. - /// Returns a map of (target_id -> cancel_sender) for streams that were started. - async fn init_targets_and_start_streams( - &self, - targets: &[Box + Send + Sync>], - ) -> HashMap> { - let mut cancellers = HashMap::new(); - for target in targets { - let target_id = target.id(); - info!("Initializing target: {}", target_id); - - let has_store = target.store().is_some(); - - if let Err(e) = target.init().await { - warn!("Target {} Initialization failed: {}", target_id, e); - // For targets without a store, init failure is fatal — skip. - // For store-backed targets, still start the stream so queued events - // can be drained when connectivity recovers (send_from_store retries). - if !has_store { - continue; - } - warn!( - "Target {} has a store, starting stream despite init failure — \ - connectivity will be retried by send_from_store", - target_id - ); - } else { - debug!("Target {} initialized successfully, enabled: {}", target_id, target.is_enabled()); - } - - if !target.is_enabled() { - info!("Target {} is not enabled, event stream processing is skipped", target_id); - continue; - } - - if let Some(store) = target.store() { - info!("Start event stream processing for target {}", target_id); - - let store_clone = store.boxed_clone(); - let target_arc = Arc::from(target.clone_dyn()); - - let cancel_tx = self.enhanced_start_event_stream( - store_clone, - target_arc, - self.metrics.clone(), - self.concurrency_limiter.clone(), - ); - - let target_id_clone = target_id.clone(); - cancellers.insert(target_id, cancel_tx); - info!("Event stream processing for target {} is started successfully", target_id_clone); - } else { - info!("Target {} No storage is configured, event stream processing is skipped", target_id); - } - } - cancellers - } - /// Initializes the notification system pub async fn init(&self) -> Result<(), NotificationError> { - info!("Initialize notification system..."); - - let config = { - let guard = self.config.read().await; - debug!( - subsystem_count = guard.0.len(), - "Initializing notification system with configuration summary" - ); - guard.clone() - }; - - let targets: Vec + Send + Sync>> = self.registry.create_targets_from_config(&config).await?; - - info!("{} notification targets were created", targets.len()); - if targets.is_empty() { - warn!("{}", notify_configuration_hint()); - } - - // Initialize targets and start event streams - let cancellers = self.init_targets_and_start_streams(&targets).await; - - // Update canceller collection - *self.stream_cancellers.write().await = cancellers; - - // Initialize the bucket target - self.notifier.init_bucket_targets(targets).await?; - info!("Notification system initialized"); - Ok(()) + self.services.config_manager.init().await } /// Gets a list of Targets for all currently active (initialized). @@ -360,7 +205,7 @@ impl NotificationSystem { /// # Return /// A Vec containing all active Targets `TargetID`. pub async fn get_active_targets(&self) -> Vec { - self.notifier.target_list().read().await.keys() + self.services.runtime_view.get_active_targets().await } /// Gets the complete Target list, including both active and inactive Targets. @@ -368,67 +213,34 @@ impl NotificationSystem { /// # Return /// An `Arc>` containing all Targets. pub async fn get_all_targets(&self) -> Arc> { - self.notifier.target_list() + self.services.runtime_view.get_all_targets() } /// Gets all Target values, including both active and inactive Targets. /// /// # Return /// A Vec containing all Targets. - pub async fn get_target_values(&self) -> Vec + Send + Sync>> { - self.notifier.target_list().read().await.values() + pub async fn get_target_values(&self) -> Vec> { + self.services.runtime_view.get_target_values().await } /// Checks if there are active subscribers for the given bucket and event name. pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool { - if !self.subscriber_view.has_subscriber(bucket, event) { - return false; - } - self.notifier.has_subscriber(bucket, event).await + self.services.bucket_config_manager.has_subscriber(bucket, event).await } /// Returns true when at least one in-process consumer is subscribed to live events. pub fn has_live_listeners(&self) -> bool { - self.live_event_sender.receiver_count() > 0 + self.services.pipeline.has_live_listeners() } /// Subscribes to the in-process live event stream. pub fn subscribe_live_events(&self) -> broadcast::Receiver> { - self.live_event_sender.subscribe() + self.services.pipeline.subscribe_live_events() } pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { - let history = self.live_event_history.read().await; - history.snapshot_since(after_sequence, limit.max(1)) - } - - async fn update_config_and_reload(&self, mut modifier: F) -> Result<(), NotificationError> - where - F: FnMut(&mut Config) -> bool, // The closure returns a boolean value indicating whether the configuration has been changed - { - let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { - return Err(NotificationError::StorageNotAvailable( - "Failed to save target configuration: server storage not initialized".to_string(), - )); - }; - - let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) - .await - .map_err(|e| NotificationError::ReadConfig(e.to_string()))?; - - if !modifier(&mut new_config) { - // If the closure indication has not changed, return in advance - info!("Configuration not changed, skipping save and reload."); - return Ok(()); - } - - // Save the modified configuration to storage - rustfs_ecstore::config::com::save_server_config(store, &new_config) - .await - .map_err(|e| NotificationError::SaveConfig(e.to_string()))?; - - info!("Configuration updated. Reloading system..."); - self.reload_config(new_config).await + self.services.pipeline.recent_live_events_since(after_sequence, limit).await } /// Accurately remove a Target and its related resources through TargetID. @@ -444,28 +256,7 @@ impl NotificationSystem { /// # return /// If successful, return `Ok(())`. pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> { - info!("Attempting to remove target: {}", target_id); - - let ttype = target_type.to_lowercase(); - let tname = target_id.id.to_lowercase(); - - self.update_config_and_reload(|config| { - let mut changed = false; - if let Some(targets_of_type) = config.0.get_mut(&ttype) { - if targets_of_type.remove(&tname).is_some() { - info!("Removed target {} from configuration", target_id); - changed = true; - } - if targets_of_type.is_empty() { - config.0.remove(&ttype); - } - } - if !changed { - warn!("Target {} not found in configuration", target_id); - } - changed - }) - .await + self.services.config_manager.remove_target(target_id, target_type).await } /// Set or update a Target configuration. @@ -481,14 +272,10 @@ impl NotificationSystem { /// If the target configuration is successfully set, it returns Ok(()). /// If the target configuration is invalid, it returns Err(NotificationError::Configuration). pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> { - info!("Setting config for target {} of type {}", target_name, target_type); - let ttype = target_type.to_lowercase(); - let tname = target_name.to_lowercase(); - self.update_config_and_reload(|config| { - config.0.entry(ttype.clone()).or_default().insert(tname.clone(), kvs.clone()); - true // The configuration is always modified - }) - .await + self.services + .config_manager + .set_target_config(target_type, target_name, kvs) + .await } /// Removes all notification configurations for a bucket. @@ -498,8 +285,10 @@ impl NotificationSystem { /// * `bucket` - The name of the bucket whose notification configuration is to be removed. /// pub async fn remove_bucket_notification_config(&self, bucket: &str) { - self.subscriber_view.clear_bucket(bucket); - self.notifier.remove_rules_map(bucket).await; + self.services + .bucket_config_manager + .remove_bucket_notification_config(bucket) + .await; } /// Removes a Target configuration. @@ -515,108 +304,15 @@ impl NotificationSystem { /// If the target configuration is successfully removed, it returns Ok(()). /// If the target configuration does not exist, it returns Ok(()) without making any changes. pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> { - info!("Removing config for target {} of type {}", target_name, target_type); - - let ttype = target_type.to_lowercase(); - let tname = target_name.to_lowercase(); - let target_id = runtime_target_id_for_subsystem(&ttype, &tname); - - // Deletion is prohibited if bucket rules refer to it - if self.notifier.is_target_bound_to_any_bucket(&target_id).await { - return Err(NotificationError::Configuration(format!( - "Target is still bound to bucket rules and deletion is prohibited: type={} name={}", - ttype, tname - ))); - } - - let config_result = self - .update_config_and_reload(|config| { - let mut changed = false; - if let Some(targets) = config.0.get_mut(&ttype) { - if targets.remove(&tname).is_some() { - changed = true; - } - if targets.is_empty() { - config.0.remove(&ttype); - } - } - if !changed { - info!("Target {} of type {} not found, no changes made.", target_name, target_type); - } - debug!( - subsystem_count = config.0.len(), - "Target config removal processed and configuration summary updated" - ); - changed - }) - .await; - - if config_result.is_ok() { - // Remove from target list - let target_list = self.notifier.target_list(); - let mut target_list_guard = target_list.write().await; - let _ = target_list_guard.remove_target_only(&target_id).await; - } - - config_result - } - - /// Enhanced event stream startup function, including monitoring and concurrency control - fn enhanced_start_event_stream( - &self, - store: Box + Send>, - target: Arc + Send + Sync>, - metrics: Arc, - semaphore: Arc, - ) -> mpsc::Sender<()> { - stream::start_event_stream_with_batching(store, target, metrics, semaphore) - } - - /// Update configuration - async fn update_config(&self, new_config: Config) { - let mut config = self.config.write().await; - *config = new_config; + self.services + .config_manager + .remove_target_config(target_type, target_name) + .await } /// Reloads the configuration pub async fn reload_config(&self, new_config: Config) -> Result<(), NotificationError> { - info!("Reload notification configuration starts"); - - // Stop all existing streaming services - let mut cancellers = self.stream_cancellers.write().await; - for (target_id, cancel_tx) in cancellers.drain() { - info!("Stop event stream processing for target {}", target_id); - let _ = cancel_tx.send(()).await; - } - - // Clear the target_list and ensure that reload is a replacement reconstruction - self.notifier.remove_all_bucket_targets().await; - - // Update the config - self.update_config(new_config.clone()).await; - - // Create new targets from configuration - let targets: Vec + Send + Sync>> = self - .registry - .create_targets_from_config(&new_config) - .await - .map_err(NotificationError::Target)?; - - info!("{} notification targets were created from the new configuration", targets.len()); - if targets.is_empty() { - warn!("{}", notify_configuration_hint()); - } - - // Initialize targets and start event streams using shared helper - let new_cancellers = self.init_targets_and_start_streams(&targets).await; - - // Update canceler collection - *cancellers = new_cancellers; - - // Initialize the bucket target - self.notifier.init_bucket_targets(targets).await?; - info!("Configuration reloaded end"); - Ok(()) + self.services.config_manager.reload_config(new_config).await } /// Loads the bucket notification configuration @@ -625,93 +321,41 @@ impl NotificationSystem { bucket: &str, cfg: &BucketNotificationConfig, ) -> Result<(), NotificationError> { - let arn_list = self.notifier.get_arn_list(&cfg.region).await; - if arn_list.is_empty() { - return Err(NotificationError::Configuration(notify_configuration_hint())); - } - info!("Available ARNs: {:?}", arn_list); - // Validate the configuration against the available ARNs - if let Err(e) = cfg.validate(&cfg.region, &arn_list) { - debug!("Bucket notification config validation region:{} failed: {}", &cfg.region, e); - if !matches!(e, ParseConfigError::ArnNotFound(_)) { - return Err(NotificationError::BucketNotification(e.to_string())); - } - warn!( - bucket = %bucket, - region = %cfg.region, - error = %e, - "Bucket notification config references missing target ARN; keeping compatibility and loading remaining rules" - ); - } - - self.subscriber_view.apply_bucket_config(bucket, cfg); - let rules_map = cfg.get_rules_map(); - self.notifier.add_rules_map(bucket, rules_map.clone()).await; - info!("Loaded notification config for bucket: {}", bucket); - Ok(()) + self.services + .bucket_config_manager + .load_bucket_notification_config(bucket, cfg) + .await } /// Sends an event pub async fn send_event(&self, event: Arc) { - self.live_event_history.write().await.record(event.clone()); - let _ = self.live_event_sender.send(event.clone()); - self.notifier.send(event).await; + self.services.pipeline.send_event(event).await; } /// Obtain system status information pub fn get_status(&self) -> HashMap { - let mut status = HashMap::new(); - - status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string()); - status.insert("processing_events".to_string(), self.metrics.processing_count().to_string()); - status.insert("processed_events".to_string(), self.metrics.processed_count().to_string()); - status.insert("failed_events".to_string(), self.metrics.failed_count().to_string()); - status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string()); - - status + self.services.status_view.get_status() } pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot { - self.metrics.snapshot() + self.services.status_view.snapshot_metrics() } pub async fn snapshot_target_metrics(&self) -> Vec { - let targets = self.notifier.target_list().read().await.values(); - let mut snapshots = Vec::with_capacity(targets.len()); + self.services.runtime_view.snapshot_target_metrics().await + } - for target in targets { - let delivery = target.delivery_snapshot(); - let target_id = target.id(); - snapshots.push(NotificationTargetMetricSnapshot { - failed_messages: delivery.failed_messages, - queue_length: delivery.queue_length, - target_id: target_id.to_string(), - target_type: target_id.name, - total_messages: delivery.total_messages, - }); - } + pub async fn snapshot_target_health(&self) -> Vec { + self.services.runtime_view.snapshot_target_health().await + } - snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id)); - snapshots + pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot { + self.services.runtime_view.runtime_status_snapshot().await } // Add a method to shut down the system pub async fn shutdown(&self) { - info!("Turn off the notification system"); - - // Get the number of active targets - let active_targets = self.stream_cancellers.read().await.len(); - info!("Stops {} active event stream processing tasks", active_targets); - - let mut cancellers = self.stream_cancellers.write().await; - for (target_id, cancel_tx) in cancellers.drain() { - info!("Stop event stream processing for target {}", target_id); - let _ = cancel_tx.send(()).await; - } - // Wait for a short while to make sure the task has a chance to complete - tokio::time::sleep(Duration::from_millis(500)).await; - - info!("Notify the system to be shut down completed"); + self.services.runtime_facade.shutdown().await; } } @@ -719,6 +363,22 @@ impl Drop for NotificationSystem { fn drop(&mut self) { // Asynchronous operation cannot be used here, but logs can be recorded. info!("Notify the system instance to be destroyed"); + + let snapshot = self.snapshot_metrics(); + for (name, value, is_gauge) in [ + (METRIC_NOTIFICATION_CURRENT_SEND_IN_PROGRESS, snapshot.current_send_in_progress, true), + (METRIC_NOTIFICATION_EVENTS_ERRORS_TOTAL, snapshot.events_errors_total, false), + (METRIC_NOTIFICATION_EVENTS_SENT_TOTAL, snapshot.events_sent_total, false), + (METRIC_NOTIFICATION_EVENTS_SKIPPED_TOTAL, snapshot.events_skipped_total, false), + ] { + if is_gauge { + gauge!(name).set(value as f64); + } else { + counter!(name).absolute(value); + } + info!("shutdown metric {}={}", name, value); + } + let status = self.get_status(); for (key, value) in status { info!("key:{}, value:{}", key, value); @@ -772,58 +432,22 @@ mod tests { assert_eq!(batch.events[0].s3.object.key, "one"); } - #[test] - fn runtime_target_id_for_subsystem_maps_notify_webhook_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_WEBHOOK_SUB_SYS, "Primary"); - assert_eq!(target_id.id, "primary"); - assert_eq!(target_id.name, "webhook"); - } + #[tokio::test] + async fn notification_system_exposes_live_event_pipeline() { + let system = NotificationSystem::new(Config::default()); + assert!(!system.has_live_listeners()); - #[test] - fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary"); - assert_eq!(target_id.id, "primary"); - assert_eq!(target_id.name, "amqp"); - } + let _rx = system.subscribe_live_events(); + assert!(system.has_live_listeners()); - #[test] - fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics"); - assert_eq!(target_id.id, "analytics"); - assert_eq!(target_id.name, "mqtt"); - } + system + .send_event(Arc::new(Event::new_test_event("bucket", "object", EventName::ObjectCreatedPut))) + .await; - #[test] - fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus"); - assert_eq!(target_id.id, "eventbus"); - assert_eq!(target_id.name, "kafka"); - } - - #[test] - fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus"); - assert_eq!(target_id.id, "bus"); - assert_eq!(target_id.name, "nats"); - } - - #[test] - fn runtime_target_id_for_subsystem_maps_notify_pulsar_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_PULSAR_SUB_SYS, "Ledger"); - assert_eq!(target_id.id, "ledger"); - assert_eq!(target_id.name, "pulsar"); - } - - #[test] - fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary"); - assert_eq!(target_id.id, "primary"); - assert_eq!(target_id.name, "redis"); - } - #[test] - fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() { - let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail"); - assert_eq!(target_id.id, "audittrail"); - assert_eq!(target_id.name, "postgres"); + let batch = system.recent_live_events_since(0, 16).await; + assert_eq!(batch.events.len(), 1); + assert_eq!(batch.events[0].s3.object.key, "object"); + assert_eq!(batch.next_sequence, 1); + assert!(!batch.truncated); } } diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index 298f7feec..b1f85587c 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -18,22 +18,39 @@ //! It supports sending events to various targets //! (like Webhook and MQTT) and includes features like event persistence and retry on failure. +mod bucket_config_manager; +mod config_manager; mod error; mod event; +mod event_bridge; pub mod factory; mod global; pub mod integration; mod notification_system_subscriber; pub mod notifier; +mod pipeline; pub mod registry; +mod rule_engine; pub mod rules; -pub mod stream; +mod runtime_facade; +mod runtime_view; +mod services; +mod status_view; +pub use bucket_config_manager::NotifyBucketConfigManager; +pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem}; pub use error::{LifecycleError, NotificationError}; pub use event::{Event, EventArgs, EventArgsBuilder}; +pub use event_bridge::{LiveEventHistory, NotifyEventBridge}; pub use global::{ initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system, notification_target_metrics, notifier_global, }; pub use integration::{NotificationMetricSnapshot, NotificationSystem, NotificationTargetMetricSnapshot}; +pub use pipeline::NotifyPipeline; +pub use rule_engine::NotifyRuleEngine; pub use rules::BucketNotificationConfig; +pub use runtime_facade::NotifyRuntimeFacade; +pub use runtime_view::NotifyRuntimeView; +pub use services::NotifyServices; +pub use status_view::NotifyStatusView; diff --git a/crates/notify/src/notifier.rs b/crates/notify/src/notifier.rs index d904f9437..38daad89f 100644 --- a/crates/notify/src/notifier.rs +++ b/crates/notify/src/notifier.rs @@ -12,52 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{ - error::NotificationError, - event::Event, - integration::NotificationMetrics, - rules::{RulesMap, TargetIdSet}, -}; -use hashbrown::HashMap; -use percent_encoding::percent_decode_str; +use crate::{error::NotificationError, event::Event, integration::NotificationMetrics, rule_engine::NotifyRuleEngine}; use rustfs_config::notify::{DEFAULT_NOTIFY_SEND_CONCURRENCY, ENV_NOTIFY_SEND_CONCURRENCY}; -use rustfs_s3_common::EventName; -use rustfs_targets::Target; use rustfs_targets::arn::TargetID; use rustfs_targets::target::EntityTarget; -use starshard::AsyncShardedHashMap; +use rustfs_targets::{SharedTarget, Target, TargetRuntimeManager}; use std::sync::Arc; use tokio::sync::{RwLock, Semaphore}; use tracing::{debug, error, info, instrument, warn}; -fn decoded_object_key_for_matching(object_key: &str) -> Option { - if !object_key.contains('%') { - return None; - } - - let decoded = percent_decode_str(object_key).decode_utf8().ok()?; - (decoded != object_key).then(|| decoded.into_owned()) -} - -fn match_event_targets(rules: &RulesMap, event_name: EventName, object_key: &str) -> TargetIdSet { - let mut target_ids = rules.match_rules(event_name, object_key); - if let Some(decoded_key) = decoded_object_key_for_matching(object_key) { - target_ids.extend(rules.match_rules(event_name, &decoded_key)); - } - target_ids -} +pub type SharedNotifyTargetList = Arc>; /// Manages event notification to targets based on rules pub struct EventNotifier { metrics: Arc, - target_list: Arc>, - bucket_rules_map: Arc>, + rule_engine: NotifyRuleEngine, + target_list: SharedNotifyTargetList, send_limiter: Arc, } impl Default for EventNotifier { fn default() -> Self { - Self::new(Arc::new(NotificationMetrics::new())) + Self::new(Arc::new(NotificationMetrics::new()), NotifyRuleEngine::new()) } } @@ -66,56 +42,25 @@ impl EventNotifier { /// /// # Returns /// Returns a new instance of EventNotifier. - pub fn new(metrics: Arc) -> Self { + pub fn new(metrics: Arc, rule_engine: NotifyRuleEngine) -> Self { let max_inflight = rustfs_utils::get_env_usize(ENV_NOTIFY_SEND_CONCURRENCY, DEFAULT_NOTIFY_SEND_CONCURRENCY); EventNotifier { metrics, + rule_engine, target_list: Arc::new(RwLock::new(TargetList::new())), - bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)), send_limiter: Arc::new(Semaphore::new(max_inflight)), } } - /// Checks whether a TargetID is still referenced by any bucket's rules. - /// - /// # Arguments - /// * `target_id` - The TargetID to check. - /// - /// # Returns - /// Returns `true` if the TargetID is bound to any bucket, otherwise `false`. - pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool { - // `AsyncShardedHashMap::iter()`: Traverse (bucket_name, rules_map) - let items = self.bucket_rules_map.iter().await; - for (_bucket, rules_map) in items { - if rules_map.contains_target_id(target_id) { - return true; - } - } - false - } - /// Returns a reference to the target list /// This method provides access to the target list for external use. /// /// # Returns /// Returns an `Arc>` representing the target list. - pub fn target_list(&self) -> Arc> { + pub fn target_list(&self) -> SharedNotifyTargetList { Arc::clone(&self.target_list) } - /// Removes all notification rules for a bucket - /// - /// # Arguments - /// * `bucket` - The name of the bucket for which to remove rules - /// - /// This method removes all rules associated with the specified bucket name. - /// It will log a message indicating the removal of rules. - pub async fn remove_rules_map(&self, bucket: &str) { - if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() { - info!("Removed all notification rules for bucket: {}", bucket); - } - } - /// Returns a list of ARNs for the registered targets /// /// # Arguments @@ -132,40 +77,6 @@ impl EventNotifier { .collect() } - /// Adds a rules map for a bucket - /// - /// # Arguments - /// * `bucket` - The name of the bucket for which to add the rules map - /// * `rules_map` - The rules map to add for the bucket - pub async fn add_rules_map(&self, bucket: &str, rules_map: RulesMap) { - if rules_map.is_empty() { - self.bucket_rules_map.remove(&bucket.to_string()).await; - } else { - self.bucket_rules_map.insert(bucket.to_string(), rules_map).await; - } - info!("Added rules for bucket: {}", bucket); - } - - /// Gets the rules map for a specific bucket. - /// - /// # Arguments - /// * `bucket` - The name of the bucket for which to get the rules map - /// - /// # Returns - /// Returns `Some(RulesMap)` if rules exist for the bucket, otherwise returns `None`. - pub async fn get_rules_map(&self, bucket: &str) -> Option { - self.bucket_rules_map.get(&bucket.to_string()).await - } - - /// Removes notification rules for a bucket - /// - /// # Arguments - /// * `bucket` - The name of the bucket for which to remove notification rules - pub async fn remove_notification(&self, bucket: &str) { - self.bucket_rules_map.remove(&bucket.to_string()).await; - info!("Removed notification rules for bucket: {}", bucket); - } - /// Removes all targets pub async fn remove_all_bucket_targets(&self) { let mut target_list_guard = self.target_list.write().await; @@ -175,26 +86,6 @@ impl EventNotifier { info!("Removed all targets and their streams"); } - /// Checks if there are active subscribers for the given bucket and event name. - /// - /// # Parameters - /// * `bucket_name` - bucket name. - /// * `event_name` - Event name. - /// - /// # Return value - /// Return `true` if at least one matching notification rule exists. - pub async fn has_subscriber(&self, bucket_name: &str, event_name: &EventName) -> bool { - // Rules to check if the bucket exists - if let Some(rules_map) = self.bucket_rules_map.get(&bucket_name.to_string()).await { - // A composite event (such as ObjectCreatedAll) is expanded to multiple single events. - // We need to check whether any of these single events have the rules configured. - rules_map.has_subscriber(event_name) - } else { - // If no bucket is found, no subscribers - false - } - } - /// Sends an event to the appropriate targets based on the bucket rules /// /// # Arguments @@ -205,13 +96,7 @@ impl EventNotifier { let object_key = &event.s3.object.key; let event_name = event.event_name; - let Some(rules) = self.bucket_rules_map.get(bucket_name).await else { - debug!("No rules found for bucket: {}", bucket_name); - self.metrics.increment_skipped(); - return; - }; - - let target_ids = match_event_targets(&rules, event_name, object_key); + let target_ids = self.rule_engine.match_targets(bucket_name, event_name, object_key).await; if target_ids.is_empty() { debug!("No matching targets for event in bucket: {}", bucket_name); self.metrics.increment_skipped(); @@ -287,45 +172,30 @@ impl EventNotifier { info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name); } - /// Initializes the targets for buckets - /// - /// # Arguments - /// * `targets_to_init` - A vector of boxed targets to initialize - /// - /// # Returns - /// Returns `Ok(())` if initialization is successful, otherwise returns a `NotificationError`. + /// Initializes the targets for buckets from shared target handles. #[instrument(skip(self, targets_to_init))] - pub async fn init_bucket_targets( - &self, - targets_to_init: Vec + Send + Sync>>, - ) -> Result<(), NotificationError> { - // Currently active, simpler logic - let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList - - // Clear existing targets first - rebuild from scratch to ensure consistency with new configuration + pub async fn init_bucket_targets_shared(&self, targets_to_init: Vec>) -> Result<(), NotificationError> { + let mut target_list_guard = self.target_list.write().await; target_list_guard.clear(); - for target_boxed in targets_to_init { - // Traverse the incoming Box - debug!("init bucket target: {}", target_boxed.name()); - // TargetList::add method expectations Arc - // Therefore, you need to convert Box to Arc - let target_arc: Arc + Send + Sync> = Arc::from(target_boxed); - target_list_guard.add(target_arc)?; // Add Arc to the list + for target in targets_to_init { + debug!("init bucket target: {}", target.name()); + target_list_guard.add(target)?; } + info!( - "Initialized {} targets, list size: {}", // Clearer logs + "Initialized {} shared targets, list size: {}", target_list_guard.len(), target_list_guard.len() ); - Ok(()) // Make sure to return a Result + Ok(()) } } /// A thread-safe list of targets pub struct TargetList { /// Map of TargetID to Target - targets: HashMap + Send + Sync>>, + runtime: TargetRuntimeManager, } impl Default for TargetList { @@ -337,7 +207,9 @@ impl Default for TargetList { impl TargetList { /// Creates a new TargetList pub fn new() -> Self { - TargetList { targets: HashMap::new() } + TargetList { + runtime: TargetRuntimeManager::new(), + } } /// Adds a target to the list @@ -349,17 +221,17 @@ impl TargetList { /// Returns `Ok(())` if the target was added successfully, or a `NotificationError` if an error occurred. pub fn add(&mut self, target: Arc + Send + Sync>) -> Result<(), NotificationError> { let id = target.id(); - if self.targets.contains_key(&id) { + if self.runtime.get_by_target_id(&id).is_some() { // Potentially update or log a warning/error if replacing an existing target. warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id); } - self.targets.insert(id, target); + self.runtime.add_arc(target); Ok(()) } /// Clears all targets from the list pub fn clear(&mut self) { - self.targets.clear(); + self.runtime.clear(); } /// Removes a target by ID. Note: This does not stop its associated event stream. @@ -370,30 +242,14 @@ impl TargetList { /// /// # Returns /// Returns the removed target if it existed, otherwise `None`. - pub async fn remove_target_only(&mut self, id: &TargetID) -> Option + Send + Sync>> { - if let Some(target_arc) = self.targets.remove(id) { - if let Err(e) = target_arc.close().await { - // Target's own close logic - error!("Failed to close target {} during removal: {}", id, e); - } - Some(target_arc) - } else { - None - } + pub async fn remove_target_only(&mut self, id: &TargetID) -> Option> { + self.runtime.remove_by_target_id_and_close(id).await } /// Clears all targets from the list. Note: This does not stop their associated event streams. /// Stream cancellation should be handled by EventNotifier. pub async fn clear_targets_only(&mut self) { - let target_ids_to_clear: Vec = self.targets.keys().cloned().collect(); - for id in target_ids_to_clear { - if let Some(target_arc) = self.targets.remove(&id) - && let Err(e) = target_arc.close().await - { - error!("Failed to close target {} during clear: {}", id, e); - } - } - self.targets.clear(); + self.runtime.clear_and_close().await; } /// Returns a target by ID @@ -403,34 +259,54 @@ impl TargetList { /// /// # Returns /// Returns the target if it exists, otherwise `None`. - pub fn get(&self, id: &TargetID) -> Option + Send + Sync>> { - self.targets.get(id).cloned() + pub fn get(&self, id: &TargetID) -> Option> { + self.runtime.get_by_target_id(id) } /// Returns all target IDs pub fn keys(&self) -> Vec { - self.targets.keys().cloned().collect() + self.runtime.target_ids() } /// Returns all targets in the list - pub fn values(&self) -> Vec + Send + Sync>> { - self.targets.values().cloned().collect() + pub fn values(&self) -> Vec> { + self.runtime.values() + } + + pub fn runtime_snapshots(&self) -> Vec { + self.runtime.snapshots() + } + + pub async fn runtime_health_snapshots(&self) -> Vec { + self.runtime.health_snapshots().await + } + + pub fn runtime_status_snapshot( + &self, + replay_workers: &rustfs_targets::ReplayWorkerManager, + ) -> rustfs_targets::RuntimeStatusSnapshot { + self.runtime.status_snapshot(replay_workers) + } + + pub fn runtime_mut(&mut self) -> &mut TargetRuntimeManager { + &mut self.runtime } /// Returns the number of targets pub fn len(&self) -> usize { - self.targets.len() + self.runtime.len() } /// is_empty can be derived from len() pub fn is_empty(&self) -> bool { - self.targets.is_empty() + self.runtime.is_empty() } } #[cfg(test)] mod tests { use super::*; + use crate::{rule_engine::NotifyRuleEngine, rules::RulesMap}; use async_trait::async_trait; use rustfs_s3_common::EventName; use rustfs_targets::StoreError; @@ -445,40 +321,57 @@ mod tests { atomic::{AtomicUsize, Ordering}, }; - #[test] - fn encoded_event_key_matches_raw_prefix_suffix_filter() { + #[tokio::test] + async fn encoded_event_key_matches_raw_prefix_suffix_filter() { let target_id = TargetID::new("primary".to_string(), "webhook".to_string()); let mut rules_map = RulesMap::new(); rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id.clone()); - let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv"); + let rule_engine = NotifyRuleEngine::new(); + rule_engine.set_bucket_rules("test-bucket", rules_map).await; + + let targets = rule_engine + .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv") + .await; assert!(targets.contains(&target_id)); } - #[test] - fn encoded_event_key_matches_raw_and_decoded_rule_targets() { + #[tokio::test] + async fn encoded_event_key_matches_raw_and_decoded_rule_targets() { let raw_target = TargetID::new("raw".to_string(), "webhook".to_string()); let decoded_target = TargetID::new("decoded".to_string(), "webhook".to_string()); let mut rules_map = RulesMap::new(); rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads%2F*.csv".to_string(), raw_target.clone()); rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), decoded_target.clone()); - let targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.csv"); + let rule_engine = NotifyRuleEngine::new(); + rule_engine.set_bucket_rules("test-bucket", rules_map).await; + + let targets = rule_engine + .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.csv") + .await; assert_eq!(targets.len(), 2); assert!(targets.contains(&raw_target)); assert!(targets.contains(&decoded_target)); } - #[test] - fn encoded_event_key_does_not_bypass_suffix_filter() { + #[tokio::test] + async fn encoded_event_key_does_not_bypass_suffix_filter() { let target_id = TargetID::new("primary".to_string(), "webhook".to_string()); let mut rules_map = RulesMap::new(); rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "uploads/*.csv".to_string(), target_id); - let root_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "report.csv"); - let suffix_targets = match_event_targets(&rules_map, EventName::ObjectCreatedPut, "uploads%2Freport.txt"); + let rule_engine = NotifyRuleEngine::new(); + rule_engine.set_bucket_rules("test-bucket", rules_map).await; + + let root_targets = rule_engine + .match_targets("test-bucket", EventName::ObjectCreatedPut, "report.csv") + .await; + let suffix_targets = rule_engine + .match_targets("test-bucket", EventName::ObjectCreatedPut, "uploads%2Freport.txt") + .await; assert!(root_targets.is_empty()); assert!(suffix_targets.is_empty()); @@ -547,7 +440,8 @@ mod tests { #[tokio::test] async fn test_send_event_skips_disabled_target() { - let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new())); + let rule_engine = NotifyRuleEngine::new(); + let notifier = EventNotifier::new(Arc::new(NotificationMetrics::new()), rule_engine.clone()); let enabled_target = TestTarget::new("enabled-target", "webhook", true); let disabled_target = TestTarget::new("disabled-target", "webhook", false); @@ -556,7 +450,7 @@ mod tests { rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), enabled_target.id.clone()); rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), disabled_target.id.clone()); - notifier.add_rules_map("bucket", rules_map).await; + rule_engine.set_bucket_rules("bucket", rules_map).await; notifier .target_list() .write() diff --git a/crates/notify/src/pipeline.rs b/crates/notify/src/pipeline.rs new file mode 100644 index 000000000..6e611d82e --- /dev/null +++ b/crates/notify/src/pipeline.rs @@ -0,0 +1,141 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Event, integration::LiveEventBatch, notifier::EventNotifier}; +use std::collections::VecDeque; +use std::sync::Arc; +use tokio::sync::{RwLock, broadcast}; + +const MAX_RECENT_LIVE_EVENTS: usize = 1024; + +#[derive(Default)] +pub struct LiveEventHistory { + next_sequence: u64, + events: VecDeque<(u64, Arc)>, +} + +impl LiveEventHistory { + pub fn record(&mut self, event: Arc) { + self.next_sequence = self.next_sequence.saturating_add(1); + self.events.push_back((self.next_sequence, event)); + while self.events.len() > MAX_RECENT_LIVE_EVENTS { + self.events.pop_front(); + } + } + + pub fn snapshot_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { + let mut events = Vec::new(); + let mut next_sequence = after_sequence; + let mut truncated = false; + + for (sequence, event) in self.events.iter() { + if *sequence <= after_sequence { + continue; + } + if events.len() >= limit { + truncated = true; + break; + } + next_sequence = *sequence; + events.push(event.clone()); + } + + LiveEventBatch { + events, + next_sequence, + truncated, + } + } +} + +#[derive(Clone)] +pub struct NotifyPipeline { + notifier: Arc, + live_event_sender: broadcast::Sender>, + live_event_history: Arc>, +} + +impl NotifyPipeline { + pub fn new( + notifier: Arc, + live_event_sender: broadcast::Sender>, + live_event_history: Arc>, + ) -> Self { + Self { + notifier, + live_event_sender, + live_event_history, + } + } + + pub fn has_live_listeners(&self) -> bool { + self.live_event_sender.receiver_count() > 0 + } + + pub fn subscribe_live_events(&self) -> broadcast::Receiver> { + self.live_event_sender.subscribe() + } + + pub async fn recent_live_events_since(&self, after_sequence: u64, limit: usize) -> LiveEventBatch { + let history = self.live_event_history.read().await; + history.snapshot_since(after_sequence, limit.max(1)) + } + + pub async fn send_event(&self, event: Arc) { + self.live_event_history.write().await.record(event.clone()); + let _ = self.live_event_sender.send(event.clone()); + self.notifier.send(event).await; + } +} + +pub type NotifyEventBridge = NotifyPipeline; + +#[cfg(test)] +mod tests { + use super::{LiveEventHistory, NotifyPipeline}; + use crate::{Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine}; + use rustfs_s3_common::EventName; + use std::sync::Arc; + use tokio::sync::{RwLock, broadcast}; + + fn build_pipeline() -> NotifyPipeline { + let metrics = Arc::new(NotificationMetrics::new()); + let notifier = Arc::new(EventNotifier::new(metrics, NotifyRuleEngine::new())); + let (live_event_sender, _) = broadcast::channel(16); + NotifyPipeline::new(notifier, live_event_sender, Arc::new(RwLock::new(LiveEventHistory::default()))) + } + + #[tokio::test] + async fn pipeline_reports_live_listener_subscription_state() { + let pipeline = build_pipeline(); + assert!(!pipeline.has_live_listeners()); + + let _rx = pipeline.subscribe_live_events(); + assert!(pipeline.has_live_listeners()); + } + + #[tokio::test] + async fn pipeline_records_recent_live_events() { + let pipeline = build_pipeline(); + let event = Arc::new(Event::new_test_event("bucket", "one", EventName::ObjectCreatedPut)); + + pipeline.send_event(event).await; + + let batch = pipeline.recent_live_events_since(0, 16).await; + assert_eq!(batch.next_sequence, 1); + assert!(!batch.truncated); + assert_eq!(batch.events.len(), 1); + assert_eq!(batch.events[0].s3.object.key, "one"); + } +} diff --git a/crates/notify/src/rule_engine.rs b/crates/notify/src/rule_engine.rs new file mode 100644 index 000000000..c2e05d719 --- /dev/null +++ b/crates/notify/src/rule_engine.rs @@ -0,0 +1,133 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::rules::{RulesMap, TargetIdSet}; +use percent_encoding::percent_decode_str; +use rustfs_s3_common::EventName; +use rustfs_targets::arn::TargetID; +use starshard::AsyncShardedHashMap; +use std::sync::Arc; +use tracing::info; + +fn decoded_object_key_for_matching(object_key: &str) -> Option { + if !object_key.contains('%') { + return None; + } + + let decoded = percent_decode_str(object_key).decode_utf8().ok()?; + (decoded != object_key).then(|| decoded.into_owned()) +} + +#[derive(Clone)] +pub struct NotifyRuleEngine { + bucket_rules_map: Arc>, +} + +impl NotifyRuleEngine { + pub fn new() -> Self { + Self { + bucket_rules_map: Arc::new(AsyncShardedHashMap::new(0)), + } + } + + pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool { + let items = self.bucket_rules_map.iter().await; + for (_bucket, rules_map) in items { + if rules_map.contains_target_id(target_id) { + return true; + } + } + false + } + + pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) { + if rules_map.is_empty() { + self.bucket_rules_map.remove(&bucket.to_string()).await; + } else { + self.bucket_rules_map.insert(bucket.to_string(), rules_map).await; + } + info!("Updated notification rules for bucket: {}", bucket); + } + + pub async fn get_bucket_rules(&self, bucket: &str) -> Option { + self.bucket_rules_map.get(&bucket.to_string()).await + } + + pub async fn clear_bucket_rules(&self, bucket: &str) { + if self.bucket_rules_map.remove(&bucket.to_string()).await.is_some() { + info!("Removed all notification rules for bucket: {}", bucket); + } + } + + pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool { + self.get_bucket_rules(bucket) + .await + .is_some_and(|rules_map| rules_map.has_subscriber(event)) + } + + pub async fn match_targets(&self, bucket: &str, event_name: EventName, object_key: &str) -> TargetIdSet { + self.get_bucket_rules(bucket) + .await + .map_or_else(TargetIdSet::new, |rules_map| { + let mut target_ids = rules_map.match_rules(event_name, object_key); + if let Some(decoded_key) = decoded_object_key_for_matching(object_key) { + target_ids.extend(rules_map.match_rules(event_name, &decoded_key)); + } + target_ids + }) + } +} + +impl Default for NotifyRuleEngine { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::NotifyRuleEngine; + use crate::rules::RulesMap; + use rustfs_s3_common::EventName; + use rustfs_targets::arn::TargetID; + + #[tokio::test] + async fn rule_engine_tracks_bucket_rule_lifecycle() { + let engine = NotifyRuleEngine::new(); + let target_id = TargetID::new("primary".to_string(), "webhook".to_string()); + let mut rules_map = RulesMap::new(); + rules_map.add_rule_config(&[EventName::ObjectCreatedPut], "*".to_string(), target_id.clone()); + + assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await); + assert!(!engine.is_target_bound_to_any_bucket(&target_id).await); + + engine.set_bucket_rules("bucket", rules_map).await; + + assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await); + assert!(engine.is_target_bound_to_any_bucket(&target_id).await); + assert_eq!( + engine + .match_targets("bucket", EventName::ObjectCreatedPut, "object") + .await + .into_iter() + .collect::>(), + vec![target_id.clone()] + ); + + engine.clear_bucket_rules("bucket").await; + + assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await); + assert!(!engine.is_target_bound_to_any_bucket(&target_id).await); + } +} diff --git a/crates/notify/src/runtime_facade.rs b/crates/notify/src/runtime_facade.rs new file mode 100644 index 000000000..1507f7f66 --- /dev/null +++ b/crates/notify/src/runtime_facade.rs @@ -0,0 +1,239 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Event, NotificationError, integration::NotificationMetrics, notifier::SharedNotifyTargetList}; +use rustfs_targets::{ + BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter, ReplayEvent, ReplayWorkerManager, RuntimeActivation, Target, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{RwLock, Semaphore}; +use tracing::info; + +#[derive(Clone)] +pub struct NotifyRuntimeFacade { + target_list: SharedNotifyTargetList, + replay_workers: Arc>, + runtime_adapter: Arc>, +} + +impl NotifyRuntimeFacade { + pub fn new( + target_list: SharedNotifyTargetList, + replay_workers: Arc>, + concurrency_limiter: Arc, + metrics: Arc, + ) -> Self { + let replay_metrics = metrics; + let runtime_adapter = BuiltinPluginRuntimeAdapter::new( + Arc::new(move |event: ReplayEvent| { + let metrics = replay_metrics.clone(); + Box::pin(async move { + match event { + ReplayEvent::Delivered { .. } => metrics.increment_processed(), + ReplayEvent::RetryableError { .. } => {} + ReplayEvent::Dropped { target, .. } + | ReplayEvent::PermanentFailure { target, .. } + | ReplayEvent::RetryExhausted { target, .. } => { + target.record_final_failure(); + metrics.increment_failed(); + } + ReplayEvent::UnreadableEntry { .. } => {} + } + }) + }), + Arc::new(|target_id, has_replay| { + if has_replay { + info!("Event stream processing for target {} is started successfully", target_id); + } else { + info!("Target {} has no replay worker to start", target_id); + } + }), + Some(concurrency_limiter), + Duration::from_secs(5), + Duration::from_millis(500), + "Stop event stream processing for target", + ); + + Self { + target_list, + replay_workers, + runtime_adapter: Arc::new(runtime_adapter), + } + } + + pub async fn activate_targets_with_replay( + &self, + targets: Vec + Send + Sync>>, + ) -> RuntimeActivation { + self.runtime_adapter.activate_with_replay(targets).await + } + + pub async fn replace_targets(&self, activation: RuntimeActivation) -> Result<(), NotificationError> { + let mut target_list = self.target_list.write().await; + let mut replay_workers = self.replay_workers.write().await; + self.runtime_adapter + .replace_runtime_targets(target_list.runtime_mut(), &mut replay_workers, activation) + .await + .map_err(NotificationError::Target)?; + Ok(()) + } + + pub async fn stop_replay_workers(&self) { + let mut replay_workers = self.replay_workers.write().await; + self.runtime_adapter.stop_replay_workers(&mut replay_workers).await; + } + + pub async fn shutdown(&self) { + info!("Turn off the notification system"); + + let active_targets = self.replay_workers.read().await.len(); + info!("Stops {} active event stream processing tasks", active_targets); + + { + let mut target_list = self.target_list.write().await; + let mut replay_workers = self.replay_workers.write().await; + if let Err(err) = self + .runtime_adapter + .shutdown(target_list.runtime_mut(), &mut replay_workers) + .await + { + tracing::error!(error = %err, "Failed to shutdown notify runtime cleanly"); + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + + info!("Notify the system to be shut down completed"); + } +} + +#[cfg(test)] +mod tests { + use super::NotifyRuntimeFacade; + use crate::{ + Event, integration::NotificationMetrics, notifier::EventNotifier, rule_engine::NotifyRuleEngine, + runtime_view::NotifyRuntimeView, + }; + use async_trait::async_trait; + use rustfs_targets::arn::TargetID; + use rustfs_targets::store::{Key, Store}; + use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use rustfs_targets::{ReplayWorkerManager, SharedTarget, StoreError, Target, TargetError}; + use serde::{Serialize, de::DeserializeOwned}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::{RwLock, Semaphore}; + + #[derive(Clone)] + struct TestTarget { + close_calls: Arc, + id: TargetID, + } + + impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + close_calls: Arc::new(AtomicUsize::new(0)), + id: TargetID::new(id.to_string(), name.to_string()), + } + } + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + fn build_facade() -> (NotifyRuntimeFacade, Arc, Arc>) { + let metrics = Arc::new(NotificationMetrics::new()); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), NotifyRuleEngine::new())); + let target_list = notifier.target_list(); + let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new())); + let facade = NotifyRuntimeFacade::new(target_list, replay_workers.clone(), Arc::new(Semaphore::new(4)), metrics); + (facade, notifier, replay_workers) + } + + #[tokio::test] + async fn runtime_facade_stops_empty_replay_workers() { + let (facade, _, _) = build_facade(); + facade.stop_replay_workers().await; + } + + #[tokio::test] + async fn runtime_facade_activates_empty_target_list() { + let (facade, _, _) = build_facade(); + let activation = facade.activate_targets_with_replay(Vec::new()).await; + + assert!(activation.targets.is_empty()); + assert_eq!(activation.replay_workers.len(), 0); + } + + #[tokio::test] + async fn runtime_facade_replace_targets_commits_runtime_state() { + let (facade, notifier, replay_workers) = build_facade(); + let target = TestTarget::new("primary", "webhook"); + let activation = rustfs_targets::RuntimeActivation { + replay_workers: ReplayWorkerManager::new(), + targets: vec![Arc::new(target) as SharedTarget], + }; + + facade + .replace_targets(activation) + .await + .expect("replace_targets should succeed"); + + let runtime_view = NotifyRuntimeView::new(notifier.target_list(), replay_workers.clone()); + let active_targets = runtime_view.get_active_targets().await; + assert_eq!(active_targets, vec![TargetID::new("primary".to_string(), "webhook".to_string())]); + assert_eq!(replay_workers.read().await.len(), 0); + } +} diff --git a/crates/notify/src/runtime_view.rs b/crates/notify/src/runtime_view.rs new file mode 100644 index 000000000..00e3f3767 --- /dev/null +++ b/crates/notify/src/runtime_view.rs @@ -0,0 +1,261 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{Event, NotificationTargetMetricSnapshot, notifier::SharedNotifyTargetList}; +use rustfs_targets::{ReplayWorkerManager, RuntimeTargetHealthSnapshot, SharedTarget, arn::TargetID}; +use std::sync::Arc; +use tokio::sync::RwLock; + +#[derive(Clone)] +pub struct NotifyRuntimeView { + target_list: SharedNotifyTargetList, + stream_cancellers: Arc>, +} + +impl NotifyRuntimeView { + pub fn new(target_list: SharedNotifyTargetList, stream_cancellers: Arc>) -> Self { + Self { + target_list, + stream_cancellers, + } + } + + pub async fn get_active_targets(&self) -> Vec { + self.target_list.read().await.keys() + } + + pub fn get_all_targets(&self) -> SharedNotifyTargetList { + self.target_list.clone() + } + + pub async fn get_target_values(&self) -> Vec> { + self.target_list.read().await.values() + } + + pub async fn snapshot_target_metrics(&self) -> Vec { + self.target_list + .read() + .await + .runtime_snapshots() + .into_iter() + .map(|snapshot| NotificationTargetMetricSnapshot { + failed_messages: snapshot.failed_messages, + queue_length: snapshot.queue_length, + target_id: snapshot.target_id, + target_type: snapshot.target_type, + total_messages: snapshot.total_messages, + }) + .collect() + } + + pub async fn snapshot_target_health(&self) -> Vec { + self.target_list.read().await.runtime_health_snapshots().await + } + + pub async fn runtime_status_snapshot(&self) -> rustfs_targets::RuntimeStatusSnapshot { + let replay_workers = self.stream_cancellers.read().await; + let target_list = self.target_list.read().await; + target_list.runtime_status_snapshot(&replay_workers) + } +} + +#[cfg(test)] +mod tests { + use super::NotifyRuntimeView; + use crate::{Event, notifier::TargetList}; + use async_trait::async_trait; + use rustfs_targets::arn::TargetID; + use rustfs_targets::store::{Key, Store}; + use rustfs_targets::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliverySnapshot}; + use rustfs_targets::{ReplayWorkerManager, StoreError, Target, TargetError}; + use serde::{Serialize, de::DeserializeOwned}; + use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }; + use tokio::sync::RwLock; + + #[derive(Clone)] + struct TestTarget { + active: bool, + enabled: bool, + failed_messages: Arc, + id: TargetID, + total_messages: Arc, + } + + impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + active: true, + enabled: true, + failed_messages: Arc::new(AtomicU64::new(0)), + id: TargetID::new(id.to_string(), name.to_string()), + total_messages: Arc::new(AtomicU64::new(0)), + } + } + + fn with_active(mut self, active: bool) -> Self { + self.active = active; + self + } + + fn with_enabled(mut self, enabled: bool) -> Self { + self.enabled = enabled; + self + } + + fn record_successes(&self, count: u64) { + self.total_messages.store(count, Ordering::Relaxed); + } + + fn record_failures(&self, count: u64) { + self.failed_messages.store(count, Ordering::Relaxed); + } + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(self.active) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn is_enabled(&self) -> bool { + self.enabled + } + + fn delivery_snapshot(&self) -> TargetDeliverySnapshot { + TargetDeliverySnapshot { + failed_messages: self.failed_messages.load(Ordering::Relaxed), + queue_length: 0, + total_messages: self.total_messages.load(Ordering::Relaxed), + } + } + } + + #[tokio::test] + async fn runtime_view_reports_empty_runtime_queries() { + let runtime_view = NotifyRuntimeView::new( + Arc::new(RwLock::new(TargetList::new())), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + ); + + assert!(runtime_view.get_active_targets().await.is_empty()); + assert!(runtime_view.get_target_values().await.is_empty()); + assert!(runtime_view.get_all_targets().read().await.is_empty()); + } + + #[tokio::test] + async fn runtime_view_reports_empty_runtime_snapshots() { + let runtime_view = NotifyRuntimeView::new( + Arc::new(RwLock::new(TargetList::new())), + Arc::new(RwLock::new(ReplayWorkerManager::new())), + ); + + assert!(runtime_view.snapshot_target_metrics().await.is_empty()); + assert!(runtime_view.snapshot_target_health().await.is_empty()); + + let status = runtime_view.runtime_status_snapshot().await; + assert_eq!(status.target_count, 0); + assert_eq!(status.replay_worker_count, 0); + } + + #[tokio::test] + async fn runtime_view_reports_non_empty_runtime_queries_and_snapshots() { + let target_list = Arc::new(RwLock::new(TargetList::new())); + let replay_workers = Arc::new(RwLock::new(ReplayWorkerManager::new())); + + let online = Arc::new(TestTarget::new("primary", "webhook")); + online.record_successes(3); + online.record_failures(1); + + let disabled = Arc::new(TestTarget::new("backup", "mqtt").with_enabled(false).with_active(false)); + disabled.record_successes(2); + + { + let mut targets = target_list.write().await; + targets.add(online.clone() as Arc + Send + Sync>).unwrap(); + targets.add(disabled.clone() as Arc + Send + Sync>).unwrap(); + } + + let runtime_view = NotifyRuntimeView::new(target_list.clone(), replay_workers.clone()); + + let mut active_targets = runtime_view.get_active_targets().await; + active_targets.sort(); + assert_eq!( + active_targets, + vec![ + TargetID::new("backup".to_string(), "mqtt".to_string()), + TargetID::new("primary".to_string(), "webhook".to_string()) + ] + ); + + let target_values = runtime_view.get_target_values().await; + assert_eq!(target_values.len(), 2); + assert_eq!(runtime_view.get_all_targets().read().await.len(), 2); + + let metric_snapshots = runtime_view.snapshot_target_metrics().await; + assert_eq!(metric_snapshots.len(), 2); + assert_eq!(metric_snapshots[0].target_id, "backup:mqtt"); + assert_eq!(metric_snapshots[0].failed_messages, 0); + assert_eq!(metric_snapshots[0].total_messages, 2); + assert_eq!(metric_snapshots[1].target_id, "primary:webhook"); + assert_eq!(metric_snapshots[1].failed_messages, 1); + assert_eq!(metric_snapshots[1].total_messages, 3); + + let health_snapshots = runtime_view.snapshot_target_health().await; + assert_eq!(health_snapshots.len(), 2); + assert_eq!(health_snapshots[0].target_id, "backup:mqtt"); + assert!(!health_snapshots[0].enabled); + assert_eq!(health_snapshots[0].state, rustfs_targets::RuntimeTargetHealthState::Disabled); + assert_eq!(health_snapshots[1].target_id, "primary:webhook"); + assert!(health_snapshots[1].enabled); + assert_eq!(health_snapshots[1].state, rustfs_targets::RuntimeTargetHealthState::Online); + + let status = runtime_view.runtime_status_snapshot().await; + assert_eq!(status.target_count, 2); + assert_eq!(status.replay_worker_count, 0); + } +} diff --git a/crates/notify/src/services.rs b/crates/notify/src/services.rs new file mode 100644 index 000000000..49a7114c8 --- /dev/null +++ b/crates/notify/src/services.rs @@ -0,0 +1,120 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + Event, + bucket_config_manager::NotifyBucketConfigManager, + config_manager::NotifyConfigManager, + integration::NotificationMetrics, + notification_system_subscriber::NotificationSystemSubscriberView, + notifier::{EventNotifier, SharedNotifyTargetList}, + pipeline::{LiveEventHistory, NotifyPipeline}, + registry::TargetRegistry, + rule_engine::NotifyRuleEngine, + runtime_facade::NotifyRuntimeFacade, + runtime_view::NotifyRuntimeView, + status_view::NotifyStatusView, +}; +use rustfs_ecstore::config::Config; +use rustfs_targets::ReplayWorkerManager; +use std::sync::Arc; +use tokio::sync::{RwLock, Semaphore, broadcast}; + +#[derive(Clone)] +pub struct NotifyServices { + pub bucket_config_manager: NotifyBucketConfigManager, + pub config_manager: NotifyConfigManager, + pub pipeline: NotifyPipeline, + pub runtime_facade: NotifyRuntimeFacade, + pub runtime_view: NotifyRuntimeView, + pub status_view: NotifyStatusView, +} + +impl NotifyServices { + #[allow(clippy::too_many_arguments)] + pub fn new( + notifier: Arc, + rule_engine: NotifyRuleEngine, + target_list: SharedNotifyTargetList, + registry: Arc, + config: Arc>, + stream_cancellers: Arc>, + concurrency_limiter: Arc, + metrics: Arc, + subscriber_view: Arc, + live_event_sender: broadcast::Sender>, + live_event_history: Arc>, + ) -> Self { + let runtime_view = NotifyRuntimeView::new(target_list.clone(), stream_cancellers.clone()); + let runtime_facade = NotifyRuntimeFacade::new(target_list, stream_cancellers, concurrency_limiter, metrics.clone()); + let config_manager = NotifyConfigManager::new(config, registry, rule_engine.clone(), runtime_facade.clone()); + let bucket_config_manager = NotifyBucketConfigManager::new(notifier.clone(), rule_engine, subscriber_view); + let pipeline = NotifyPipeline::new(notifier, live_event_sender, live_event_history); + let status_view = NotifyStatusView::new(metrics); + + Self { + bucket_config_manager, + config_manager, + pipeline, + runtime_facade, + runtime_view, + status_view, + } + } +} + +#[cfg(test)] +mod tests { + use super::NotifyServices; + use crate::{ + integration::NotificationMetrics, notification_system_subscriber::NotificationSystemSubscriberView, + notifier::EventNotifier, pipeline::LiveEventHistory, registry::TargetRegistry, rule_engine::NotifyRuleEngine, + }; + use rustfs_ecstore::config::Config; + use rustfs_targets::ReplayWorkerManager; + use std::sync::Arc; + use tokio::sync::{RwLock, Semaphore, broadcast}; + + #[tokio::test] + async fn services_build_empty_runtime_views() { + let metrics = Arc::new(NotificationMetrics::new()); + let rule_engine = NotifyRuleEngine::new(); + let notifier = Arc::new(EventNotifier::new(metrics.clone(), rule_engine.clone())); + let target_list = notifier.target_list(); + let registry = Arc::new(TargetRegistry::new()); + let config = Arc::new(RwLock::new(Config::default())); + let stream_cancellers = Arc::new(RwLock::new(ReplayWorkerManager::new())); + let concurrency_limiter = Arc::new(Semaphore::new(4)); + let subscriber_view = Arc::new(NotificationSystemSubscriberView::new()); + let (live_event_sender, _) = broadcast::channel(16); + let live_event_history = Arc::new(RwLock::new(LiveEventHistory::default())); + + let services = NotifyServices::new( + notifier, + rule_engine, + target_list, + registry, + config, + stream_cancellers, + concurrency_limiter, + metrics, + subscriber_view, + live_event_sender, + live_event_history, + ); + + assert!(services.runtime_view.get_active_targets().await.is_empty()); + assert_eq!(services.status_view.snapshot_metrics().events_sent_total, 0); + } +} diff --git a/crates/notify/src/status_view.rs b/crates/notify/src/status_view.rs new file mode 100644 index 000000000..4a1aba7b3 --- /dev/null +++ b/crates/notify/src/status_view.rs @@ -0,0 +1,74 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::integration::{NotificationMetricSnapshot, NotificationMetrics}; +use hashbrown::HashMap; +use std::sync::Arc; + +#[derive(Clone)] +pub struct NotifyStatusView { + metrics: Arc, +} + +impl NotifyStatusView { + pub fn new(metrics: Arc) -> Self { + Self { metrics } + } + + pub fn get_status(&self) -> HashMap { + let mut status = HashMap::new(); + + status.insert("uptime_seconds".to_string(), self.metrics.uptime().as_secs().to_string()); + status.insert("processing_events".to_string(), self.metrics.processing_count().to_string()); + status.insert("processed_events".to_string(), self.metrics.processed_count().to_string()); + status.insert("failed_events".to_string(), self.metrics.failed_count().to_string()); + status.insert("skipped_events".to_string(), self.metrics.skipped_count().to_string()); + + status + } + + pub fn snapshot_metrics(&self) -> NotificationMetricSnapshot { + self.metrics.snapshot() + } +} + +#[cfg(test)] +mod tests { + use super::NotifyStatusView; + use crate::integration::NotificationMetrics; + use std::sync::Arc; + + #[test] + fn status_view_reports_empty_metrics_snapshot() { + let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new())); + + let snapshot = status_view.snapshot_metrics(); + assert_eq!(snapshot.current_send_in_progress, 0); + assert_eq!(snapshot.events_errors_total, 0); + assert_eq!(snapshot.events_sent_total, 0); + assert_eq!(snapshot.events_skipped_total, 0); + } + + #[test] + fn status_view_exposes_status_map_keys() { + let status_view = NotifyStatusView::new(Arc::new(NotificationMetrics::new())); + let status = status_view.get_status(); + + assert!(status.contains_key("uptime_seconds")); + assert!(status.contains_key("processing_events")); + assert!(status.contains_key("processed_events")); + assert!(status.contains_key("failed_events")); + assert!(status.contains_key("skipped_events")); + } +} diff --git a/crates/notify/src/stream.rs b/crates/notify/src/stream.rs deleted file mode 100644 index 19f5ade2e..000000000 --- a/crates/notify/src/stream.rs +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{Event, integration::NotificationMetrics}; -use rustfs_targets::{ - StoreError, Target, TargetError, - store::{Key, Store, ensure_store_entry_raw_readable}, - target::QueuedPayload, -}; -use rustfs_utils::get_env_usize; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Semaphore, mpsc}; -use tokio::time::sleep; -use tracing::{debug, error, info, warn}; - -/// Streams events from the store to the target with retry logic -/// -/// # Arguments -/// - `store`: The event store -/// - `target`: The target to send events to -/// - `cancel_rx`: Receiver to listen for cancellation signals -pub async fn stream_events( - store: &mut (dyn Store + Send), - target: &dyn Target, - mut cancel_rx: mpsc::Receiver<()>, -) { - info!("Starting event stream for target: {}", target.name()); - - // Retry configuration - const MAX_RETRIES: usize = 5; - const RETRY_DELAY: Duration = Duration::from_secs(5); - - loop { - // Check for cancellation signal - if cancel_rx.try_recv().is_ok() { - info!("Cancellation received for target: {}", target.name()); - return; - } - - // Get list of events in the store - let keys = store.list(); - if keys.is_empty() { - // No events, wait before checking again - sleep(Duration::from_secs(1)).await; - continue; - } - - // Process each event - for key in keys { - // Check for cancellation before processing each event - if cancel_rx.try_recv().is_ok() { - info!("Cancellation received during processing for target: {}", target.name()); - return; - } - - let mut retry_count = 0; - let mut success = false; - - // Retry logic - while retry_count < MAX_RETRIES && !success { - match target.send_from_store(key.clone()).await { - Ok(_) => { - info!("Successfully sent event for target: {}", target.name()); - // send_from_store deletes the event from store on success - success = true; - } - Err(e) => { - // Handle specific errors - match &e { - TargetError::NotConnected => { - warn!("Target {} not connected, retrying...", target.name()); - retry_count += 1; - sleep(RETRY_DELAY).await; - } - TargetError::Timeout(_) => { - warn!("Timeout for target {}, retrying...", target.name()); - retry_count += 1; - sleep(Duration::from_secs((retry_count * 5) as u64)).await; // Exponential backoff - } - _ => { - // Permanent error, skip this event - error!("Permanent error for target {}: {}", target.name(), e); - break; - } - } - } - } - } - - // Remove event from store if successfully sent - if retry_count >= MAX_RETRIES && !success { - warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name()); - } - } - - // Small delay before next iteration - sleep(Duration::from_millis(100)).await; - } -} - -/// Starts the event streaming process for a target -/// -/// # Arguments -/// - `store`: The event store -/// - `target`: The target to send events to -/// -/// # Returns -/// A sender to signal cancellation of the event stream -pub fn start_event_stream( - mut store: Box + Send>, - target: Arc + Send + Sync>, -) -> mpsc::Sender<()> { - let (cancel_tx, cancel_rx) = mpsc::channel(1); - - tokio::spawn(async move { - stream_events(&mut *store, &*target, cancel_rx).await; - info!("Event stream stopped for target: {}", target.name()); - }); - - cancel_tx -} - -/// Start event stream with batch processing -/// -/// # Arguments -/// - `store`: The event store -/// - `target`: The target to send events to clients -/// - `metrics`: Metrics for monitoring -/// - `semaphore`: Semaphore to limit concurrency -/// -/// # Returns -/// A sender to signal cancellation of the event stream -pub fn start_event_stream_with_batching( - mut store: Box + Send>, - target: Arc + Send + Sync>, - metrics: Arc, - semaphore: Arc, -) -> mpsc::Sender<()> { - let (cancel_tx, cancel_rx) = mpsc::channel(1); - debug!("Starting event stream with batching for target: {}", target.name()); - tokio::spawn(async move { - stream_events_with_batching(&mut *store, &*target, cancel_rx, metrics, semaphore).await; - info!("Event stream stopped for target: {}", target.name()); - }); - - cancel_tx -} - -/// Event stream processing with batch processing -/// -/// # Arguments -/// - `store`: The event store -/// - `target`: The target to send events to clients -/// - `cancel_rx`: Receiver to listen for cancellation signals -/// - `metrics`: Metrics for monitoring -/// - `semaphore`: Semaphore to limit concurrency -/// -/// # Notes -/// This function processes events in batches to improve efficiency. -pub async fn stream_events_with_batching( - store: &mut (dyn Store + Send), - target: &dyn Target, - mut cancel_rx: mpsc::Receiver<()>, - metrics: Arc, - semaphore: Arc, -) { - info!("Starting event stream with batching for target: {}", target.name()); - - // Configuration parameters - const DEFAULT_BATCH_SIZE: usize = 1; - let batch_size = get_env_usize("RUSTFS_EVENT_BATCH_SIZE", DEFAULT_BATCH_SIZE); - const BATCH_TIMEOUT: Duration = Duration::from_secs(5); - const MAX_RETRIES: usize = 5; - const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); - - let mut batch_keys = Vec::with_capacity(batch_size); - let mut last_flush = Instant::now(); - - loop { - // Check the cancel signal - if cancel_rx.try_recv().is_ok() { - info!("Cancellation received for target: {}", target.name()); - return; - } - - // Get a list of events in storage - let keys = store.list(); - debug!("Found {} keys in store for target: {}", keys.len(), target.name()); - if keys.is_empty() { - // If there is data in the batch and timeout, refresh the batch - if !batch_keys.is_empty() && last_flush.elapsed() >= BATCH_TIMEOUT { - process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; - last_flush = Instant::now(); - } - - // No event, wait before checking - tokio::time::sleep(Duration::from_millis(500)).await; - continue; - } - - // Handle each event - for key in keys { - // Check the cancel signal again - if cancel_rx.try_recv().is_ok() { - info!("Cancellation received during processing for target: {}", target.name()); - - // Processing collected batches before exiting - if !batch_keys.is_empty() { - process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; - } - return; - } - - // Skip unreadable entries so a single corrupt file cannot stall the stream. - // ensure_store_entry_raw_readable attempts get_raw; on I/O error it calls del() to - // remove the corrupt entry before returning Err, so no cleanup is needed here. - match ensure_store_entry_raw_readable(&*store, &key) { - Ok(true) => {} // entry is readable, proceed - Ok(false) => continue, // entry not found (already removed), skip - Err(err) => { - warn!("Skipping unreadable store entry {} for target {}: {}", key, target.name(), err); - continue; // corrupt entry was already deleted by ensure_store_entry_raw_readable - } - } - - batch_keys.push(key); - metrics.increment_processing(); - - // If the batch is full or enough time has passed since the last refresh, the batch will be processed - if batch_keys.len() >= batch_size || last_flush.elapsed() >= BATCH_TIMEOUT { - process_batch(&mut batch_keys, target, MAX_RETRIES, BASE_RETRY_DELAY, &metrics, &semaphore).await; - last_flush = Instant::now(); - } - } - - // A small delay will be conducted to check the next round - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - -/// Processing event batches for targets -/// # Arguments -/// - `batch`: The batch of events to process -/// - `batch_keys`: The corresponding keys of the events in the batch -/// - `target`: The target to send events to clients -/// - `max_retries`: Maximum number of retries for sending an event -/// - `base_delay`: Base delay duration for retries -/// - `metrics`: Metrics for monitoring -/// - `semaphore`: Semaphore to limit concurrency -/// # Notes -/// This function processes a batch of events, sending each event to the target with retry -async fn process_batch( - batch_keys: &mut Vec, - target: &dyn Target, - max_retries: usize, - base_delay: Duration, - metrics: &Arc, - semaphore: &Arc, -) { - debug!("Processing batch of {} events for target: {}", batch_keys.len(), target.name()); - if batch_keys.is_empty() { - return; - } - - // Obtain semaphore permission to limit concurrency - let permit = match semaphore.clone().acquire_owned().await { - Ok(permit) => permit, - Err(e) => { - error!("Failed to acquire semaphore permit: {}", e); - return; - } - }; - - // Handle every event in the batch - for key in batch_keys.iter() { - let mut retry_count = 0; - let mut success = false; - - // Retry logic - while retry_count < max_retries && !success { - match target.send_from_store(key.clone()).await { - Ok(_) => { - debug!("Successfully sent event for target: {}, Key: {}", target.name(), key.to_string()); - success = true; - metrics.increment_processed(); - } - Err(e) => match &e { - TargetError::NotConnected => { - warn!("Target {} not connected, retrying...", target.name()); - retry_count += 1; - let jitter = Duration::from_millis(key.to_string().len() as u64 % 500); - let backoff = 1u32 << retry_count as u32; - tokio::time::sleep(base_delay * backoff + jitter).await; - } - TargetError::Timeout(_) => { - warn!("Timeout for target {}, retrying...", target.name()); - retry_count += 1; - let jitter = Duration::from_millis(key.to_string().len() as u64 % 500); - let backoff = 1u32 << retry_count as u32; - tokio::time::sleep(base_delay * backoff + jitter).await; - } - TargetError::Dropped(reason) => { - warn!("Dropped queued payload for target {}: {}", target.name(), reason); - metrics.increment_failed(); - break; - } - _ => { - error!("Permanent error for target {}: {}", target.name(), e); - target.record_final_failure(); - metrics.increment_failed(); - break; - } - }, - } - } - - if retry_count >= max_retries && !success { - warn!("Max retries exceeded for event {}, target: {}, skipping", key.to_string(), target.name()); - target.record_final_failure(); - metrics.increment_failed(); - } - } - - // Clear processed batches - batch_keys.clear(); - - // Release semaphore permission (via drop) - drop(permit); -} diff --git a/crates/targets/AGENTS.md b/crates/targets/AGENTS.md index bb2cdabd9..03d79992e 100644 --- a/crates/targets/AGENTS.md +++ b/crates/targets/AGENTS.md @@ -2,34 +2,68 @@ Applies to `crates/targets/`. -`rustfs-targets` provides the notification target abstraction layer: -the `Target` trait, built-in implementations (Webhook, Kafka, MQTT, -NATS, Pulsar, MySQL, Postgres, Redis), persistent queue store, -DSN/configuration builders, and the `ChannelTargetType` registry that -maps target types to their runtime factories. +`rustfs-targets` is the shared target-plugin foundation for `audit` and +`notify`. It owns plugin metadata, builtin target descriptors, runtime +orchestration primitives, and plugin control-plane state modeling. + +## Current Module Boundaries + +- `manifest.rs`: declarative plugin metadata and marketplace-facing shape. + Keep this layer declarative only; do not add runtime execution logic here. +- `catalog/`: centralized builtin descriptor registration and example external + plugin assembly. Keep admin-facing plugin source data here instead of + spreading it into handlers. +- `plugin.rs`: `TargetPluginDescriptor`, `TargetPluginRegistry`, + `BuiltinTargetDescriptor`, and admin descriptor metadata. +- `runtime/`: shared runtime lifecycle and replay orchestration: + `TargetRuntimeManager`, `ReplayWorkerManager`, `PluginRuntimeAdapter`, + `BuiltinPluginRuntimeAdapter`, and sidecar protocol/runtime MVP types. +- `control_plane.rs`: install/enable/runtime state models and install policy + validation helpers. Keep install/governance state logic centralized here. +- `config/`, `target/`, `store/`, `check/`: target config normalization, + target implementations, queue/store, and endpoint connectivity checks. + +## Change Style and Ownership Rules + +- Preserve the layering above. Do not move install/runtime/governance logic + into admin handlers or manifest structs. +- Prefer extending shared abstractions (`TargetPluginRegistry`, + `PluginRuntimeAdapter`, `TargetRuntimeManager`) over duplicating per-domain + orchestration logic. +- Keep external sidecar behavior scoped to current MVP boundaries unless the + task explicitly includes real installer/transport integration. +- Reuse existing constants/keys from `rustfs_config`; avoid introducing + duplicate literals for target field names and subsystem keys. ## Library Design - Treat crate code as reusable library code by default. -- Prefer `thiserror` for library-facing error types. -- Do not use `unwrap()`, `expect()`, or panic-driven control flow outside tests. +- Return structured `TargetError`/`StoreError` results; avoid panic-driven + control flow outside tests. +- Keep serialization contracts stable for types re-exported by `lib.rs`. ## Testing - Keep unit tests close to the module they test. -- Keep integration tests under `crates/targets/tests/` directory. -- Add regression tests for bug fixes and behavior changes. +- Keep integration tests under `crates/targets/tests/`. +- Add regression tests for behavior changes in: + - plugin manifest/catalog/control-plane contracts + - runtime adapter lifecycle behavior + - target config normalization and validation + - sidecar handshake/policy validation paths ## Async and Performance - Keep async paths non-blocking. -- Move CPU-heavy operations out of async hot paths with `tokio::task::spawn_blocking` when appropriate. +- Avoid hot-path allocations and repeated config normalization when a cached + snapshot can be reused. +- Use bounded concurrency and timeout guards for runtime and health checks. ## Integration Tests Integration tests under `tests/` are `#[ignore]` by default so CI never runs -them. See the module-level doc comment in each test file for prerequisites -and run commands: +them. See module-level doc comments in each test file for prerequisites and +run commands. - `tests/mysql_integration.rs` — MySQL 8.0+ / TiDB 8.5+ - `tests/postgres_integration.rs` — PostgreSQL @@ -37,4 +71,8 @@ and run commands: ## Suggested Validation - `cargo test -p rustfs-targets` +- If runtime/plugin contracts changed, run focused tests under: + - `cargo test -p rustfs-targets plugin` + - `cargo test -p rustfs-targets runtime` + - `cargo test -p rustfs-targets control_plane` - Full gate before commit: `make pre-commit` diff --git a/crates/targets/src/catalog/builtin.rs b/crates/targets/src/catalog/builtin.rs new file mode 100644 index 000000000..83eb82dff --- /dev/null +++ b/crates/targets/src/catalog/builtin.rs @@ -0,0 +1,426 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::plugin::{ + BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetRequestValidator, + boxed_target, +}; +use crate::target::{ChannelTargetType, TargetType}; +use crate::{Target, TargetError}; +use rustfs_config::audit::{ + AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, + AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS, +}; +use rustfs_config::notify::{ + NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, + NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, + NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, + NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS, +}; +use rustfs_config::{ + AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR, + audit::{ + AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, + AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS, + }, +}; +use rustfs_ecstore::config::KVS; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::config::{ + build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, + build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config, + validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, + validate_webhook_config, +}; + +type BoxedTarget = Box + Send + Sync>; + +fn build_descriptor( + subsystem: &'static str, + request_validator: TargetRequestValidator, + target_type: &'static str, + valid_fields: &'static [&'static str], + validate_config: Validate, + create_target: Create, +) -> BuiltinTargetDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + Create: Fn(String, &KVS) -> Result, TargetError> + Send + Sync + 'static, + Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static, +{ + BuiltinTargetDescriptor::new( + subsystem, + request_validator, + TargetPluginDescriptor::new(target_type, valid_fields, validate_config, create_target), + ) +} + +fn build_admin_descriptor( + subsystem: &'static str, + request_validator: TargetRequestValidator, + target_type: &'static str, + valid_fields: &'static [&'static str], +) -> BuiltinTargetAdminDescriptor { + BuiltinTargetAdminDescriptor::new( + crate::manifest::builtin_target_manifest(target_type), + valid_fields, + TargetAdminMetadata::new(subsystem, request_validator), + ) +} + +pub fn builtin_audit_target_admin_descriptors() -> Vec { + vec![ + build_admin_descriptor( + AUDIT_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::AuditLog), + ChannelTargetType::Amqp.as_str(), + AUDIT_AMQP_KEYS, + ), + build_admin_descriptor( + AUDIT_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + ChannelTargetType::Webhook.as_str(), + AUDIT_WEBHOOK_KEYS, + ), + build_admin_descriptor( + AUDIT_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + ChannelTargetType::Mqtt.as_str(), + AUDIT_MQTT_KEYS, + ), + build_admin_descriptor( + AUDIT_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::AuditLog), + ChannelTargetType::Nats.as_str(), + AUDIT_NATS_KEYS, + ), + build_admin_descriptor( + AUDIT_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::AuditLog), + ChannelTargetType::Pulsar.as_str(), + AUDIT_PULSAR_KEYS, + ), + build_admin_descriptor( + AUDIT_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::AuditLog), + ChannelTargetType::Kafka.as_str(), + AUDIT_KAFKA_KEYS, + ), + build_admin_descriptor( + AUDIT_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: AUDIT_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::AuditLog, + }, + ChannelTargetType::Redis.as_str(), + AUDIT_REDIS_KEYS, + ), + build_admin_descriptor( + AUDIT_MYSQL_SUB_SYS, + TargetRequestValidator::MySql(TargetType::AuditLog), + ChannelTargetType::MySql.as_str(), + AUDIT_MYSQL_KEYS, + ), + build_admin_descriptor( + AUDIT_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::AuditLog), + ChannelTargetType::Postgres.as_str(), + AUDIT_POSTGRES_KEYS, + ), + ] +} + +pub fn builtin_audit_target_descriptors() -> Vec> +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + vec![ + build_descriptor( + AUDIT_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::AuditLog), + ChannelTargetType::Amqp.as_str(), + AUDIT_AMQP_KEYS, + |config| validate_amqp_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::amqp::AMQPTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + ChannelTargetType::Webhook.as_str(), + AUDIT_WEBHOOK_KEYS, + |config| validate_webhook_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::webhook::WebhookTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + ChannelTargetType::Mqtt.as_str(), + AUDIT_MQTT_KEYS, + validate_mqtt_config, + |id, config| { + let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::mqtt::MQTTTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::AuditLog), + ChannelTargetType::Nats.as_str(), + AUDIT_NATS_KEYS, + |config| validate_nats_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::nats::NATSTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::AuditLog), + ChannelTargetType::Pulsar.as_str(), + AUDIT_PULSAR_KEYS, + |config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::pulsar::PulsarTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::AuditLog), + ChannelTargetType::Kafka.as_str(), + AUDIT_KAFKA_KEYS, + |config| validate_kafka_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::kafka::KafkaTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: AUDIT_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::AuditLog, + }, + ChannelTargetType::Redis.as_str(), + AUDIT_REDIS_KEYS, + |config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL), + |id, config| { + let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::redis::RedisTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_MYSQL_SUB_SYS, + TargetRequestValidator::MySql(TargetType::AuditLog), + ChannelTargetType::MySql.as_str(), + AUDIT_MYSQL_KEYS, + |config| validate_mysql_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::mysql::MySqlTarget::::new(id, args)?)) + }, + ), + build_descriptor( + AUDIT_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::AuditLog), + ChannelTargetType::Postgres.as_str(), + AUDIT_POSTGRES_KEYS, + |config| validate_postgres_config(config, AUDIT_DEFAULT_DIR), + |id, config| { + let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?; + Ok(boxed_target(crate::target::postgres::PostgresTarget::::new(id, args)?)) + }, + ), + ] +} + +pub fn builtin_notify_target_admin_descriptors() -> Vec { + vec![ + build_admin_descriptor( + NOTIFY_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + ChannelTargetType::Webhook.as_str(), + NOTIFY_WEBHOOK_KEYS, + ), + build_admin_descriptor( + NOTIFY_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::NotifyEvent), + ChannelTargetType::Amqp.as_str(), + NOTIFY_AMQP_KEYS, + ), + build_admin_descriptor( + NOTIFY_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::NotifyEvent), + ChannelTargetType::Kafka.as_str(), + NOTIFY_KAFKA_KEYS, + ), + build_admin_descriptor( + NOTIFY_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + ChannelTargetType::Mqtt.as_str(), + NOTIFY_MQTT_KEYS, + ), + build_admin_descriptor( + NOTIFY_MYSQL_SUB_SYS, + TargetRequestValidator::MySql(TargetType::NotifyEvent), + ChannelTargetType::MySql.as_str(), + NOTIFY_MYSQL_KEYS, + ), + build_admin_descriptor( + NOTIFY_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::NotifyEvent), + ChannelTargetType::Nats.as_str(), + NOTIFY_NATS_KEYS, + ), + build_admin_descriptor( + NOTIFY_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::NotifyEvent), + ChannelTargetType::Postgres.as_str(), + NOTIFY_POSTGRES_KEYS, + ), + build_admin_descriptor( + NOTIFY_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::NotifyEvent, + }, + ChannelTargetType::Redis.as_str(), + NOTIFY_REDIS_KEYS, + ), + build_admin_descriptor( + NOTIFY_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::NotifyEvent), + ChannelTargetType::Pulsar.as_str(), + NOTIFY_PULSAR_KEYS, + ), + ] +} + +pub fn builtin_notify_target_descriptors() -> Vec> +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + vec![ + build_descriptor( + NOTIFY_WEBHOOK_SUB_SYS, + TargetRequestValidator::Webhook, + ChannelTargetType::Webhook.as_str(), + NOTIFY_WEBHOOK_KEYS, + |config| validate_webhook_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::webhook::WebhookTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_AMQP_SUB_SYS, + TargetRequestValidator::Amqp(TargetType::NotifyEvent), + ChannelTargetType::Amqp.as_str(), + NOTIFY_AMQP_KEYS, + |config| validate_amqp_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::amqp::AMQPTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_KAFKA_SUB_SYS, + TargetRequestValidator::Kafka(TargetType::NotifyEvent), + ChannelTargetType::Kafka.as_str(), + NOTIFY_KAFKA_KEYS, + |config| validate_kafka_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::kafka::KafkaTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_MQTT_SUB_SYS, + TargetRequestValidator::Mqtt, + ChannelTargetType::Mqtt.as_str(), + NOTIFY_MQTT_KEYS, + validate_mqtt_config, + |id, config| { + let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::mqtt::MQTTTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_MYSQL_SUB_SYS, + TargetRequestValidator::MySql(TargetType::NotifyEvent), + ChannelTargetType::MySql.as_str(), + NOTIFY_MYSQL_KEYS, + |config| validate_mysql_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::mysql::MySqlTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_NATS_SUB_SYS, + TargetRequestValidator::Nats(TargetType::NotifyEvent), + ChannelTargetType::Nats.as_str(), + NOTIFY_NATS_KEYS, + |config| validate_nats_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::nats::NATSTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_POSTGRES_SUB_SYS, + TargetRequestValidator::Postgres(TargetType::NotifyEvent), + ChannelTargetType::Postgres.as_str(), + NOTIFY_POSTGRES_KEYS, + |config| validate_postgres_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::postgres::PostgresTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_REDIS_SUB_SYS, + TargetRequestValidator::Redis { + default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL, + target_type: TargetType::NotifyEvent, + }, + ChannelTargetType::Redis.as_str(), + NOTIFY_REDIS_KEYS, + |config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL), + |id, config| { + let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::redis::RedisTarget::::new(id, args)?)) + }, + ), + build_descriptor( + NOTIFY_PULSAR_SUB_SYS, + TargetRequestValidator::Pulsar(TargetType::NotifyEvent), + ChannelTargetType::Pulsar.as_str(), + NOTIFY_PULSAR_KEYS, + |config| validate_pulsar_config(config, EVENT_DEFAULT_DIR), + |id, config| { + let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?; + Ok(boxed_target(crate::target::pulsar::PulsarTarget::::new(id, args)?)) + }, + ), + ] +} diff --git a/crates/targets/src/catalog/mod.rs b/crates/targets/src/catalog/mod.rs new file mode 100644 index 000000000..c3e46a7b8 --- /dev/null +++ b/crates/targets/src/catalog/mod.rs @@ -0,0 +1,105 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod builtin; + +use crate::control_plane::external_target_plugin_installation; +use crate::domain::TargetDomain; +use crate::manifest::{ + TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind, + TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginRuntimeTransport, + installable_target_marketplace_manifest, +}; +use crate::runtime::sidecar::SidecarPluginRuntime; +use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExampleInstallableTargetPlugin { + pub manifest: TargetPluginMarketplaceManifest, + pub installation: crate::TargetPluginInstallation, + pub runtime: SidecarPluginRuntime, + pub valid_fields: Vec, +} + +pub fn example_external_webhook_plugin() -> ExampleInstallableTargetPlugin { + let base = TargetPluginManifest { + plugin_id: "external:webhook-sidecar", + display_name: "Webhook Sidecar", + provider: "rustfs-labs", + version: "1.0.0", + target_type: "webhook", + supported_domains: &[TargetDomain::Notify], + secret_fields: &["auth_token"], + }; + let manifest = installable_target_marketplace_manifest( + base, + TargetPluginEntrypointKind::Sidecar, + TargetPluginExternalRuntimeContract { + protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION, + transport: TargetPluginRuntimeTransport::Grpc, + }, + TargetPluginDistributionManifest { + artifacts: &[TargetPluginArtifactManifest { + artifact_id: "sidecar-linux-amd64", + target_triple: "x86_64-unknown-linux-gnu", + download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst", + digest_sha256: "0123456789abcdef0123456789abcdef", + size_bytes: 8192, + }], + }, + ); + + let handshake = SidecarHandshake { + protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(), + plugin_id: base.plugin_id.to_string(), + plugin_version: base.version.to_string(), + supported_domains: vec![TargetDomain::Notify], + capabilities: vec![ + SidecarPluginCapability::HealthCheck, + SidecarPluginCapability::SendEvent, + SidecarPluginCapability::Shutdown, + ], + }; + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", handshake); + runtime + .enable(base.plugin_id, TargetDomain::Notify) + .expect("example sidecar plugin handshake should validate"); + + ExampleInstallableTargetPlugin { + manifest, + installation: external_target_plugin_installation( + base.version, + "0123456789abcdef0123456789abcdef", + "sidecar-linux-amd64", + Some("2026-05-13T20:00:00Z".to_string()), + ), + runtime, + valid_fields: vec!["endpoint".to_string(), "auth_token".to_string()], + } +} + +#[cfg(test)] +mod tests { + use super::example_external_webhook_plugin; + + #[test] + fn example_external_plugin_exposes_installation_and_runtime_metadata() { + let example = example_external_webhook_plugin(); + + assert_eq!(example.manifest.plugin_id, "external:webhook-sidecar"); + assert_eq!(example.installation.install_state, crate::TargetPluginInstallState::Installed); + assert!(example.runtime.healthy); + assert_eq!(example.valid_fields, vec!["endpoint".to_string(), "auth_token".to_string()]); + } +} diff --git a/crates/targets/src/config/instance.rs b/crates/targets/src/config/instance.rs new file mode 100644 index 000000000..85b5f49ce --- /dev/null +++ b/crates/targets/src/config/instance.rs @@ -0,0 +1,346 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::loader::collect_merged_target_configs_from_env; +use crate::domain::TargetDomain; +use rustfs_ecstore::config::{Config, KVS}; +use std::collections::HashSet; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginInstanceCompatDescriptor<'a> { + pub domain: TargetDomain, + pub plugin_id: &'a str, + pub target_type: &'a str, + pub subsystem: &'a str, + pub route_prefix: &'a str, + pub valid_fields: &'a [&'a str], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetInstanceSourceClass { + Config, + Env, + Mixed, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct TargetInstanceSourceHints { + pub has_file_default: bool, + pub has_file_instance: bool, + pub has_env_default: bool, + pub has_env_instance: bool, +} + +impl TargetInstanceSourceHints { + #[inline] + pub fn has_config_source(self) -> bool { + self.has_file_default || self.has_file_instance + } + + #[inline] + pub fn has_env_source(self) -> bool { + self.has_env_default || self.has_env_instance + } + + #[inline] + pub fn classification(self) -> TargetInstanceSourceClass { + match (self.has_config_source(), self.has_env_source()) { + (true, true) => TargetInstanceSourceClass::Mixed, + (true, false) => TargetInstanceSourceClass::Config, + (false, true) => TargetInstanceSourceClass::Env, + (false, false) => TargetInstanceSourceClass::Config, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TargetPluginInstanceRecord { + pub domain: TargetDomain, + pub plugin_id: String, + pub target_type: String, + pub subsystem: String, + pub instance_id: String, + pub enabled: bool, + pub source_hints: TargetInstanceSourceHints, + pub effective_config: KVS, +} + +pub type LegacyTargetInstanceDescriptor<'a> = TargetPluginInstanceCompatDescriptor<'a>; +pub type TargetPluginInstance = TargetPluginInstanceRecord; + +pub fn normalize_target_plugin_instances( + config: &Config, + descriptor: &TargetPluginInstanceCompatDescriptor<'_>, +) -> Vec { + normalize_target_plugin_instances_from_env(config, descriptor, std::env::vars()) +} + +pub fn normalize_target_plugin_instances_from_env( + config: &Config, + descriptor: &TargetPluginInstanceCompatDescriptor<'_>, + env_vars: I, +) -> Vec +where + I: IntoIterator, +{ + let valid_fields = descriptor + .valid_fields + .iter() + .map(|field| (*field).to_string()) + .collect::>(); + + collect_merged_target_configs_from_env( + config, + descriptor.subsystem, + descriptor.route_prefix, + descriptor.target_type, + &valid_fields, + env_vars, + ) + .into_iter() + .map(|record| TargetPluginInstanceRecord { + domain: descriptor.domain, + plugin_id: descriptor.plugin_id.to_string(), + target_type: descriptor.target_type.to_string(), + subsystem: descriptor.subsystem.to_string(), + instance_id: record.instance_id, + enabled: record.enabled, + source_hints: TargetInstanceSourceHints { + has_file_default: record.has_file_default, + has_file_instance: record.has_file_instance, + has_env_default: record.has_env_default, + has_env_instance: record.has_env_instance, + }, + effective_config: record.effective_config, + }) + .collect() +} + +pub fn normalize_legacy_target_instances( + config: &Config, + descriptor: &LegacyTargetInstanceDescriptor<'_>, +) -> Vec { + normalize_target_plugin_instances(config, descriptor) +} + +pub fn normalize_legacy_target_instances_from_env( + config: &Config, + descriptor: &LegacyTargetInstanceDescriptor<'_>, + env_vars: I, +) -> Vec +where + I: IntoIterator, +{ + normalize_target_plugin_instances_from_env(config, descriptor, env_vars) +} + +#[cfg(test)] +mod tests { + use super::{ + TargetInstanceSourceClass, TargetPluginInstanceCompatDescriptor, normalize_legacy_target_instances_from_env, + normalize_target_plugin_instances_from_env, + }; + use crate::domain::TargetDomain; + use crate::manifest::builtin_target_manifest; + use rustfs_config::audit::{AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; + use rustfs_config::notify::{NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS}; + use rustfs_config::{ENABLE_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_LIMIT}; + use rustfs_ecstore::config::{Config, KVS}; + use std::collections::HashMap; + + fn notify_webhook_descriptor() -> TargetPluginInstanceCompatDescriptor<'static> { + TargetPluginInstanceCompatDescriptor { + domain: TargetDomain::Notify, + plugin_id: builtin_target_manifest("webhook").plugin_id, + target_type: "webhook", + subsystem: NOTIFY_WEBHOOK_SUB_SYS, + route_prefix: NOTIFY_ROUTE_PREFIX, + valid_fields: NOTIFY_WEBHOOK_KEYS, + } + } + + fn audit_webhook_descriptor() -> TargetPluginInstanceCompatDescriptor<'static> { + TargetPluginInstanceCompatDescriptor { + domain: TargetDomain::Audit, + plugin_id: builtin_target_manifest("webhook").plugin_id, + target_type: "webhook", + subsystem: AUDIT_WEBHOOK_SUB_SYS, + route_prefix: AUDIT_ROUTE_PREFIX, + valid_fields: AUDIT_WEBHOOK_KEYS, + } + } + + #[test] + fn normalize_notify_instances_merges_file_and_env_sources() { + let mut cfg = Config(HashMap::new()); + let mut subsystem = HashMap::new(); + + let mut default_kvs = KVS::new(); + default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string()); + default_kvs.insert(WEBHOOK_QUEUE_LIMIT.to_string(), "10".to_string()); + subsystem.insert("_".to_string(), default_kvs); + + let mut primary = KVS::new(); + primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/primary".to_string()); + subsystem.insert("primary".to_string(), primary); + + cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem); + + let instances = normalize_legacy_target_instances_from_env( + &cfg, + ¬ify_webhook_descriptor(), + vec![ + ("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "42".to_string()), + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_SECONDARY".to_string(), "on".to_string()), + ( + "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(), + "https://example.com/secondary".to_string(), + ), + ], + ); + + assert_eq!(instances.len(), 2); + + let primary = instances + .iter() + .find(|instance| instance.instance_id == "primary") + .expect("primary notify instance should be normalized"); + assert_eq!(primary.domain, TargetDomain::Notify); + assert_eq!(primary.plugin_id, "builtin:webhook"); + assert!(primary.enabled); + assert_eq!(primary.effective_config.lookup(WEBHOOK_QUEUE_LIMIT).as_deref(), Some("42")); + assert_eq!( + primary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(), + Some("https://example.com/primary") + ); + assert_eq!(primary.source_hints.classification(), TargetInstanceSourceClass::Mixed); + assert!(primary.source_hints.has_file_default); + assert!(primary.source_hints.has_file_instance); + assert!(primary.source_hints.has_env_default); + assert!(!primary.source_hints.has_env_instance); + + let secondary = instances + .iter() + .find(|instance| instance.instance_id == "secondary") + .expect("secondary env notify instance should be normalized"); + assert!(secondary.enabled); + assert_eq!( + secondary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(), + Some("https://example.com/secondary") + ); + assert_eq!(secondary.effective_config.lookup(WEBHOOK_QUEUE_LIMIT).as_deref(), Some("42")); + assert_eq!(secondary.source_hints.classification(), TargetInstanceSourceClass::Mixed); + assert!(secondary.source_hints.has_file_default); + assert!(!secondary.source_hints.has_file_instance); + assert!(secondary.source_hints.has_env_default); + assert!(secondary.source_hints.has_env_instance); + } + + #[test] + fn normalize_audit_instances_preserves_domain_and_subsystem() { + let mut cfg = Config(HashMap::new()); + let mut subsystem = HashMap::new(); + + let mut default_kvs = KVS::new(); + default_kvs.insert(ENABLE_KEY.to_string(), "off".to_string()); + subsystem.insert("_".to_string(), default_kvs); + + let mut primary = KVS::new(); + primary.insert(ENABLE_KEY.to_string(), "on".to_string()); + primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/audit".to_string()); + subsystem.insert("primary".to_string(), primary); + + cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), subsystem); + + let instances = normalize_legacy_target_instances_from_env(&cfg, &audit_webhook_descriptor(), Vec::new()); + + assert_eq!(instances.len(), 1); + let primary = &instances[0]; + assert_eq!(primary.domain, TargetDomain::Audit); + assert_eq!(primary.target_type, "webhook"); + assert_eq!(primary.subsystem, AUDIT_WEBHOOK_SUB_SYS); + assert_eq!(primary.instance_id, "primary"); + assert!(primary.enabled); + assert_eq!( + primary.effective_config.lookup(WEBHOOK_ENDPOINT).as_deref(), + Some("https://example.com/audit") + ); + assert_eq!(primary.source_hints.classification(), TargetInstanceSourceClass::Config); + } + + #[test] + fn normalize_instances_keeps_disabled_records() { + let cfg = Config(HashMap::new()); + + let instances = normalize_legacy_target_instances_from_env( + &cfg, + ¬ify_webhook_descriptor(), + vec![ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_DISABLED".to_string(), "off".to_string()), + ( + "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_DISABLED".to_string(), + "https://example.com/disabled".to_string(), + ), + ], + ); + + assert_eq!(instances.len(), 1); + let disabled = &instances[0]; + assert_eq!(disabled.instance_id, "disabled"); + assert!(!disabled.enabled); + assert_eq!(disabled.source_hints.classification(), TargetInstanceSourceClass::Env); + assert!(disabled.source_hints.has_env_instance); + } + + #[test] + fn normalize_instances_excludes_default_only_entries() { + let mut cfg = Config(HashMap::new()); + let mut subsystem = HashMap::new(); + + let mut default_kvs = KVS::new(); + default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string()); + default_kvs.insert(WEBHOOK_QUEUE_LIMIT.to_string(), "99".to_string()); + subsystem.insert("_".to_string(), default_kvs); + + cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem); + + let instances = normalize_legacy_target_instances_from_env( + &cfg, + ¬ify_webhook_descriptor(), + vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "100".to_string())], + ); + + assert!(instances.is_empty()); + } + + #[test] + fn compatibility_wrapper_matches_canonical_instance_model() { + let mut cfg = Config(HashMap::new()); + let mut subsystem = HashMap::new(); + + let mut primary = KVS::new(); + primary.insert(ENABLE_KEY.to_string(), "on".to_string()); + primary.insert(WEBHOOK_ENDPOINT.to_string(), "https://example.com/primary".to_string()); + subsystem.insert("primary".to_string(), primary); + cfg.0.insert(NOTIFY_WEBHOOK_SUB_SYS.to_string(), subsystem); + + let descriptor = notify_webhook_descriptor(); + let env = vec![("RUSTFS_NOTIFY_WEBHOOK_QUEUE_LIMIT".to_string(), "7".to_string())]; + + let canonical = normalize_target_plugin_instances_from_env(&cfg, &descriptor, env.clone()); + let compatibility = normalize_legacy_target_instances_from_env(&cfg, &descriptor, env); + + assert_eq!(canonical, compatibility); + } +} diff --git a/crates/targets/src/config/loader.rs b/crates/targets/src/config/loader.rs index ae232f06f..2fd5cbd50 100644 --- a/crates/targets/src/config/loader.rs +++ b/crates/targets/src/config/loader.rs @@ -13,10 +13,9 @@ // limitations under the License. use super::common::{is_target_enabled, split_env_field_and_instance}; -use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, EnableState}; +use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX}; use rustfs_ecstore::config::{Config, KVS}; use std::collections::{HashMap, HashSet}; -use std::str::FromStr; use tracing::{debug, warn}; pub fn collect_target_configs( @@ -72,6 +71,17 @@ fn redacted_target_config(config: &KVS) -> Vec<(String, String)> { .collect() } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct MergedTargetConfigRecord { + pub instance_id: String, + pub effective_config: KVS, + pub enabled: bool, + pub has_file_default: bool, + pub has_file_instance: bool, + pub has_env_default: bool, + pub has_env_instance: bool, +} + pub fn collect_env_target_instance_ids(route_prefix: &str, target_type: &str, valid_fields: &HashSet) -> HashSet { collect_env_target_instance_ids_from_env(route_prefix, target_type, valid_fields, std::env::vars()) } @@ -113,25 +123,40 @@ pub fn collect_target_configs_from_env( where I: IntoIterator, { - let all_env: Vec<(String, String)> = env_vars.into_iter().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect(); - let section_name = format!("{route_prefix}{target_type}").to_lowercase(); - let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default(); - let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default(); + collect_merged_target_configs_from_env( + config, + &format!("{route_prefix}{target_type}").to_lowercase(), + route_prefix, + target_type, + valid_fields, + env_vars, + ) + .into_iter() + .filter(|record| record.enabled) + .map(|record| (record.instance_id, record.effective_config)) + .collect() +} + +pub(crate) fn collect_merged_target_configs_from_env( + config: &Config, + section_name: &str, + route_prefix: &str, + target_type: &str, + valid_fields: &HashSet, + env_vars: I, +) -> Vec +where + I: IntoIterator, +{ + let all_env: Vec<(String, String)> = env_vars.into_iter().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect(); + let file_configs = config.0.get(section_name).cloned().unwrap_or_default(); + let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default(); + let has_file_default = file_configs.contains_key(DEFAULT_DELIMITER); - let enable_prefix = - format!("{ENV_PREFIX}{route_prefix}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}").to_uppercase(); let env_prefix = format!("{ENV_PREFIX}{route_prefix}{target_type}{DEFAULT_DELIMITER}").to_uppercase(); - let mut instance_ids_from_env = HashSet::new(); let mut env_overrides: HashMap = HashMap::new(); for (key, value) in &all_env { - if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false) - && let Some(id) = key.strip_prefix(&enable_prefix) - && !id.is_empty() - { - instance_ids_from_env.insert(id.to_lowercase()); - } - let Some(rest) = key.strip_prefix(&env_prefix) else { continue; }; @@ -158,6 +183,7 @@ where } let mut effective_default = default_cfg; + let has_env_default = env_overrides.contains_key(DEFAULT_DELIMITER); if let Some(default_env_cfg) = env_overrides.remove(DEFAULT_DELIMITER) { effective_default.extend(default_env_cfg); } @@ -167,16 +193,25 @@ where .filter(|key| key.as_str() != DEFAULT_DELIMITER) .cloned() .collect(); - all_instance_ids.extend(instance_ids_from_env); + all_instance_ids.extend( + env_overrides + .iter() + .filter(|(instance_id, env_cfg)| { + instance_id.as_str() != DEFAULT_DELIMITER && env_cfg.lookup(rustfs_config::ENABLE_KEY).is_some() + }) + .map(|(instance_id, _)| instance_id.clone()), + ); all_instance_ids.sort(); all_instance_ids.dedup(); let mut merged_configs = Vec::new(); for id in all_instance_ids { let mut merged_config = effective_default.clone(); + let has_file_instance = file_configs.contains_key(&id); if let Some(file_instance_cfg) = file_configs.get(&id) { merged_config.extend(file_instance_cfg.clone()); } + let has_env_instance = env_overrides.contains_key(&id); if let Some(env_instance_cfg) = env_overrides.get(&id) { merged_config.extend(env_instance_cfg.clone()); } @@ -185,9 +220,15 @@ where let redacted_config = redacted_target_config(&merged_config); debug!(instance_id = %id, ?redacted_config, "Merged target configuration"); } - if is_target_enabled(&merged_config) { - merged_configs.push((id, merged_config)); - } + merged_configs.push(MergedTargetConfigRecord { + instance_id: id, + enabled: is_target_enabled(&merged_config), + effective_config: merged_config, + has_file_default, + has_file_instance, + has_env_default, + has_env_instance, + }); } merged_configs @@ -273,6 +314,29 @@ mod tests { assert_eq!(configs[0].1.lookup(WEBHOOK_ENDPOINT).as_deref(), Some("https://example.com/from-env")); } + #[test] + fn collect_target_configs_does_not_materialize_env_only_instance_without_enable_flag() { + let mut cfg = Config(HashMap::new()); + let mut subsystem = HashMap::new(); + let mut default_kvs = KVS::new(); + default_kvs.insert(ENABLE_KEY.to_string(), "on".to_string()); + subsystem.insert("_".to_string(), default_kvs); + cfg.0.insert("notify_webhook".to_string(), subsystem); + + let configs = collect_target_configs_from_env( + &cfg, + NOTIFY_ROUTE_PREFIX, + "webhook", + &HashSet::from([ENABLE_KEY.to_string(), WEBHOOK_ENDPOINT.to_string()]), + vec![( + "RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SECONDARY".to_string(), + "https://example.com/secondary".to_string(), + )], + ); + + assert!(configs.is_empty()); + } + #[test] fn collect_env_target_instance_ids_handles_keys_with_internal_underscores() { let ids = collect_env_target_instance_ids_from_env( diff --git a/crates/targets/src/config/mod.rs b/crates/targets/src/config/mod.rs index 6386f7939..0c255441d 100644 --- a/crates/targets/src/config/mod.rs +++ b/crates/targets/src/config/mod.rs @@ -13,9 +13,15 @@ // limitations under the License. mod common; +mod instance; mod loader; mod target_args; +pub use instance::{ + LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance, + TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances, + normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env, +}; pub use loader::{ collect_env_target_instance_ids, collect_env_target_instance_ids_from_env, collect_target_configs, collect_target_configs_from_env, diff --git a/crates/targets/src/control_plane.rs b/crates/targets/src/control_plane.rs new file mode 100644 index 000000000..f53bff0e4 --- /dev/null +++ b/crates/targets/src/control_plane.rs @@ -0,0 +1,424 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::manifest::{ + TargetPluginDistributionManifest, TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginRuntimeTransport, +}; +use crate::runtime::sidecar_protocol::SIDECAR_RUNTIME_PROTOCOL_VERSION; +use serde::{Deserialize, Serialize}; +use url::Url; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TargetPluginInstallState { + NotInstalled, + Installed, + InstallFailed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TargetPluginEnableState { + Enabled, + Disabled, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TargetPluginRuntimeState { + Running, + Offline, + Error, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct TargetPluginRevision { + pub version: String, + pub digest_sha256: Option, + pub source: String, + pub installed_at: Option, + pub artifact_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct TargetPluginInstallation { + pub install_state: TargetPluginInstallState, + pub current_revision: Option, + pub previous_revision: Option, + pub validation_error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct TargetPluginOperationalState { + pub install_state: TargetPluginInstallState, + pub enable_state: TargetPluginEnableState, + pub runtime_state: TargetPluginRuntimeState, +} + +pub fn builtin_target_plugin_installation(manifest: &TargetPluginManifest) -> TargetPluginInstallation { + TargetPluginInstallation { + install_state: TargetPluginInstallState::Installed, + current_revision: Some(TargetPluginRevision { + version: manifest.version.to_string(), + digest_sha256: None, + source: "builtin".to_string(), + installed_at: None, + artifact_id: None, + }), + previous_revision: None, + validation_error: None, + } +} + +pub fn external_target_plugin_installation( + version: impl Into, + digest_sha256: impl Into, + artifact_id: impl Into, + installed_at: Option, +) -> TargetPluginInstallation { + TargetPluginInstallation { + install_state: TargetPluginInstallState::Installed, + current_revision: Some(TargetPluginRevision { + version: version.into(), + digest_sha256: Some(digest_sha256.into()), + source: "external".to_string(), + installed_at, + artifact_id: Some(artifact_id.into()), + }), + previous_revision: None, + validation_error: None, + } +} + +pub fn failed_external_target_plugin_installation( + version: impl Into, + artifact_id: impl Into, + validation_error: impl Into, +) -> TargetPluginInstallation { + TargetPluginInstallation { + install_state: TargetPluginInstallState::InstallFailed, + current_revision: Some(TargetPluginRevision { + version: version.into(), + digest_sha256: None, + source: "external".to_string(), + installed_at: None, + artifact_id: Some(artifact_id.into()), + }), + previous_revision: None, + validation_error: Some(validation_error.into()), + } +} + +pub fn rollback_target_plugin_installation( + current: TargetPluginRevision, + previous: TargetPluginRevision, +) -> TargetPluginInstallation { + TargetPluginInstallation { + install_state: TargetPluginInstallState::Installed, + current_revision: Some(previous), + previous_revision: Some(current), + validation_error: None, + } +} + +pub fn builtin_target_plugin_operational_state( + enabled: bool, + runtime_state: TargetPluginRuntimeState, +) -> TargetPluginOperationalState { + TargetPluginOperationalState { + install_state: TargetPluginInstallState::Installed, + enable_state: if enabled { + TargetPluginEnableState::Enabled + } else { + TargetPluginEnableState::Disabled + }, + runtime_state, + } +} + +pub fn runtime_state_from_status_label(status: &str) -> TargetPluginRuntimeState { + if status.eq_ignore_ascii_case("online") { + TargetPluginRuntimeState::Running + } else if status.eq_ignore_ascii_case("offline") { + TargetPluginRuntimeState::Offline + } else if status.eq_ignore_ascii_case("error") { + TargetPluginRuntimeState::Error + } else { + TargetPluginRuntimeState::Unknown + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TargetPluginInstallPolicy { + pub allowed_providers: Vec, + pub allowed_download_hosts: Vec, + pub require_https: bool, + pub require_signature: bool, +} + +impl Default for TargetPluginInstallPolicy { + fn default() -> Self { + Self { + allowed_providers: vec!["rustfs".to_string(), "rustfs-labs".to_string()], + allowed_download_hosts: vec!["plugins.example.test".to_string()], + require_https: true, + require_signature: false, + } + } +} + +pub fn validate_external_plugin_installation( + manifest: &TargetPluginManifest, + runtime_contract: &TargetPluginExternalRuntimeContract, + distribution: Option, + policy: &TargetPluginInstallPolicy, +) -> Result<(), String> { + if !policy.allowed_providers.iter().any(|provider| provider == manifest.provider) { + return Err(format!("provider {} is not allowed by install policy", manifest.provider)); + } + + if runtime_contract.transport == TargetPluginRuntimeTransport::Grpc + && runtime_contract.protocol_version != SIDECAR_RUNTIME_PROTOCOL_VERSION + { + return Err(format!( + "sidecar runtime protocol mismatch: expected {}, got {}", + SIDECAR_RUNTIME_PROTOCOL_VERSION, runtime_contract.protocol_version + )); + } + + if policy.require_signature { + return Err( + "signature verification is required by install policy but manifests do not expose signatures yet".to_string(), + ); + } + + let distribution = distribution.ok_or_else(|| "external plugin is missing distribution metadata".to_string())?; + if distribution.artifacts.is_empty() { + return Err("external plugin distribution has no artifacts".to_string()); + } + + for artifact in distribution.artifacts { + let parsed_uri = Url::parse(artifact.download_uri) + .map_err(|err| format!("invalid artifact download uri {}: {}", artifact.download_uri, err))?; + if policy.require_https && parsed_uri.scheme() != "https" { + return Err(format!( + "artifact {} must use https download uri, got {}", + artifact.artifact_id, artifact.download_uri + )); + } + let host = parsed_uri + .host_str() + .ok_or_else(|| format!("artifact {} download uri has no host", artifact.artifact_id))?; + if !policy.allowed_download_hosts.iter().any(|allowed| allowed == host) { + return Err(format!("artifact {} download host {} is not allowed", artifact.artifact_id, host)); + } + if artifact.size_bytes == 0 { + return Err(format!("artifact {} must declare a non-zero size", artifact.artifact_id)); + } + if artifact.digest_sha256.len() < 16 || !artifact.digest_sha256.chars().all(|ch| ch.is_ascii_hexdigit()) { + return Err(format!( + "artifact {} has invalid digest_sha256 {}", + artifact.artifact_id, artifact.digest_sha256 + )); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + TargetPluginEnableState, TargetPluginInstallPolicy, TargetPluginInstallState, TargetPluginRevision, + TargetPluginRuntimeState, builtin_target_plugin_installation, builtin_target_plugin_operational_state, + external_target_plugin_installation, failed_external_target_plugin_installation, rollback_target_plugin_installation, + runtime_state_from_status_label, validate_external_plugin_installation, + }; + use crate::manifest::{ + TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginExternalRuntimeContract, + TargetPluginManifest, TargetPluginRuntimeTransport, builtin_target_manifest, + }; + + #[test] + fn builtin_installation_maps_to_virtual_installed_revision() { + let installation = builtin_target_plugin_installation(&builtin_target_manifest("webhook")); + + assert_eq!(installation.install_state, TargetPluginInstallState::Installed); + assert_eq!( + installation + .current_revision + .as_ref() + .expect("builtin installation should expose current revision") + .source, + "builtin" + ); + assert_eq!( + installation + .current_revision + .as_ref() + .expect("builtin installation should expose current revision") + .artifact_id, + None + ); + assert!(installation.previous_revision.is_none()); + assert_eq!(installation.validation_error, None); + } + + #[test] + fn builtin_operational_state_tracks_enablement_and_runtime() { + let enabled = builtin_target_plugin_operational_state(true, TargetPluginRuntimeState::Running); + let disabled = builtin_target_plugin_operational_state(false, TargetPluginRuntimeState::Offline); + + assert_eq!(enabled.install_state, TargetPluginInstallState::Installed); + assert_eq!(enabled.enable_state, TargetPluginEnableState::Enabled); + assert_eq!(enabled.runtime_state, TargetPluginRuntimeState::Running); + + assert_eq!(disabled.enable_state, TargetPluginEnableState::Disabled); + assert_eq!(disabled.runtime_state, TargetPluginRuntimeState::Offline); + } + + #[test] + fn runtime_state_from_status_maps_known_labels() { + assert_eq!(runtime_state_from_status_label("online"), TargetPluginRuntimeState::Running); + assert_eq!(runtime_state_from_status_label("offline"), TargetPluginRuntimeState::Offline); + assert_eq!(runtime_state_from_status_label("error"), TargetPluginRuntimeState::Error); + assert_eq!(runtime_state_from_status_label("unexpected"), TargetPluginRuntimeState::Unknown); + } + + #[test] + fn external_installation_captures_revision_metadata() { + let installation = external_target_plugin_installation( + "1.2.3", + "0123456789abcdef", + "sidecar-linux-amd64", + Some("2026-05-13T12:00:00Z".to_string()), + ); + + let revision = installation + .current_revision + .as_ref() + .expect("external installation should expose current revision"); + assert_eq!(installation.install_state, TargetPluginInstallState::Installed); + assert_eq!(revision.source, "external"); + assert_eq!(revision.digest_sha256.as_deref(), Some("0123456789abcdef")); + assert_eq!(revision.artifact_id.as_deref(), Some("sidecar-linux-amd64")); + assert_eq!(installation.validation_error, None); + } + + #[test] + fn rollback_swaps_current_and_previous_revisions() { + let current = TargetPluginRevision { + version: "2.0.0".to_string(), + digest_sha256: Some("new-digest".to_string()), + source: "external".to_string(), + installed_at: Some("2026-05-13T12:05:00Z".to_string()), + artifact_id: Some("sidecar-linux-amd64-v2".to_string()), + }; + let previous = TargetPluginRevision { + version: "1.9.0".to_string(), + digest_sha256: Some("old-digest".to_string()), + source: "external".to_string(), + installed_at: Some("2026-05-13T11:55:00Z".to_string()), + artifact_id: Some("sidecar-linux-amd64-v1".to_string()), + }; + + let installation = rollback_target_plugin_installation(current.clone(), previous.clone()); + + assert_eq!(installation.current_revision, Some(previous)); + assert_eq!(installation.previous_revision, Some(current)); + assert_eq!(installation.validation_error, None); + } + + #[test] + fn failed_external_installation_preserves_error_context() { + let installation = + failed_external_target_plugin_installation("1.2.3", "sidecar-linux-amd64", "digest mismatch during install"); + + assert_eq!(installation.install_state, TargetPluginInstallState::InstallFailed); + assert_eq!(installation.validation_error.as_deref(), Some("digest mismatch during install")); + } + + #[test] + fn validate_external_installation_accepts_allowed_https_artifact() { + let manifest = TargetPluginManifest { + plugin_id: "external:webhook-sidecar", + display_name: "Webhook Sidecar", + provider: "rustfs-labs", + version: "1.0.0", + target_type: "webhook", + supported_domains: &[], + secret_fields: &[], + }; + let distribution = TargetPluginDistributionManifest { + artifacts: &[TargetPluginArtifactManifest { + artifact_id: "sidecar-linux-amd64", + target_triple: "x86_64-unknown-linux-gnu", + download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst", + digest_sha256: "0123456789abcdef0123456789abcdef", + size_bytes: 8192, + }], + }; + let policy = TargetPluginInstallPolicy::default(); + + let result = validate_external_plugin_installation( + &manifest, + &TargetPluginExternalRuntimeContract { + protocol_version: crate::SIDECAR_RUNTIME_PROTOCOL_VERSION, + transport: TargetPluginRuntimeTransport::Grpc, + }, + Some(distribution), + &policy, + ); + + assert!(result.is_ok()); + } + + #[test] + fn validate_external_installation_rejects_disallowed_provider() { + let manifest = TargetPluginManifest { + plugin_id: "external:webhook-sidecar", + display_name: "Webhook Sidecar", + provider: "unknown-vendor", + version: "1.0.0", + target_type: "webhook", + supported_domains: &[], + secret_fields: &[], + }; + let policy = TargetPluginInstallPolicy::default(); + + let result = validate_external_plugin_installation( + &manifest, + &TargetPluginExternalRuntimeContract { + protocol_version: crate::SIDECAR_RUNTIME_PROTOCOL_VERSION, + transport: TargetPluginRuntimeTransport::Grpc, + }, + Some(TargetPluginDistributionManifest { + artifacts: &[TargetPluginArtifactManifest { + artifact_id: "sidecar-linux-amd64", + target_triple: "x86_64-unknown-linux-gnu", + download_uri: "https://plugins.example.test/webhook-sidecar.tar.zst", + digest_sha256: "0123456789abcdef0123456789abcdef", + size_bytes: 8192, + }], + }), + &policy, + ); + + assert!(result.is_err()); + } +} diff --git a/crates/targets/src/domain.rs b/crates/targets/src/domain.rs new file mode 100644 index 000000000..d0b19aebb --- /dev/null +++ b/crates/targets/src/domain.rs @@ -0,0 +1,43 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::target::TargetType; +use serde::{Deserialize, Serialize}; + +/// Logical target domains supported by RustFS target plugins. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TargetDomain { + Notify, + Audit, +} + +impl TargetDomain { + #[inline] + pub fn runtime_target_type(self) -> TargetType { + match self { + TargetDomain::Notify => TargetType::NotifyEvent, + TargetDomain::Audit => TargetType::AuditLog, + } + } +} + +impl From for TargetDomain { + fn from(value: TargetType) -> Self { + match value { + TargetType::NotifyEvent => TargetDomain::Notify, + TargetType::AuditLog => TargetDomain::Audit, + } + } +} diff --git a/crates/targets/src/lib.rs b/crates/targets/src/lib.rs index a2c407e98..a173816fc 100644 --- a/crates/targets/src/lib.rs +++ b/crates/targets/src/lib.rs @@ -13,10 +13,15 @@ // limitations under the License. pub mod arn; +pub mod catalog; mod check; pub mod config; +pub mod control_plane; +pub mod domain; pub mod error; +pub mod manifest; pub mod plugin; +pub mod runtime; pub mod store; pub mod sys; pub mod target; @@ -26,8 +31,36 @@ pub use check::{ check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, }; +pub use config::{ + LegacyTargetInstanceDescriptor, TargetInstanceSourceClass, TargetInstanceSourceHints, TargetPluginInstance, + TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, normalize_legacy_target_instances, + normalize_legacy_target_instances_from_env, normalize_target_plugin_instances, normalize_target_plugin_instances_from_env, +}; +pub use control_plane::{ + TargetPluginEnableState, TargetPluginInstallState, TargetPluginInstallation, TargetPluginOperationalState, + TargetPluginRevision, TargetPluginRuntimeState, builtin_target_plugin_installation, builtin_target_plugin_operational_state, + external_target_plugin_installation, rollback_target_plugin_installation, runtime_state_from_status_label, +}; +pub use domain::TargetDomain; pub use error::{StoreError, TargetError}; -pub use plugin::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetPluginRegistry, TargetRequestValidator, boxed_target}; +pub use manifest::{ + TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind, + TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginPackaging, + TargetPluginRuntimeTransport, builtin_target_marketplace_manifest, installable_target_marketplace_manifest, +}; +pub use plugin::{ + BuiltinTargetAdminDescriptor, BuiltinTargetDescriptor, TargetAdminMetadata, TargetPluginDescriptor, TargetPluginRegistry, + TargetRequestValidator, boxed_target, +}; +pub use runtime::{ + ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, + RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay, + adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter}, + init_target_and_optionally_start_replay, + sidecar::SidecarPluginRuntime, + sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability}, + start_replay_worker, +}; pub use rustfs_s3_common::EventName; use serde::{Deserialize, Serialize}; pub use sys::user_agent::*; diff --git a/crates/targets/src/manifest.rs b/crates/targets/src/manifest.rs new file mode 100644 index 000000000..d858b1092 --- /dev/null +++ b/crates/targets/src/manifest.rs @@ -0,0 +1,314 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::domain::TargetDomain; +use rustfs_config::{ + AMQP_PASSWORD, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, MQTT_PASSWORD, + MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MYSQL_DSN_STRING, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, + NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TOKEN, POSTGRES_DSN_STRING, + POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, REDIS_PASSWORD, REDIS_TLS_CLIENT_CERT, + REDIS_TLS_CLIENT_KEY, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, +}; + +/// Shared plugin manifest metadata for a target implementation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginManifest { + pub plugin_id: &'static str, + pub display_name: &'static str, + pub provider: &'static str, + pub version: &'static str, + pub target_type: &'static str, + pub supported_domains: &'static [TargetDomain], + pub secret_fields: &'static [&'static str], +} + +/// Declares how a plugin is packaged relative to the RustFS process boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetPluginPackaging { + Builtin, + External, +} + +/// Declares what kind of entrypoint a plugin would use when instantiated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetPluginEntrypointKind { + Builtin, + Sidecar, + Wasm, +} + +/// Declares the transport boundary RustFS would use to communicate with a +/// plugin runtime without committing to any concrete loader implementation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetPluginRuntimeTransport { + InProcess, + Grpc, + WasmHost, +} + +/// Declarative external runtime contract for future installable plugins. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginExternalRuntimeContract { + pub protocol_version: &'static str, + pub transport: TargetPluginRuntimeTransport, +} + +/// Declarative distribution metadata for an installable target plugin. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginArtifactManifest { + pub artifact_id: &'static str, + pub target_triple: &'static str, + pub download_uri: &'static str, + pub digest_sha256: &'static str, + pub size_bytes: u64, +} + +/// Declarative distribution metadata for an installable target plugin. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginDistributionManifest { + pub artifacts: &'static [TargetPluginArtifactManifest], +} + +/// Marketplace-oriented manifest metadata that is explicit about future +/// installable plugin boundaries without introducing any loading behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetPluginMarketplaceManifest { + pub plugin_id: &'static str, + pub display_name: &'static str, + pub provider: &'static str, + pub version: &'static str, + pub target_type: &'static str, + pub supported_domains: &'static [TargetDomain], + pub secret_fields: &'static [&'static str], + pub packaging: TargetPluginPackaging, + pub entrypoint_kind: TargetPluginEntrypointKind, + pub api_compatibility_version: &'static str, + pub runtime_contract: TargetPluginExternalRuntimeContract, + pub distribution: Option, +} + +const BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION: &str = "rustfs.target-plugin.v1"; +const BUILTIN_PLUGIN_RUNTIME_PROTOCOL_VERSION: &str = "rustfs.target-runtime.v1"; + +const SUPPORTED_BUILTIN_DOMAINS: &[TargetDomain] = &[TargetDomain::Audit, TargetDomain::Notify]; +const NO_SECRET_FIELDS: &[&str] = &[]; + +const WEBHOOK_SECRET_FIELDS: &[&str] = &[WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY]; +const MQTT_SECRET_FIELDS: &[&str] = &[MQTT_PASSWORD, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY]; +const KAFKA_SECRET_FIELDS: &[&str] = &[KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY]; +const AMQP_SECRET_FIELDS: &[&str] = &[AMQP_PASSWORD, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY]; +const NATS_SECRET_FIELDS: &[&str] = &[ + NATS_PASSWORD, + NATS_TOKEN, + NATS_CREDENTIALS_FILE, + NATS_TLS_CLIENT_CERT, + NATS_TLS_CLIENT_KEY, +]; +const PULSAR_SECRET_FIELDS: &[&str] = &[PULSAR_AUTH_TOKEN, PULSAR_PASSWORD]; +const MYSQL_SECRET_FIELDS: &[&str] = &[MYSQL_DSN_STRING, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY]; +const REDIS_SECRET_FIELDS: &[&str] = &[REDIS_PASSWORD, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY]; +const POSTGRES_SECRET_FIELDS: &[&str] = &[POSTGRES_DSN_STRING, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY]; + +#[inline] +pub fn builtin_target_manifest(target_type: &'static str) -> TargetPluginManifest { + let (display_name, secret_fields) = match target_type { + "webhook" => ("Webhook", WEBHOOK_SECRET_FIELDS), + "mqtt" => ("MQTT", MQTT_SECRET_FIELDS), + "kafka" => ("Kafka", KAFKA_SECRET_FIELDS), + "amqp" => ("AMQP", AMQP_SECRET_FIELDS), + "nats" => ("NATS", NATS_SECRET_FIELDS), + "pulsar" => ("Pulsar", PULSAR_SECRET_FIELDS), + "mysql" => ("MySQL", MYSQL_SECRET_FIELDS), + "redis" => ("Redis", REDIS_SECRET_FIELDS), + "postgres" => ("Postgres", POSTGRES_SECRET_FIELDS), + _ => ("Custom Target", NO_SECRET_FIELDS), + }; + + TargetPluginManifest { + plugin_id: builtin_plugin_id(target_type), + display_name, + provider: "rustfs", + version: env!("CARGO_PKG_VERSION"), + target_type, + supported_domains: SUPPORTED_BUILTIN_DOMAINS, + secret_fields, + } +} + +#[inline] +pub fn builtin_target_marketplace_manifest(target_type: &'static str) -> TargetPluginMarketplaceManifest { + TargetPluginMarketplaceManifest::from(builtin_target_manifest(target_type)) +} + +impl From for TargetPluginMarketplaceManifest { + fn from(value: TargetPluginManifest) -> Self { + Self { + plugin_id: value.plugin_id, + display_name: value.display_name, + provider: value.provider, + version: value.version, + target_type: value.target_type, + supported_domains: value.supported_domains, + secret_fields: value.secret_fields, + packaging: TargetPluginPackaging::Builtin, + entrypoint_kind: TargetPluginEntrypointKind::Builtin, + api_compatibility_version: BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION, + runtime_contract: TargetPluginExternalRuntimeContract { + protocol_version: BUILTIN_PLUGIN_RUNTIME_PROTOCOL_VERSION, + transport: TargetPluginRuntimeTransport::InProcess, + }, + distribution: None, + } + } +} + +#[inline] +pub fn installable_target_marketplace_manifest( + base: TargetPluginManifest, + entrypoint_kind: TargetPluginEntrypointKind, + runtime_contract: TargetPluginExternalRuntimeContract, + distribution: TargetPluginDistributionManifest, +) -> TargetPluginMarketplaceManifest { + TargetPluginMarketplaceManifest { + plugin_id: base.plugin_id, + display_name: base.display_name, + provider: base.provider, + version: base.version, + target_type: base.target_type, + supported_domains: base.supported_domains, + secret_fields: base.secret_fields, + packaging: TargetPluginPackaging::External, + entrypoint_kind, + api_compatibility_version: BUILTIN_PLUGIN_API_COMPATIBILITY_VERSION, + runtime_contract, + distribution: Some(distribution), + } +} + +#[inline] +fn builtin_plugin_id(target_type: &'static str) -> &'static str { + match target_type { + "webhook" => "builtin:webhook", + "mqtt" => "builtin:mqtt", + "kafka" => "builtin:kafka", + "amqp" => "builtin:amqp", + "nats" => "builtin:nats", + "pulsar" => "builtin:pulsar", + "mysql" => "builtin:mysql", + "redis" => "builtin:redis", + "postgres" => "builtin:postgres", + _ => "custom:target", + } +} + +#[cfg(test)] +mod tests { + use super::{ + TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEntrypointKind, + TargetPluginExternalRuntimeContract, TargetPluginMarketplaceManifest, TargetPluginPackaging, + TargetPluginRuntimeTransport, builtin_target_manifest, builtin_target_marketplace_manifest, + installable_target_marketplace_manifest, + }; + use crate::domain::TargetDomain; + use rustfs_config::{WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY}; + + #[test] + fn builtin_webhook_manifest_marks_secret_fields() { + let manifest = builtin_target_manifest("webhook"); + + assert_eq!(manifest.plugin_id, "builtin:webhook"); + assert_eq!(manifest.display_name, "Webhook"); + assert!(manifest.secret_fields.contains(&WEBHOOK_AUTH_TOKEN)); + assert!(manifest.secret_fields.contains(&WEBHOOK_CLIENT_CERT)); + assert!(manifest.secret_fields.contains(&WEBHOOK_CLIENT_KEY)); + } + + #[test] + fn builtin_manifest_derives_marketplace_boundary_metadata() { + let manifest = builtin_target_marketplace_manifest("webhook"); + + assert_eq!(manifest.plugin_id, "builtin:webhook"); + assert_eq!(manifest.display_name, "Webhook"); + assert_eq!(manifest.target_type, "webhook"); + assert_eq!(manifest.packaging, TargetPluginPackaging::Builtin); + assert_eq!(manifest.entrypoint_kind, TargetPluginEntrypointKind::Builtin); + assert_eq!(manifest.api_compatibility_version, "rustfs.target-plugin.v1"); + assert_eq!( + manifest.runtime_contract, + TargetPluginExternalRuntimeContract { + protocol_version: "rustfs.target-runtime.v1", + transport: TargetPluginRuntimeTransport::InProcess, + } + ); + assert_eq!(manifest.distribution, None); + } + + #[test] + fn marketplace_manifest_preserves_supported_domains() { + let manifest = builtin_target_marketplace_manifest("kafka"); + + assert_eq!(manifest.supported_domains, &[TargetDomain::Audit, TargetDomain::Notify]); + } + + #[test] + fn marketplace_manifest_from_builtin_manifest_is_stable() { + let base = builtin_target_manifest("redis"); + let derived = TargetPluginMarketplaceManifest::from(base); + + assert_eq!(derived.plugin_id, "builtin:redis"); + assert_eq!(derived.target_type, "redis"); + assert_eq!(derived.packaging, TargetPluginPackaging::Builtin); + assert_eq!(derived.entrypoint_kind, TargetPluginEntrypointKind::Builtin); + assert_eq!(derived.runtime_contract.transport, TargetPluginRuntimeTransport::InProcess); + assert_eq!(derived.distribution, None); + } + + #[test] + fn installable_manifest_expresses_external_boundary_declaratively() { + let base = builtin_target_manifest("webhook"); + let manifest = installable_target_marketplace_manifest( + base, + TargetPluginEntrypointKind::Sidecar, + TargetPluginExternalRuntimeContract { + protocol_version: "rustfs.target-runtime.v1", + transport: TargetPluginRuntimeTransport::Grpc, + }, + TargetPluginDistributionManifest { + artifacts: &[TargetPluginArtifactManifest { + artifact_id: "sidecar-linux-amd64", + target_triple: "x86_64-unknown-linux-gnu", + download_uri: "https://plugins.example.test/webhook-plugin.tar.zst", + digest_sha256: "0123456789abcdef", + size_bytes: 4096, + }], + }, + ); + + assert_eq!(manifest.packaging, TargetPluginPackaging::External); + assert_eq!(manifest.entrypoint_kind, TargetPluginEntrypointKind::Sidecar); + assert_eq!(manifest.runtime_contract.transport, TargetPluginRuntimeTransport::Grpc); + assert_eq!( + manifest.distribution, + Some(TargetPluginDistributionManifest { + artifacts: &[TargetPluginArtifactManifest { + artifact_id: "sidecar-linux-amd64", + target_triple: "x86_64-unknown-linux-gnu", + download_uri: "https://plugins.example.test/webhook-plugin.tar.zst", + digest_sha256: "0123456789abcdef", + size_bytes: 4096, + }], + }) + ); + } +} diff --git a/crates/targets/src/plugin.rs b/crates/targets/src/plugin.rs index 4e2ce2d5f..21d9d09df 100644 --- a/crates/targets/src/plugin.rs +++ b/crates/targets/src/plugin.rs @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Target, TargetError, config::collect_target_configs}; +use crate::{ + PluginRuntimeAdapter, RuntimeActivation, Target, TargetError, + config::collect_target_configs, + manifest::{TargetPluginManifest, builtin_target_manifest}, +}; use hashbrown::HashMap; use rustfs_ecstore::config::{Config, KVS}; use serde::Serialize; @@ -41,12 +45,70 @@ pub enum TargetRequestValidator { }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TargetAdminMetadata { + subsystem: &'static str, + request_validator: TargetRequestValidator, +} + +impl TargetAdminMetadata { + pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator) -> Self { + Self { + subsystem, + request_validator, + } + } + + #[inline] + pub fn subsystem(&self) -> &'static str { + self.subsystem + } + + #[inline] + pub fn request_validator(&self) -> TargetRequestValidator { + self.request_validator + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BuiltinTargetAdminDescriptor { + manifest: TargetPluginManifest, + valid_fields: &'static [&'static str], + admin: TargetAdminMetadata, +} + +impl BuiltinTargetAdminDescriptor { + pub fn new(manifest: TargetPluginManifest, valid_fields: &'static [&'static str], admin: TargetAdminMetadata) -> Self { + Self { + manifest, + valid_fields, + admin, + } + } + + #[inline] + pub fn manifest(&self) -> &TargetPluginManifest { + &self.manifest + } + + #[inline] + pub fn valid_fields(&self) -> &'static [&'static str] { + self.valid_fields + } + + #[inline] + pub fn admin_metadata(&self) -> TargetAdminMetadata { + self.admin + } +} + #[derive(Clone)] pub struct TargetPluginDescriptor where E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, { create_target: TargetCreateFn, + manifest: TargetPluginManifest, target_type: &'static str, valid_fields: &'static [&'static str], valid_fields_set: Arc>, @@ -63,13 +125,27 @@ where validate_config: Validate, create_target: Create, ) -> Self + where + Create: Fn(String, &KVS) -> Result, TargetError> + Send + Sync + 'static, + Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static, + { + Self::with_manifest(builtin_target_manifest(target_type), valid_fields, validate_config, create_target) + } + + pub fn with_manifest( + manifest: TargetPluginManifest, + valid_fields: &'static [&'static str], + validate_config: Validate, + create_target: Create, + ) -> Self where Create: Fn(String, &KVS) -> Result, TargetError> + Send + Sync + 'static, Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static, { Self { create_target: Arc::new(create_target), - target_type, + manifest, + target_type: manifest.target_type, valid_fields, valid_fields_set: Arc::new(valid_fields.iter().map(|field| (*field).to_string()).collect()), validate_config: Arc::new(validate_config), @@ -81,6 +157,11 @@ where self.target_type } + #[inline] + pub fn manifest(&self) -> &TargetPluginManifest { + &self.manifest + } + #[inline] pub fn valid_fields(&self) -> &'static [&'static str] { self.valid_fields @@ -108,8 +189,7 @@ where E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, { plugin: TargetPluginDescriptor, - request_validator: TargetRequestValidator, - subsystem: &'static str, + admin: TargetAdminMetadata, } impl BuiltinTargetDescriptor @@ -119,8 +199,7 @@ where pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator, plugin: TargetPluginDescriptor) -> Self { Self { plugin, - request_validator, - subsystem, + admin: TargetAdminMetadata::new(subsystem, request_validator), } } @@ -129,14 +208,32 @@ where &self.plugin } + #[inline] + pub fn admin_metadata(&self) -> TargetAdminMetadata { + self.admin + } + #[inline] pub fn request_validator(&self) -> TargetRequestValidator { - self.request_validator + self.admin.request_validator() } #[inline] pub fn subsystem(&self) -> &'static str { - self.subsystem + self.admin.subsystem() + } +} + +impl From> for BuiltinTargetAdminDescriptor +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn from(descriptor: BuiltinTargetDescriptor) -> Self { + Self::new( + *descriptor.plugin().manifest(), + descriptor.plugin().valid_fields(), + descriptor.admin_metadata(), + ) } } @@ -220,6 +317,19 @@ where info!(count = successful_targets.len(), "All target processing completed"); Ok(successful_targets) } + + pub async fn create_activation_from_config( + &self, + config: &Config, + route_prefix: &str, + adapter: &A, + ) -> Result, TargetError> + where + A: PluginRuntimeAdapter + ?Sized, + { + let targets = self.create_targets_from_config(config, route_prefix).await?; + Ok(adapter.activate_with_replay(targets).await) + } } pub fn boxed_target(target: T) -> BoxedTarget @@ -229,3 +339,106 @@ where { Box::new(target) } + +#[cfg(test)] +mod tests { + use super::{TargetPluginDescriptor, TargetPluginRegistry}; + use crate::runtime::adapter::BuiltinPluginRuntimeAdapter; + use crate::store::{Key, Store}; + use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use crate::{StoreError, Target, TargetError}; + use async_trait::async_trait; + use rustfs_config::ENABLE_KEY; + use rustfs_ecstore::config::{Config, KVS}; + use serde::{Serialize, de::DeserializeOwned}; + use std::collections::HashMap; + use std::sync::Arc; + use std::time::Duration; + + #[derive(Clone)] + struct TestTarget { + id: crate::arn::TargetID, + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> crate::arn::TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + fn builtin_adapter() -> BuiltinPluginRuntimeAdapter { + BuiltinPluginRuntimeAdapter::new( + Arc::new(|_event| Box::pin(async {})), + Arc::new(|_target_id, _has_replay| {}), + None, + Duration::from_millis(10), + Duration::from_millis(10), + "stopping plugin registry test replay worker", + ) + } + + #[tokio::test] + async fn registry_creates_activation_from_config_via_runtime_adapter() { + let mut registry = TargetPluginRegistry::new(); + registry.register(TargetPluginDescriptor::new( + "test", + &[ENABLE_KEY, "endpoint"], + |_config| Ok(()), + |id, _config| { + Ok(Box::new(TestTarget { + id: crate::arn::TargetID::new(id, "test".to_string()), + })) + }, + )); + + let mut cfg = Config(HashMap::new()); + let mut section = HashMap::new(); + let mut primary = KVS::new(); + primary.insert(ENABLE_KEY.to_string(), "on".to_string()); + primary.insert("endpoint".to_string(), "https://example.com/hook".to_string()); + section.insert("primary".to_string(), primary); + cfg.0.insert("notify_test".to_string(), section); + + let adapter = builtin_adapter(); + let activation = registry + .create_activation_from_config(&cfg, "notify_", &adapter) + .await + .expect("activation should be created through runtime adapter"); + + assert_eq!(activation.targets.len(), 1); + assert_eq!(activation.targets[0].id().to_string(), "primary:test"); + assert!(activation.replay_workers.is_empty()); + } +} diff --git a/crates/targets/src/runtime/adapter.rs b/crates/targets/src/runtime/adapter.rs new file mode 100644 index 000000000..084efd0e5 --- /dev/null +++ b/crates/targets/src/runtime/adapter.rs @@ -0,0 +1,346 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::{ + ReplayEvent, ReplayWorkerManager, RuntimeActivation, RuntimeStatusSnapshot, RuntimeTargetHealthSnapshot, + TargetRuntimeManager, activate_targets_with_replay, init_target_and_optionally_start_replay, start_replay_worker, +}; +use crate::{Target, TargetError}; +use async_trait::async_trait; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; + +type ReplayHook = Arc) -> Pin + Send>> + Send + Sync>; +type ReplayStartObserver = Arc; + +/// Shared runtime contract for target plugins. +#[async_trait] +pub trait PluginRuntimeAdapter: Send + Sync +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + async fn activate_with_replay(&self, targets: Vec + Send + Sync>>) -> RuntimeActivation; + + async fn replace_runtime_targets( + &self, + runtime: &mut TargetRuntimeManager, + replay_workers: &mut ReplayWorkerManager, + activation: RuntimeActivation, + ) -> Result<(), TargetError>; + + async fn stop_replay_workers(&self, replay_workers: &mut ReplayWorkerManager); + + fn snapshot_runtime_status( + &self, + runtime: &TargetRuntimeManager, + replay_workers: &ReplayWorkerManager, + ) -> RuntimeStatusSnapshot; + + async fn snapshot_runtime_health(&self, runtime: &TargetRuntimeManager) -> Vec; + + async fn shutdown( + &self, + runtime: &mut TargetRuntimeManager, + replay_workers: &mut ReplayWorkerManager, + ) -> Result<(), TargetError>; +} + +/// Built-in in-process runtime adapter that preserves the current replay and +/// activation behavior while presenting a stable runtime contract to callers. +#[derive(Clone)] +pub struct BuiltinPluginRuntimeAdapter +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + replay_hook: ReplayHook, + replay_start_observer: ReplayStartObserver, + replay_semaphore: Option>, + batch_timeout: Duration, + idle_sleep: Duration, + stop_log_prefix: Arc, +} + +impl BuiltinPluginRuntimeAdapter +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn new( + replay_hook: ReplayHook, + replay_start_observer: ReplayStartObserver, + replay_semaphore: Option>, + batch_timeout: Duration, + idle_sleep: Duration, + stop_log_prefix: impl Into>, + ) -> Self { + Self { + replay_hook, + replay_start_observer, + replay_semaphore, + batch_timeout, + idle_sleep, + stop_log_prefix: stop_log_prefix.into(), + } + } +} + +#[async_trait] +impl PluginRuntimeAdapter for BuiltinPluginRuntimeAdapter +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + async fn activate_with_replay(&self, targets: Vec + Send + Sync>>) -> RuntimeActivation { + let replay_hook = Arc::clone(&self.replay_hook); + let replay_start_observer = Arc::clone(&self.replay_start_observer); + let replay_semaphore = self.replay_semaphore.clone(); + let batch_timeout = self.batch_timeout; + let idle_sleep = self.idle_sleep; + + activate_targets_with_replay(targets, move |target| { + let replay_hook = Arc::clone(&replay_hook); + let replay_start_observer = Arc::clone(&replay_start_observer); + let replay_semaphore = replay_semaphore.clone(); + + async move { + init_target_and_optionally_start_replay( + target, + move |target_id, has_replay| replay_start_observer(target_id, has_replay), + move |store, target| { + start_replay_worker( + store, + target, + Arc::clone(&replay_hook), + replay_semaphore.clone(), + batch_timeout, + idle_sleep, + ) + }, + ) + .await + } + }) + .await + } + + async fn replace_runtime_targets( + &self, + runtime: &mut TargetRuntimeManager, + replay_workers: &mut ReplayWorkerManager, + activation: RuntimeActivation, + ) -> Result<(), TargetError> { + self.stop_replay_workers(replay_workers).await; + runtime.clear_and_close().await; + + for target in activation.targets { + runtime.add_arc(target); + } + + *replay_workers = activation.replay_workers; + Ok(()) + } + + async fn stop_replay_workers(&self, replay_workers: &mut ReplayWorkerManager) { + replay_workers.stop_all(&self.stop_log_prefix).await; + } + + fn snapshot_runtime_status( + &self, + runtime: &TargetRuntimeManager, + replay_workers: &ReplayWorkerManager, + ) -> RuntimeStatusSnapshot { + runtime.status_snapshot(replay_workers) + } + + async fn snapshot_runtime_health(&self, runtime: &TargetRuntimeManager) -> Vec { + runtime.health_snapshots().await + } + + async fn shutdown( + &self, + runtime: &mut TargetRuntimeManager, + replay_workers: &mut ReplayWorkerManager, + ) -> Result<(), TargetError> { + self.stop_replay_workers(replay_workers).await; + runtime.clear_and_close().await; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter}; + use crate::arn::TargetID; + use crate::store::{Key, QueueStore, Store}; + use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use crate::{StoreError, Target, TargetError}; + use async_trait::async_trait; + use serde::{Serialize, de::DeserializeOwned}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + use tempfile::tempdir; + + #[derive(Clone)] + struct TestTarget { + close_calls: Arc, + id: TargetID, + init_fails: bool, + store: Option>, + } + + impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + close_calls: Arc::new(AtomicUsize::new(0)), + id: TargetID::new(id.to_string(), name.to_string()), + init_fails: false, + store: None, + } + } + + fn with_failed_init(mut self) -> Self { + self.init_fails = true; + self + } + + fn with_store(mut self) -> Self { + let dir = tempdir().expect("tempdir should be created for queue store tests"); + let store = QueueStore::::new(dir.path(), 16, ".queue"); + store.open().expect("queue store should open"); + self.store = Some(store); + self + } + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + self.store + .as_ref() + .map(|store| store as &(dyn Store + Send + Sync)) + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + async fn init(&self) -> Result<(), TargetError> { + if self.init_fails { + return Err(TargetError::Configuration("forced init failure".to_string())); + } + Ok(()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + fn builtin_adapter() -> BuiltinPluginRuntimeAdapter { + BuiltinPluginRuntimeAdapter::new( + Arc::new(|_event| Box::pin(async {})), + Arc::new(|_target_id, _has_replay| {}), + None, + Duration::from_millis(10), + Duration::from_millis(10), + "stopping test replay worker", + ) + } + + #[tokio::test] + async fn builtin_adapter_handles_empty_target_activation() { + let adapter = builtin_adapter(); + let activation = adapter.activate_with_replay(Vec::new()).await; + + assert!(activation.targets.is_empty()); + assert!(activation.replay_workers.is_empty()); + } + + #[tokio::test] + async fn builtin_adapter_skips_non_store_target_when_init_fails() { + let adapter = builtin_adapter(); + let target = TestTarget::new("primary", "webhook").with_failed_init(); + + let activation = adapter.activate_with_replay(vec![Box::new(target)]).await; + + assert!(activation.targets.is_empty()); + assert!(activation.replay_workers.is_empty()); + } + + #[tokio::test] + async fn builtin_adapter_keeps_store_backed_target_when_init_fails() { + let adapter = builtin_adapter(); + let target = TestTarget::new("primary", "webhook").with_failed_init().with_store(); + + let activation = adapter.activate_with_replay(vec![Box::new(target)]).await; + + assert_eq!(activation.targets.len(), 1); + assert_eq!(activation.replay_workers.len(), 1); + } + + #[tokio::test] + async fn builtin_adapter_shutdown_clears_runtime_and_replay_workers() { + let adapter = builtin_adapter(); + let target = TestTarget::new("primary", "webhook"); + let close_calls = Arc::clone(&target.close_calls); + let mut runtime = crate::runtime::TargetRuntimeManager::new(); + let mut replay_workers = crate::runtime::ReplayWorkerManager::new(); + + let activation = adapter.activate_with_replay(vec![Box::new(target)]).await; + adapter + .replace_runtime_targets(&mut runtime, &mut replay_workers, activation) + .await + .expect("replace_runtime_targets should succeed"); + + assert_eq!(runtime.len(), 1); + assert_eq!(replay_workers.len(), 0); + + adapter + .shutdown(&mut runtime, &mut replay_workers) + .await + .expect("shutdown should succeed"); + + assert!(runtime.is_empty()); + assert!(replay_workers.is_empty()); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/targets/src/runtime/mod.rs b/crates/targets/src/runtime/mod.rs new file mode 100644 index 000000000..cee4e1136 --- /dev/null +++ b/crates/targets/src/runtime/mod.rs @@ -0,0 +1,641 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod adapter; +pub mod sidecar; +pub mod sidecar_protocol; + +use crate::Target; +use crate::arn::TargetID; +use crate::store::{Key, Store, ensure_store_entry_raw_readable}; +use crate::target::QueuedPayload; +use crate::target::TargetDeliverySnapshot; +use crate::{StoreError, TargetError}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::sync::Arc; +use std::{collections::HashMap, fmt::Debug}; +use std::{future::Future, pin::Pin, time::Duration}; +use tokio::sync::{Semaphore, mpsc}; + +/// Shared target trait object used by the runtime manager. +pub type SharedTarget = Arc + Send + Sync>; +type ReplayHook = Arc) -> Pin + Send>> + Send + Sync>; + +#[derive(Debug, Default)] +pub struct ReplayWorkerManager { + cancellers: HashMap>, +} + +impl ReplayWorkerManager { + pub fn new() -> Self { + Self { + cancellers: HashMap::new(), + } + } + + pub fn insert(&mut self, target_id: String, cancel_tx: mpsc::Sender<()>) { + self.cancellers.insert(target_id, cancel_tx); + } + + pub fn len(&self) -> usize { + self.cancellers.len() + } + + pub fn is_empty(&self) -> bool { + self.cancellers.is_empty() + } + + pub fn snapshot(&self, target_count: usize) -> RuntimeStatusSnapshot { + RuntimeStatusSnapshot { + replay_worker_count: self.len(), + target_count, + } + } + + pub async fn stop_all(&mut self, log_prefix: &str) { + for (target_id, cancel_tx) in self.cancellers.drain() { + tracing::info!(target_id = %target_id, "{log_prefix}"); + let _ = cancel_tx.send(()).await; + } + } +} + +pub struct RuntimeActivation +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub replay_workers: ReplayWorkerManager, + pub targets: Vec>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RuntimeStatusSnapshot { + pub replay_worker_count: usize, + pub target_count: usize, +} + +/// A read-only runtime snapshot for a target instance. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RuntimeTargetSnapshot { + pub failed_messages: u64, + pub queue_length: u64, + pub target_id: String, + pub target_type: String, + pub total_messages: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeTargetHealthState { + Disabled, + Error, + Offline, + Online, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimeTargetHealthSnapshot { + pub enabled: bool, + pub error_message: Option, + pub state: RuntimeTargetHealthState, + pub target_id: String, + pub target_type: String, +} + +pub enum ReplayEvent +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + Delivered { + key: Key, + target: SharedTarget, + }, + RetryableError { + error: TargetError, + key: Key, + retry_count: usize, + target: SharedTarget, + }, + Dropped { + key: Key, + reason: String, + target: SharedTarget, + }, + PermanentFailure { + error: TargetError, + key: Key, + target: SharedTarget, + }, + RetryExhausted { + key: Key, + target: SharedTarget, + }, + UnreadableEntry { + error: StoreError, + key: Key, + target: SharedTarget, + }, +} + +/// Shared runtime container for managing instantiated targets. +/// +/// This intentionally focuses on low-risk shared lifecycle primitives first: +/// add/remove/close/list/snapshot. Replay workers and reload orchestration can +/// be layered on top in later phases. +pub struct TargetRuntimeManager +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + targets: HashMap>, +} + +impl Default for TargetRuntimeManager +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn default() -> Self { + Self::new() + } +} + +impl Debug for TargetRuntimeManager +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TargetRuntimeManager") + .field("target_count", &self.targets.len()) + .finish() + } +} + +impl TargetRuntimeManager +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + pub fn new() -> Self { + Self { targets: HashMap::new() } + } + + pub fn add_arc(&mut self, target: SharedTarget) -> Option> { + let key = target.id().to_string(); + self.targets.insert(key, target) + } + + pub fn add_boxed(&mut self, target: Box + Send + Sync>) -> Option> { + self.add_arc(Arc::from(target)) + } + + pub fn get(&self, key: &str) -> Option> { + self.targets.get(key).cloned() + } + + pub fn get_by_target_id(&self, target_id: &TargetID) -> Option> { + self.get(&target_id.to_string()) + } + + pub fn remove(&mut self, key: &str) -> Option> { + self.targets.remove(key) + } + + pub fn remove_by_target_id(&mut self, target_id: &TargetID) -> Option> { + self.remove(&target_id.to_string()) + } + + pub fn clear(&mut self) { + self.targets.clear(); + } + + pub async fn remove_and_close(&mut self, key: &str) -> Option> { + let target = self.targets.remove(key)?; + if let Err(err) = target.close().await { + tracing::error!(target_id = %key, error = %err, "Failed to close target during removal"); + } + Some(target) + } + + pub async fn remove_by_target_id_and_close(&mut self, target_id: &TargetID) -> Option> { + self.remove_and_close(&target_id.to_string()).await + } + + pub async fn clear_and_close(&mut self) { + let target_ids: Vec = self.targets.keys().cloned().collect(); + for target_id in target_ids { + let _ = self.remove_and_close(&target_id).await; + } + self.targets.clear(); + } + + pub fn target_ids(&self) -> Vec { + self.targets.values().map(|target| target.id()).collect() + } + + pub fn keys(&self) -> Vec { + self.targets.keys().cloned().collect() + } + + pub fn values(&self) -> Vec> { + self.targets.values().cloned().collect() + } + + pub fn len(&self) -> usize { + self.targets.len() + } + + pub fn is_empty(&self) -> bool { + self.targets.is_empty() + } + + pub fn snapshots(&self) -> Vec { + let mut snapshots = Vec::with_capacity(self.targets.len()); + for target in self.targets.values() { + let delivery = target.delivery_snapshot(); + let target_id = target.id(); + snapshots.push(snapshot_from_delivery(target_id, delivery)); + } + snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id)); + snapshots + } + + pub fn status_snapshot(&self, replay_workers: &ReplayWorkerManager) -> RuntimeStatusSnapshot { + replay_workers.snapshot(self.len()) + } + + pub async fn health_snapshots(&self) -> Vec { + let mut snapshots = Vec::with_capacity(self.targets.len()); + for target in self.targets.values() { + let enabled = target.is_enabled(); + let target_id = target.id(); + let (state, error_message) = if !enabled { + (RuntimeTargetHealthState::Disabled, None) + } else { + match target.is_active().await { + Ok(true) => (RuntimeTargetHealthState::Online, None), + Ok(false) => (RuntimeTargetHealthState::Offline, None), + Err(err) => (RuntimeTargetHealthState::Error, Some(err.to_string())), + } + }; + + snapshots.push(RuntimeTargetHealthSnapshot { + enabled, + error_message, + state, + target_id: target_id.to_string(), + target_type: target_id.name, + }); + } + snapshots.sort_by(|a, b| a.target_id.cmp(&b.target_id)); + snapshots + } +} + +fn snapshot_from_delivery(target_id: TargetID, delivery: TargetDeliverySnapshot) -> RuntimeTargetSnapshot { + RuntimeTargetSnapshot { + failed_messages: delivery.failed_messages, + queue_length: delivery.queue_length, + target_id: target_id.to_string(), + target_type: target_id.name, + total_messages: delivery.total_messages, + } +} + +pub async fn init_target_and_optionally_start_replay( + target: Box + Send + Sync>, + on_replay_start: F, + start_replay: G, +) -> Option<(SharedTarget, Option>)> +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + F: FnOnce(&str, bool), + G: FnOnce(Box + Send>, SharedTarget) -> mpsc::Sender<()>, +{ + let target_id = target.id().to_string(); + let has_store = target.store().is_some(); + + if let Err(err) = target.init().await { + tracing::error!(target_id = %target_id, error = %err, "Failed to initialize target"); + if !has_store { + return None; + } + tracing::warn!( + target_id = %target_id, + "Proceeding with store-backed target despite init failure" + ); + } + + let shared: SharedTarget = Arc::from(target); + if !shared.is_enabled() { + on_replay_start(&target_id, false); + return Some((shared, None)); + } + + let cancel = shared + .store() + .map(|store| start_replay(store.boxed_clone(), Arc::clone(&shared))); + on_replay_start(&target_id, cancel.is_some()); + Some((shared, cancel)) +} + +pub async fn activate_targets_with_replay( + targets: Vec + Send + Sync>>, + mut activate_one: F, +) -> RuntimeActivation +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + F: FnMut(Box + Send + Sync>) -> Fut, + Fut: Future, Option>)>>, +{ + let mut replay_workers = ReplayWorkerManager::new(); + let mut shared_targets = Vec::new(); + + for target in targets { + if let Some((shared_target, cancel_tx)) = activate_one(target).await { + let target_id = shared_target.id().to_string(); + if let Some(cancel_tx) = cancel_tx { + replay_workers.insert(target_id, cancel_tx); + } + shared_targets.push(shared_target); + } + } + + RuntimeActivation { + replay_workers, + targets: shared_targets, + } +} + +pub fn start_replay_worker( + mut store: Box + Send>, + target: SharedTarget, + hook: ReplayHook, + semaphore: Option>, + batch_timeout: Duration, + idle_sleep: Duration, +) -> mpsc::Sender<()> +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + let (cancel_tx, cancel_rx) = mpsc::channel(1); + + tokio::spawn(async move { + stream_replay_worker(&mut *store, target, cancel_rx, hook, semaphore, batch_timeout, idle_sleep).await; + }); + + cancel_tx +} + +async fn stream_replay_worker( + store: &mut (dyn Store + Send), + target: SharedTarget, + mut cancel_rx: mpsc::Receiver<()>, + hook: ReplayHook, + semaphore: Option>, + batch_timeout: Duration, + idle_sleep: Duration, +) where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + const MAX_RETRIES: usize = 5; + const BASE_RETRY_DELAY: Duration = Duration::from_secs(2); + + let mut batch_keys = Vec::with_capacity(1); + let mut last_flush = tokio::time::Instant::now(); + + loop { + if cancel_rx.try_recv().is_ok() { + return; + } + + let keys = store.list(); + if keys.is_empty() { + if !batch_keys.is_empty() && last_flush.elapsed() >= batch_timeout { + process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await; + last_flush = tokio::time::Instant::now(); + } + tokio::time::sleep(idle_sleep).await; + continue; + } + + for key in keys { + if cancel_rx.try_recv().is_ok() { + if !batch_keys.is_empty() { + process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await; + } + return; + } + + match ensure_store_entry_raw_readable(&*store, &key) { + Ok(true) => {} + Ok(false) => continue, + Err(err) => { + hook(ReplayEvent::UnreadableEntry { + error: err, + key, + target: target.clone(), + }) + .await; + continue; + } + } + + batch_keys.push(key); + if !batch_keys.is_empty() || last_flush.elapsed() >= batch_timeout { + process_replay_batch(&mut batch_keys, target.clone(), &hook, semaphore.clone()).await; + last_flush = tokio::time::Instant::now(); + } + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } + + async fn process_replay_batch( + batch_keys: &mut Vec, + target: SharedTarget, + hook: &ReplayHook, + semaphore: Option>, + ) where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + if batch_keys.is_empty() { + return; + } + + let _permit = match semaphore { + Some(ref semaphore) => match semaphore.clone().acquire_owned().await { + Ok(permit) => Some(permit), + Err(err) => { + tracing::error!(error = %err, "Failed to acquire replay semaphore permit"); + return; + } + }, + None => None, + }; + + for key in batch_keys.iter() { + let mut retry_count = 0usize; + let mut success = false; + + while retry_count < MAX_RETRIES && !success { + match target.send_from_store(key.clone()).await { + Ok(_) => { + hook(ReplayEvent::Delivered { + key: key.clone(), + target: target.clone(), + }) + .await; + success = true; + } + Err(err) => match err { + TargetError::NotConnected | TargetError::Timeout(_) => { + retry_count += 1; + hook(ReplayEvent::RetryableError { + error: err, + key: key.clone(), + retry_count, + target: target.clone(), + }) + .await; + + let jitter = Duration::from_millis(key.to_string().len() as u64 % 500); + let backoff = 1u32 << retry_count as u32; + tokio::time::sleep(BASE_RETRY_DELAY * backoff + jitter).await; + } + TargetError::Dropped(reason) => { + hook(ReplayEvent::Dropped { + key: key.clone(), + reason, + target: target.clone(), + }) + .await; + break; + } + other => { + hook(ReplayEvent::PermanentFailure { + error: other, + key: key.clone(), + target: target.clone(), + }) + .await; + break; + } + }, + } + } + + if retry_count >= MAX_RETRIES && !success { + hook(ReplayEvent::RetryExhausted { + key: key.clone(), + target: target.clone(), + }) + .await; + } + } + + batch_keys.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::TargetRuntimeManager; + use crate::StoreError; + use crate::arn::TargetID; + use crate::store::{Key, Store}; + use crate::target::{EntityTarget, QueuedPayload, QueuedPayloadMeta}; + use crate::{Target, TargetError}; + use async_trait::async_trait; + use serde::{Serialize, de::DeserializeOwned}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[derive(Clone)] + struct TestTarget { + id: TargetID, + close_calls: Arc, + } + + impl TestTarget { + fn new(id: &str, name: &str) -> Self { + Self { + id: TargetID::new(id.to_string(), name.to_string()), + close_calls: Arc::new(AtomicUsize::new(0)), + } + } + } + + #[async_trait] + impl Target for TestTarget + where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + { + fn id(&self) -> TargetID { + self.id.clone() + } + + async fn is_active(&self) -> Result { + Ok(true) + } + + async fn save(&self, _event: Arc>) -> Result<(), TargetError> { + Ok(()) + } + + async fn send_raw_from_store(&self, _key: Key, _body: Vec, _meta: QueuedPayloadMeta) -> Result<(), TargetError> { + Ok(()) + } + + async fn close(&self) -> Result<(), TargetError> { + self.close_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn store(&self) -> Option<&(dyn Store + Send + Sync)> { + None + } + + fn clone_dyn(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + + fn is_enabled(&self) -> bool { + true + } + } + + #[tokio::test] + async fn runtime_manager_removes_and_closes_target() { + let mut manager = TargetRuntimeManager::::new(); + let target = TestTarget::new("primary", "webhook"); + let close_calls = Arc::clone(&target.close_calls); + + manager.add_boxed(Box::new(target)); + assert_eq!(manager.len(), 1); + + let removed = manager.remove_and_close("primary:webhook").await; + assert!(removed.is_some()); + assert_eq!(manager.len(), 0); + assert_eq!(close_calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn runtime_manager_snapshots_targets() { + let mut manager = TargetRuntimeManager::::new(); + manager.add_boxed(Box::new(TestTarget::new("primary", "webhook"))); + + let snapshots = manager.snapshots(); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].target_id, "primary:webhook"); + assert_eq!(snapshots[0].target_type, "webhook"); + } +} diff --git a/crates/targets/src/runtime/sidecar.rs b/crates/targets/src/runtime/sidecar.rs new file mode 100644 index 000000000..a54b8568f --- /dev/null +++ b/crates/targets/src/runtime/sidecar.rs @@ -0,0 +1,171 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::TargetDomain; +use crate::runtime::sidecar_protocol::SidecarHandshake; +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +const DEFAULT_FAILURE_THRESHOLD: usize = 3; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct SidecarPluginRuntime { + pub endpoint: String, + pub handshake: SidecarHandshake, + pub healthy: bool, + pub failure_count: usize, + pub degraded_to_builtin: bool, + pub last_error: Option, +} + +impl SidecarPluginRuntime { + pub fn new(endpoint: impl Into, handshake: SidecarHandshake) -> Self { + Self { + endpoint: endpoint.into(), + handshake, + healthy: false, + failure_count: 0, + degraded_to_builtin: false, + last_error: None, + } + } + + pub fn enable(&mut self, expected_plugin_id: &str, required_domain: TargetDomain) -> Result<(), String> { + self.handshake.validate(expected_plugin_id)?; + if !self.handshake.supported_domains.contains(&required_domain) { + return Err(format!( + "sidecar plugin {} does not support required domain {:?}", + self.handshake.plugin_id, required_domain + )); + } + + self.healthy = true; + self.degraded_to_builtin = false; + self.last_error = None; + self.failure_count = 0; + Ok(()) + } + + pub fn mark_unhealthy(&mut self) { + self.healthy = false; + } + + pub fn record_failure(&mut self, error: impl Into) { + self.failure_count = self.failure_count.saturating_add(1); + self.healthy = false; + self.last_error = Some(error.into()); + if self.failure_count >= DEFAULT_FAILURE_THRESHOLD { + self.degraded_to_builtin = true; + } + } + + pub fn send_with_timeout(&mut self, operation_timeout: Duration, simulated_latency: Duration) -> Result<(), String> { + if simulated_latency > operation_timeout { + self.record_failure(format!( + "sidecar send timeout after {:?} (budget {:?})", + simulated_latency, operation_timeout + )); + return Err(self + .last_error + .clone() + .unwrap_or_else(|| "sidecar timeout without recorded error".to_string())); + } + self.healthy = true; + self.last_error = None; + Ok(()) + } + + pub fn shutdown(&mut self) { + self.healthy = false; + } +} + +#[cfg(test)] +mod tests { + use super::SidecarPluginRuntime; + use crate::TargetDomain; + use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability}; + use std::time::Duration; + + fn notify_sidecar_handshake() -> SidecarHandshake { + SidecarHandshake { + protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(), + plugin_id: "external:webhook".to_string(), + plugin_version: "1.2.3".to_string(), + supported_domains: vec![TargetDomain::Notify], + capabilities: vec![ + SidecarPluginCapability::HealthCheck, + SidecarPluginCapability::SendEvent, + SidecarPluginCapability::Shutdown, + ], + } + } + + #[test] + fn sidecar_runtime_enable_marks_runtime_healthy() { + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake()); + + runtime + .enable("external:webhook", TargetDomain::Notify) + .expect("sidecar runtime should enable"); + + assert!(runtime.healthy); + } + + #[test] + fn sidecar_runtime_enable_rejects_domain_mismatch() { + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake()); + + let result = runtime.enable("external:webhook", TargetDomain::Audit); + + assert!(result.is_err()); + assert!(!runtime.healthy); + } + + #[test] + fn sidecar_runtime_shutdown_marks_runtime_unhealthy() { + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake()); + runtime + .enable("external:webhook", TargetDomain::Notify) + .expect("sidecar runtime should enable"); + + runtime.shutdown(); + + assert!(!runtime.healthy); + } + + #[test] + fn sidecar_runtime_degrades_to_builtin_after_failure_threshold() { + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake()); + + runtime.record_failure("send failed"); + runtime.record_failure("send failed again"); + runtime.record_failure("send failed third time"); + + assert!(runtime.degraded_to_builtin); + assert!(!runtime.healthy); + assert_eq!(runtime.failure_count, 3); + } + + #[test] + fn sidecar_runtime_send_timeout_records_last_error() { + let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake()); + + let result = runtime.send_with_timeout(Duration::from_millis(50), Duration::from_millis(75)); + + assert!(result.is_err()); + assert_eq!(runtime.last_error.as_deref(), Some("sidecar send timeout after 75ms (budget 50ms)")); + } +} diff --git a/crates/targets/src/runtime/sidecar_protocol.rs b/crates/targets/src/runtime/sidecar_protocol.rs new file mode 100644 index 000000000..04aee028c --- /dev/null +++ b/crates/targets/src/runtime/sidecar_protocol.rs @@ -0,0 +1,106 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::TargetDomain; +use serde::{Deserialize, Serialize}; + +pub const SIDECAR_RUNTIME_PROTOCOL_VERSION: &str = "rustfs.target-runtime.v1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SidecarPluginCapability { + HealthCheck, + SendEvent, + Shutdown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct SidecarHandshake { + pub protocol_version: String, + pub plugin_id: String, + pub plugin_version: String, + pub supported_domains: Vec, + pub capabilities: Vec, +} + +impl SidecarHandshake { + pub fn validate(&self, expected_plugin_id: &str) -> Result<(), String> { + if self.protocol_version != SIDECAR_RUNTIME_PROTOCOL_VERSION { + return Err(format!( + "unsupported sidecar protocol version: expected {}, got {}", + SIDECAR_RUNTIME_PROTOCOL_VERSION, self.protocol_version + )); + } + + if self.plugin_id != expected_plugin_id { + return Err(format!( + "sidecar plugin id mismatch: expected {}, got {}", + expected_plugin_id, self.plugin_id + )); + } + + for capability in [ + SidecarPluginCapability::HealthCheck, + SidecarPluginCapability::SendEvent, + SidecarPluginCapability::Shutdown, + ] { + if !self.capabilities.contains(&capability) { + return Err(format!("sidecar handshake missing required capability: {:?}", capability)); + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability}; + use crate::TargetDomain; + + #[test] + fn sidecar_handshake_accepts_expected_contract() { + let handshake = SidecarHandshake { + protocol_version: SIDECAR_RUNTIME_PROTOCOL_VERSION.to_string(), + plugin_id: "external:webhook".to_string(), + plugin_version: "1.2.3".to_string(), + supported_domains: vec![TargetDomain::Notify], + capabilities: vec![ + SidecarPluginCapability::HealthCheck, + SidecarPluginCapability::SendEvent, + SidecarPluginCapability::Shutdown, + ], + }; + + assert!(handshake.validate("external:webhook").is_ok()); + } + + #[test] + fn sidecar_handshake_rejects_protocol_mismatch() { + let handshake = SidecarHandshake { + protocol_version: "rustfs.target-runtime.v0".to_string(), + plugin_id: "external:webhook".to_string(), + plugin_version: "1.2.3".to_string(), + supported_domains: vec![TargetDomain::Notify], + capabilities: vec![ + SidecarPluginCapability::HealthCheck, + SidecarPluginCapability::SendEvent, + SidecarPluginCapability::Shutdown, + ], + }; + + assert!(handshake.validate("external:webhook").is_err()); + } +} diff --git a/crates/targets/src/target/amqp.rs b/crates/targets/src/target/amqp.rs index 58408c916..1173bdb2c 100644 --- a/crates/targets/src/target/amqp.rs +++ b/crates/targets/src/target/amqp.rs @@ -19,13 +19,14 @@ //! body through `send_raw_from_store`. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload_with_records, is_connectivity_error, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -39,11 +40,11 @@ use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY}; use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use std::time::Duration; use tokio::sync::Mutex as AsyncMutex; -use tracing::{error, info, instrument, warn}; +use tracing::{info, instrument, warn}; use url::Url; #[derive(Clone)] @@ -315,22 +316,14 @@ where pub fn new(id: String, args: AMQPArgs) -> Result { args.validate()?; let target_id = TargetID::new(id, ChannelTargetType::Amqp.as_str().to_string()); - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Amqp.as_str(), &target_id.id)); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!(target_id = %target_id, error = %e, "Failed to open store for AMQP target"); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Amqp.as_str(), + &target_id, + "Failed to open store for AMQP target", + )?; Ok(Self { id: target_id, @@ -344,22 +337,7 @@ where } fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.clone()], - }; - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - Ok(QueuedPayload::new(meta, body)) + build_queued_payload_with_records(event, vec![event.clone()]) } async fn get_or_connect(&self) -> Result, TargetError> { @@ -453,16 +431,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } Ok(()) } else { @@ -505,10 +476,7 @@ where } match self.get_or_connect().await { Ok(_) => Ok(()), - Err(err) - if self.store.is_some() - && matches!(err, TargetError::Network(_) | TargetError::Timeout(_) | TargetError::NotConnected) => - { + Err(err) if self.store.is_some() && is_connectivity_error(&err) => { warn!(target_id = %self.id, error = %err, "AMQP init failed; events will buffer in store"); Ok(()) } @@ -535,6 +503,7 @@ mod tests { use super::*; use rustfs_s3_common::EventName; use serde_json::json; + use std::path::PathBuf; use std::sync::Arc; use uuid::Uuid; diff --git a/crates/targets/src/target/kafka.rs b/crates/targets/src/target/kafka.rs index 3c3132ab8..6a6afba0a 100644 --- a/crates/targets/src/target/kafka.rs +++ b/crates/targets/src/target/kafka.rs @@ -13,23 +13,22 @@ // limitations under the License. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; -use rustfs_config::audit::AUDIT_STORE_EXTENSION; -use rustfs_config::notify::NOTIFY_STORE_EXTENSION; use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError}; use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig}; use serde::Serialize; use serde::de::DeserializeOwned; -use std::{marker::PhantomData, path::PathBuf, sync::Arc, time::Duration}; +use std::{marker::PhantomData, sync::Arc, time::Duration}; use tokio::sync::Mutex; use tracing::{debug, error, info, instrument, warn}; @@ -123,10 +122,6 @@ where } } - fn is_connection_error(err: &TargetError) -> bool { - matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) - } - /// Creates a new KafkaTarget #[instrument(skip(args), fields(target_id = %id))] pub fn new(id: String, args: KafkaArgs) -> Result { @@ -134,25 +129,14 @@ where let target_id = TargetID::new(id, ChannelTargetType::Kafka.as_str().to_string()); - let queue_store = if !args.queue_dir.is_empty() { - let queue_dir = - PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Kafka.as_str(), &target_id.id)); - - let extension = match args.target_type { - TargetType::AuditLog => AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION, - }; - - let store = QueueStore::::new(queue_dir, args.queue_limit, extension); - if let Err(e) = store.open() { - error!("Failed to open store for Kafka target {}: {}", target_id.id, e); - return Err(TargetError::Storage(format!("{e}"))); - } - - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Kafka.as_str(), + &target_id, + "Failed to open store for Kafka target", + )?; info!(target_id = %target_id.id, "Kafka target created"); Ok(KafkaTarget { @@ -211,26 +195,7 @@ where /// Serializes the event and builds a QueuedPayload fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.data.clone()], - }; - - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - - Ok(QueuedPayload::new(meta, body)) + build_queued_payload(event) } /// Sends the raw body to Kafka @@ -249,9 +214,7 @@ where if let Err(err) = producer.send(&Record::from_value(&self.args.topic, body.as_slice())).await { let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka"); - if Self::is_connection_error(&mapped) { - self.invalidate_cached_producer().await; - } + invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_producer()).await; return Err(mapped); } @@ -297,16 +260,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } debug!("Event saved to store for Kafka target: {}", self.id); Ok(()) diff --git a/crates/targets/src/target/mod.rs b/crates/targets/src/target/mod.rs index d10a30d0c..351d06a41 100644 --- a/crates/targets/src/target/mod.rs +++ b/crates/targets/src/target/mod.rs @@ -13,15 +13,17 @@ // limitations under the License. use crate::arn::TargetID; -use crate::store::{Key, Store}; +use crate::store::{Key, QueueStore, Store}; use crate::{StoreError, TargetError, TargetLog}; use async_trait::async_trait; use rustfs_s3_common::EventName; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::fmt::Formatter; +use std::future::Future; +use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, warn}; @@ -50,6 +52,8 @@ pub struct TargetDeliveryCounters { total_messages: AtomicU64, } +pub(crate) type BoxedQueuedStore = Box + Send + Sync>; + impl TargetDeliveryCounters { #[inline] pub fn record_success(&self) { @@ -408,6 +412,17 @@ pub fn decode_object_name(encoded: &str) -> Result { pub(crate) fn build_queued_payload(event: &EntityTarget) -> Result where E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, +{ + build_queued_payload_with_records(event, vec![event.data.clone()]) +} + +pub(crate) fn build_queued_payload_with_records( + event: &EntityTarget, + records: Vec, +) -> Result +where + E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned, + R: Serialize, { let object_name = decode_object_name(&event.object_name)?; let key = format!("{}/{}", event.bucket_name, object_name); @@ -415,7 +430,7 @@ where let log = TargetLog { event_name: event.event_name, key, - records: vec![event.data.clone()], + records, }; let body = serde_json::to_vec(&log).map_err(|err| TargetError::Serialization(format!("Failed to serialize event: {err}")))?; @@ -430,6 +445,68 @@ where Ok(QueuedPayload::new(meta, body)) } +pub(crate) fn open_target_queue_store( + queue_dir: &str, + queue_limit: u64, + target_type: TargetType, + target_type_label: &str, + target_id: &TargetID, + open_context: &str, +) -> Result, TargetError> { + fn boxed_queue_store(store: QueueStore) -> BoxedQueuedStore { + Box::new(store) + } + + if queue_dir.is_empty() { + return Ok(None); + } + + let queue_dir = PathBuf::from(queue_dir).join(queue_store_subdir_name(target_type_label, &target_id.id)); + let extension = match target_type { + TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, + TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, + }; + let store = QueueStore::::new(queue_dir, queue_limit, extension); + store + .open() + .map_err(|err| TargetError::Storage(format!("{open_context}: {err}")))?; + + Ok(Some(boxed_queue_store(store))) +} + +pub(crate) fn persist_queued_payload_to_store( + store: &(dyn Store + Send + Sync), + queued: &QueuedPayload, +) -> Result<(), TargetError> { + let encoded = queued + .encode() + .map_err(|err| TargetError::Storage(format!("Failed to encode queued payload: {err}")))?; + store + .put_raw(&encoded) + .map(|_| ()) + .map_err(|err| TargetError::Storage(format!("Failed to save event to store: {err}"))) +} + +pub(crate) fn is_connectivity_error(err: &TargetError) -> bool { + matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) +} + +pub(crate) async fn invalidate_cache_on_connectivity_error(err: &TargetError, invalidate: F) +where + F: FnOnce() -> Fut, + Fut: Future, +{ + if is_connectivity_error(err) { + invalidate().await; + } +} + +pub(crate) fn mark_target_disconnected_on_connectivity_error(connected: &AtomicBool, err: &TargetError) { + if is_connectivity_error(err) { + connected.store(false, Ordering::SeqCst); + } +} + pub(crate) fn delete_stored_payload( store: &(dyn Store + Send + Sync), key: &Key, @@ -457,6 +534,90 @@ pub(crate) fn ensure_rustls_provider_installed() { #[cfg(test)] mod tests { use super::*; + use std::fs; + use std::sync::Mutex; + use uuid::Uuid; + + #[derive(Clone)] + struct MockQueuedStore { + fail_put_raw: bool, + writes: Arc>>>, + } + + impl MockQueuedStore { + fn new(fail_put_raw: bool) -> Self { + Self { + fail_put_raw, + writes: Arc::new(Mutex::new(Vec::new())), + } + } + } + + impl Store for MockQueuedStore { + type Error = StoreError; + type Key = Key; + + fn open(&self) -> Result<(), Self::Error> { + Ok(()) + } + + fn put(&self, _item: Arc) -> Result { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn put_multiple(&self, _items: Vec) -> Result { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn put_raw(&self, data: &[u8]) -> Result { + if self.fail_put_raw { + return Err(StoreError::Internal("mock put_raw failed".to_string())); + } + self.writes.lock().expect("mock writes lock poisoned").push(data.to_vec()); + Ok(Key { + name: "mock".to_string(), + extension: ".json".to_string(), + item_count: 1, + compress: false, + }) + } + + fn get(&self, _key: &Self::Key) -> Result { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn get_multiple(&self, _key: &Self::Key) -> Result, Self::Error> { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn get_raw(&self, _key: &Self::Key) -> Result, Self::Error> { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn del(&self, _key: &Self::Key) -> Result<(), Self::Error> { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn delete(&self) -> Result<(), Self::Error> { + Err(StoreError::Internal("not implemented in mock".to_string())) + } + + fn list(&self) -> Vec { + Vec::new() + } + + fn len(&self) -> usize { + 0 + } + + fn is_empty(&self) -> bool { + true + } + + fn boxed_clone(&self) -> Box + Send + Sync> { + Box::new(self.clone()) + } + } #[test] fn channel_target_type_amqp_uses_runtime_name() { @@ -501,6 +662,139 @@ mod tests { assert_eq!(value["Records"][0], "payload-data"); } + #[test] + fn build_queued_payload_with_records_preserves_custom_record_shape() { + let event = EntityTarget { + object_name: "object.txt".to_string(), + bucket_name: "bucket-a".to_string(), + event_name: EventName::ObjectCreatedPut, + data: "ignored".to_string(), + }; + + let payload = build_queued_payload_with_records(&event, vec![event.clone()]).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&payload.body).unwrap(); + + assert_eq!(value["Records"][0]["bucket_name"], "bucket-a"); + assert_eq!(value["Records"][0]["object_name"], "object.txt"); + assert_eq!(value["Records"][0]["data"], "ignored"); + } + + #[test] + fn open_target_queue_store_returns_none_when_queue_dir_empty() { + let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Webhook.as_str().to_string()); + let store = open_target_queue_store( + "", + 100, + TargetType::NotifyEvent, + ChannelTargetType::Webhook.as_str(), + &target_id, + "open failed", + ) + .unwrap(); + assert!(store.is_none()); + } + + #[test] + fn open_target_queue_store_adds_context_on_open_error() { + let base = std::env::temp_dir().join(format!("rustfs-target-store-file-{}", Uuid::new_v4())); + fs::write(&base, b"not-a-directory").expect("failed to create file base"); + let target_id = TargetID::new("target-a".to_string(), ChannelTargetType::Kafka.as_str().to_string()); + + let result = open_target_queue_store( + base.to_str().unwrap(), + 100, + TargetType::NotifyEvent, + ChannelTargetType::Kafka.as_str(), + &target_id, + "custom open context", + ); + + match result { + Ok(_) => panic!("expected open_target_queue_store to fail on file base path"), + Err(err) => assert!(err.to_string().contains("custom open context")), + } + let _ = fs::remove_file(base); + } + + #[test] + fn persist_queued_payload_to_store_writes_encoded_payload() { + let store = MockQueuedStore::new(false); + let meta = QueuedPayloadMeta::new( + EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "obj.txt".to_string(), + "application/json", + 7, + ); + let queued = QueuedPayload::new(meta, br#"{"x":1}"#.to_vec()); + + persist_queued_payload_to_store(&store, &queued).unwrap(); + + let writes = store.writes.lock().expect("mock writes lock poisoned"); + assert_eq!(writes.len(), 1); + let decoded = QueuedPayload::decode(&writes[0]).unwrap(); + assert_eq!(decoded.body, br#"{"x":1}"#); + } + + #[test] + fn persist_queued_payload_to_store_maps_store_error() { + let store = MockQueuedStore::new(true); + let meta = QueuedPayloadMeta::new( + EventName::ObjectCreatedPut, + "bucket-a".to_string(), + "obj.txt".to_string(), + "application/json", + 7, + ); + let queued = QueuedPayload::new(meta, br#"{"x":1}"#.to_vec()); + + let err = persist_queued_payload_to_store(&store, &queued).expect_err("expected put_raw failure"); + assert!(err.to_string().contains("Failed to save event to store")); + } + + #[test] + fn is_connectivity_error_classifies_target_errors() { + assert!(is_connectivity_error(&TargetError::NotConnected)); + assert!(is_connectivity_error(&TargetError::Timeout("timeout".to_string()))); + assert!(is_connectivity_error(&TargetError::Network("network".to_string()))); + assert!(!is_connectivity_error(&TargetError::Storage("storage".to_string()))); + assert!(!is_connectivity_error(&TargetError::Serialization("serialization".to_string()))); + } + + #[tokio::test] + async fn invalidate_cache_on_connectivity_error_only_runs_for_connectivity_failures() { + let marker = Arc::new(AtomicBool::new(false)); + invalidate_cache_on_connectivity_error(&TargetError::NotConnected, { + let marker = Arc::clone(&marker); + move || async move { + marker.store(true, Ordering::SeqCst); + } + }) + .await; + assert!(marker.load(Ordering::SeqCst)); + + marker.store(false, Ordering::SeqCst); + invalidate_cache_on_connectivity_error(&TargetError::Request("request failed".to_string()), { + let marker = Arc::clone(&marker); + move || async move { + marker.store(true, Ordering::SeqCst); + } + }) + .await; + assert!(!marker.load(Ordering::SeqCst)); + } + + #[test] + fn mark_target_disconnected_on_connectivity_error_only_marks_connectivity_failures() { + let connected = AtomicBool::new(true); + mark_target_disconnected_on_connectivity_error(&connected, &TargetError::Timeout("timeout".to_string())); + assert!(!connected.load(Ordering::SeqCst)); + + connected.store(true, Ordering::SeqCst); + mark_target_disconnected_on_connectivity_error(&connected, &TargetError::Request("request failed".to_string())); + assert!(connected.load(Ordering::SeqCst)); + } + #[test] fn queued_payload_decode_rejects_invalid_magic() { let err = QueuedPayload::decode(b"bad-payload").unwrap_err(); diff --git a/crates/targets/src/target/mqtt.rs b/crates/targets/src/target/mqtt.rs index bf820fbe6..38cf5ea62 100644 --- a/crates/targets/src/target/mqtt.rs +++ b/crates/targets/src/target/mqtt.rs @@ -13,13 +13,14 @@ // limitations under the License. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload_with_records, mark_target_disconnected_on_connectivity_error, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -37,7 +38,7 @@ use serde::de::DeserializeOwned; use std::sync::Arc; use std::{ marker::PhantomData, - path::{Path, PathBuf}, + path::Path, sync::atomic::{AtomicBool, Ordering}, time::Duration, }; @@ -502,30 +503,14 @@ where pub fn new(id: String, args: MQTTArgs) -> Result { args.validate()?; let target_id = TargetID::new(id, ChannelTargetType::Mqtt.as_str().to_string()); - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let unique_dir_name = queue_store_subdir_name(ChannelTargetType::Mqtt.as_str(), &target_id.id); - // Ensure the directory name is valid for filesystem - let specific_queue_path = base_path.join(unique_dir_name); - debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target"); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!( - target_id = %target_id, - error = %e, - "Failed to open store for MQTT target" - ); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Mqtt.as_str(), + &target_id, + "Failed to open store for MQTT target", + )?; let (cancel_tx, cancel_rx) = mpsc::channel(1); let bg_task_manager = Arc::new(BgTaskManager { @@ -631,25 +616,7 @@ where } fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - - let key = format!("{}/{}", event.bucket_name, object_name); - - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.clone()], - }; - - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - Ok(QueuedPayload::new(meta, body)) + build_queued_payload_with_records(event, vec![event.clone()]) } #[instrument(skip(self, body, meta), fields(target_id = %self.id))] @@ -673,9 +640,10 @@ where .await .map_err(|e| { if e.to_string().contains("Connection") || e.to_string().contains("Timeout") { - self.connected.store(false, Ordering::SeqCst); warn!(target_id = %self.id, error = %e, "Publish failed due to connection issue, marking as not connected."); - TargetError::NotConnected + let err = TargetError::NotConnected; + mark_target_disconnected_on_connectivity_error(&self.connected, &err); + err } else { TargetError::Request(format!("Failed to publish message: {e}")) } @@ -899,14 +867,7 @@ where if let Some(store) = &self.store { debug!(target_id = %self.id, "Event saved to store start"); - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - match store.put_raw(&encoded) { + match persist_queued_payload_to_store(store.as_ref(), &queued) { Ok(_) => { debug!(target_id = %self.id, "Event saved to store for MQTT target successfully."); Ok(()) @@ -914,7 +875,7 @@ where Err(e) => { error!(target_id = %self.id, error = %e, "Failed to save event to store"); self.delivery_counters.record_final_failure(); - Err(TargetError::Storage(format!("Failed to save event to store: {e}"))) + Err(e) } } } else { diff --git a/crates/targets/src/target/mysql.rs b/crates/targets/src/target/mysql.rs index fe6135179..13656a717 100644 --- a/crates/targets/src/target/mysql.rs +++ b/crates/targets/src/target/mysql.rs @@ -16,15 +16,15 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload, delete_stored_payload, queue_store_subdir_name, + TargetType, build_queued_payload, delete_stored_payload, is_connectivity_error, open_target_queue_store, + persist_queued_payload_to_store, }, }; use async_trait::async_trait; use mysql_async::{Conn, Opts, OptsBuilder, Pool, PoolConstraints, PoolOpts, SslOpts, prelude::Queryable}; -use rustfs_config::notify::NOTIFY_STORE_EXTENSION; use rustfs_config::{MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY}; use serde::Serialize; use serde::de::DeserializeOwned; @@ -494,25 +494,14 @@ where let target_id = TargetID::new(id, ChannelTargetType::MySql.as_str().to_string()); - // If `queue_dir` is non-empty, a `QueueStore` is created for persistent at-least-once delivery. - let queue_store = if !args.queue_dir.is_empty() { - let queue_dir = - PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::MySql.as_str(), &target_id.id)); - - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION, - }; - - let store = QueueStore::::new(queue_dir, args.queue_limit, extension); - if let Err(e) = store.open() { - return Err(TargetError::Storage(format!("Failed to open MySQL queue store: {e}"))); - } - - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::MySql.as_str(), + &target_id, + "Failed to open MySQL queue store", + )?; info!(target_id = %target_id.id, table = %args.table, "MySQL target created"); @@ -733,18 +722,9 @@ where }; if let Some(store) = &self.store { - // persist the event to a local queue before attempting to insert into MySQL. This will allow us to guarantee at-least-once delivery even if the database is temporarily unreachable or if the process crashes after acknowledging receipt but before writing to the database. - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } debug!("Event saved to queue store for MySQL target: {}", self.id); @@ -789,12 +769,8 @@ where } if let Err(e) = self.insert_event(&body, &meta).await { - if matches!(e, TargetError::NotConnected) { + if is_connectivity_error(&e) { warn!(target_id = %self.id, "MySQL not reachable, event remains in queue store"); - return Err(TargetError::NotConnected); - } - if matches!(e, TargetError::Timeout(_)) { - warn!(target_id = %self.id, "MySQL timeout, event remains in queue store"); return Err(e); } error!(target_id = %self.id, error = %e, "Failed to send event from store"); diff --git a/crates/targets/src/target/nats.rs b/crates/targets/src/target/nats.rs index fdde8e3f4..943fb1dff 100644 --- a/crates/targets/src/target/nats.rs +++ b/crates/targets/src/target/nats.rs @@ -13,13 +13,13 @@ // limitations under the License. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -30,7 +30,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; -use tracing::{error, info, instrument}; +use tracing::{info, instrument}; #[derive(Debug, Clone)] pub struct NATSArgs { @@ -195,22 +195,14 @@ where pub fn new(id: String, args: NATSArgs) -> Result { args.validate()?; let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string()); - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Nats.as_str(), &target_id.id)); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!(target_id = %target_id, error = %e, "Failed to open store for NATS target"); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Nats.as_str(), + &target_id, + "Failed to open store for NATS target", + )?; Ok(Self { id: target_id, @@ -241,22 +233,7 @@ where } fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.clone()], - }; - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - Ok(QueuedPayload::new(meta, body)) + build_queued_payload_with_records(event, vec![event.clone()]) } async fn send_body(&self, body: Vec) -> Result<(), TargetError> { @@ -298,16 +275,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } Ok(()) } else { diff --git a/crates/targets/src/target/postgres.rs b/crates/targets/src/target/postgres.rs index 7a474efb8..51a4cca64 100644 --- a/crates/targets/src/target/postgres.rs +++ b/crates/targets/src/target/postgres.rs @@ -29,10 +29,10 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload, queue_store_subdir_name, + TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -44,11 +44,11 @@ use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; use std::io::BufReader; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use tokio_postgres::Config; use tokio_postgres_rustls::MakeRustlsConnect; -use tracing::{error, info, instrument, warn}; +use tracing::{info, instrument, warn}; use url::Url; use uuid::Uuid; @@ -585,23 +585,14 @@ where let target_id = TargetID::new(id, ChannelTargetType::Postgres.as_str().to_string()); let pool = build_pool(&args)?; - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = - base_path.join(queue_store_subdir_name(ChannelTargetType::Postgres.as_str(), &target_id.id)); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!(target_id = %target_id, error = %e, "Failed to open store for PostgreSQL target"); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Postgres.as_str(), + &target_id, + "Failed to open store for PostgreSQL target", + )?; Ok(Self { id: target_id, @@ -712,16 +703,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } Ok(()) } else { diff --git a/crates/targets/src/target/pulsar.rs b/crates/targets/src/target/pulsar.rs index 07a816a4a..d3d8a9403 100644 --- a/crates/targets/src/target/pulsar.rs +++ b/crates/targets/src/target/pulsar.rs @@ -13,24 +13,24 @@ // limitations under the License. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload_with_records, open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; use pulsar::{Authentication, Producer, Pulsar, TokioExecutor}; use serde::Serialize; use serde::de::DeserializeOwned; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use tokio::sync::Mutex as AsyncMutex; -use tracing::{error, info, instrument}; +use tracing::{info, instrument}; use url::Url; #[derive(Debug, Clone)] @@ -186,22 +186,14 @@ where pub fn new(id: String, args: PulsarArgs) -> Result { args.validate()?; let target_id = TargetID::new(id, ChannelTargetType::Pulsar.as_str().to_string()); - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Pulsar.as_str(), &target_id.id)); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!(target_id = %target_id, error = %e, "Failed to open store for Pulsar target"); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Pulsar.as_str(), + &target_id, + "Failed to open store for Pulsar target", + )?; Ok(Self { id: target_id, @@ -249,22 +241,7 @@ where } fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.clone()], - }; - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - Ok(QueuedPayload::new(meta, body)) + build_queued_payload_with_records(event, vec![event.clone()]) } async fn send_body(&self, body: Vec) -> Result<(), TargetError> { @@ -317,16 +294,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } Ok(()) } else { diff --git a/crates/targets/src/target/redis.rs b/crates/targets/src/target/redis.rs index d8fafa99a..c5e3508bf 100644 --- a/crates/targets/src/target/redis.rs +++ b/crates/targets/src/target/redis.rs @@ -16,10 +16,11 @@ use crate::{ StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, build_queued_payload, queue_store_subdir_name, + TargetType, build_queued_payload, invalidate_cache_on_connectivity_error, is_connectivity_error, + mark_target_disconnected_on_connectivity_error, open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; @@ -33,12 +34,12 @@ use rustfs_config::{REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, R use serde::Serialize; use serde::de::DeserializeOwned; use std::fmt; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::sync::Mutex; -use tracing::{debug, error, info, instrument, warn}; +use tracing::{debug, info, instrument, warn}; use url::Url; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -320,22 +321,14 @@ where let target_id = TargetID::new(id, ChannelTargetType::Redis.as_str().to_string()); let publisher_client = build_redis_client(&args)?; - let queue_store = if !args.queue_dir.is_empty() { - let base_path = PathBuf::from(&args.queue_dir); - let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Redis.as_str(), &target_id.id)); - let extension = match args.target_type { - TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION, - }; - let store = QueueStore::::new(specific_queue_path, args.queue_limit, extension); - if let Err(e) = store.open() { - error!(target_id = %target_id, error = %e, "Failed to open store for Redis target"); - return Err(TargetError::Storage(format!("{e}"))); - } - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Redis.as_str(), + &target_id, + "Failed to open store for Redis target", + )?; info!(target_id = %target_id, "Redis target created"); Ok(Self { @@ -391,9 +384,7 @@ where Ok(_) => Ok(()), Err(err) => { let mapped = map_redis_error(err); - if is_retryable_target_error(&mapped) { - self.invalidate_cached_publisher().await; - } + invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await; Err(mapped) } } @@ -437,9 +428,7 @@ where } Err(err) => { let mapped = map_redis_error(err); - if is_retryable_target_error(&mapped) { - self.invalidate_cached_publisher().await; - } + invalidate_cache_on_connectivity_error(&mapped, || self.invalidate_cached_publisher()).await; warn!( target_id = %self.id, @@ -450,7 +439,7 @@ where "Redis publish attempt failed" ); - if !is_retryable_target_error(&mapped) || attempt >= self.args.max_retry_attempts { + if !is_connectivity_error(&mapped) || attempt >= self.args.max_retry_attempts { last_error = Some(mapped); break; } @@ -492,14 +481,15 @@ where Ok(true) } Ok(Err(err)) => { - self.invalidate_cached_publisher().await; - self.connected.store(false, Ordering::SeqCst); + invalidate_cache_on_connectivity_error(&err, || self.invalidate_cached_publisher()).await; + mark_target_disconnected_on_connectivity_error(&self.connected, &err); Err(err) } Err(_) => { - self.invalidate_cached_publisher().await; - self.connected.store(false, Ordering::SeqCst); - Err(TargetError::Timeout("Redis connection timed out".to_string())) + let timeout_err = TargetError::Timeout("Redis connection timed out".to_string()); + invalidate_cache_on_connectivity_error(&timeout_err, || self.invalidate_cached_publisher()).await; + mark_target_disconnected_on_connectivity_error(&self.connected, &timeout_err); + Err(timeout_err) } } } @@ -514,17 +504,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } debug!(target_id = %self.id, "Event saved to store for Redis target"); @@ -556,14 +538,14 @@ where } if let Err(err) = self.init_inner().await { - if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) { + if is_connectivity_error(&err) { warn!(target_id = %self.id, error = %err, "Redis target not ready; queued event remains in store"); } return Err(err); } if let Err(err) = self.send_body(body, &meta).await { - if matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) { + if is_connectivity_error(&err) { warn!(target_id = %self.id, error = %err, "Failed to send Redis event from store: target not connected. Event remains queued."); } return Err(err); @@ -734,10 +716,6 @@ fn map_redis_error(err: RedisError) -> TargetError { } } -fn is_retryable_target_error(err: &TargetError) -> bool { - matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_)) -} - fn compute_retry_delay(attempt: usize, min_delay: Duration, max_delay: Duration) -> Duration { let shift = attempt.saturating_sub(1).min(16) as u32; let factor = 1u32 << shift; diff --git a/crates/targets/src/target/webhook.rs b/crates/targets/src/target/webhook.rs index de632b7e5..03cd4e615 100644 --- a/crates/targets/src/target/webhook.rs +++ b/crates/targets/src/target/webhook.rs @@ -13,24 +13,21 @@ // limitations under the License. use crate::{ - StoreError, Target, TargetLog, + StoreError, Target, arn::TargetID, error::TargetError, - store::{Key, QueueStore, Store}, + store::{Key, Store}, target::{ ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot, - TargetType, queue_store_subdir_name, + TargetType, build_queued_payload, open_target_queue_store, persist_queued_payload_to_store, }, }; use async_trait::async_trait; use reqwest::{Client, StatusCode, Url}; -use rustfs_config::audit::AUDIT_STORE_EXTENSION; -use rustfs_config::notify::NOTIFY_STORE_EXTENSION; use serde::Serialize; use serde::de::DeserializeOwned; use std::{ marker::PhantomData, - path::PathBuf, sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -151,28 +148,14 @@ where // Build HTTP client using the helper function let http_client = Arc::new(Self::build_http_client(&args)?); - // Build storage - let queue_store = if !args.queue_dir.is_empty() { - let queue_dir = - PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Webhook.as_str(), &target_id.id)); - - let extension = match args.target_type { - TargetType::AuditLog => AUDIT_STORE_EXTENSION, - TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION, - }; - - let store = QueueStore::::new(queue_dir, args.queue_limit, extension); - - if let Err(e) = store.open() { - error!("Failed to open store for Webhook target {}: {}", target_id.id, e); - return Err(TargetError::Storage(format!("{e}"))); - } - - // Make sure that the Store trait implemented by QueueStore matches the expected error type - Some(Box::new(store) as Box + Send + Sync>) - } else { - None - }; + let queue_store = open_target_queue_store( + &args.queue_dir, + args.queue_limit, + args.target_type, + ChannelTargetType::Webhook.as_str(), + &target_id, + "Failed to open store for Webhook target", + )?; // Create a cancel channel let (cancel_sender, _) = mpsc::channel(1); @@ -302,22 +285,7 @@ where } fn build_queued_payload(&self, event: &EntityTarget) -> Result { - let object_name = crate::target::decode_object_name(&event.object_name)?; - let key = format!("{}/{}", event.bucket_name, object_name); - let log = TargetLog { - event_name: event.event_name, - key, - records: vec![event.data.clone()], - }; - let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?; - let meta = QueuedPayloadMeta::new( - event.event_name, - event.bucket_name.clone(), - event.object_name.clone(), - "application/json", - body.len(), - ); - Ok(QueuedPayload::new(meta, body)) + build_queued_payload(event) } async fn send_body(&self, body: Vec, meta: &QueuedPayloadMeta) -> Result<(), TargetError> { @@ -408,16 +376,9 @@ where }; if let Some(store) = &self.store { - let encoded = match queued.encode() { - Ok(encoded) => encoded, - Err(err) => { - self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}"))); - } - }; - if let Err(e) = store.put_raw(&encoded) { + if let Err(e) = persist_queued_payload_to_store(store.as_ref(), &queued) { self.delivery_counters.record_final_failure(); - return Err(TargetError::Storage(format!("Failed to save event to store: {e}"))); + return Err(e); } debug!("Event saved to store for target: {}", self.id); Ok(()) diff --git a/rustfs/src/admin/handlers/audit.rs b/rustfs/src/admin/handlers/audit.rs index 33efc11de..a25e6b07f 100644 --- a/rustfs/src/admin/handlers/audit.rs +++ b/rustfs/src/admin/handlers/audit.rs @@ -14,12 +14,12 @@ use crate::admin::{ auth::validate_admin_request, + handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload}, handlers::target_descriptor::{ - AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys, - build_json_response, collect_validated_key_values as shared_collect_validated_key_values, + AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs, + build_json_response, collect_runtime_statuses, extract_supported_target_params, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, - target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, - validate_target_request, + target_mutation_block_reason as shared_target_mutation_block_reason, }, router::{AdminOperation, Operation, S3Router}, }; @@ -27,23 +27,19 @@ use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store, }; -use futures::stream::{FuturesUnordered, StreamExt}; use http::StatusCode; use hyper::Method; use matchit::Params; -use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors; -use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; +use rustfs_audit::audit_system; use rustfs_config::audit::AUDIT_ROUTE_PREFIX; -use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_config::{AUDIT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::collections::HashMap; use std::sync::LazyLock; -use tokio::sync::Semaphore; -use tokio::time::{Duration, timeout}; use tracing::{Span, warn}; pub fn register_audit_target_route(r: &mut S3Router) -> std::io::Result<()> { @@ -93,7 +89,7 @@ struct AuditEndpointsResponse { } static AUDIT_TARGET_SPECS: LazyLock> = LazyLock::new(|| { - builtin_audit_target_descriptors() + builtin_audit_target_admin_descriptors() .into_iter() .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) .collect() @@ -113,18 +109,6 @@ async fn authorize_audit_admin_request(req: &S3Request, action: AdminActio validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await } -fn has_any_audit_targets(config: &Config) -> bool { - for spec in audit_target_specs() { - let Some(targets) = config.0.get(spec.subsystem) else { - continue; - }; - if targets.keys().any(|key| key != DEFAULT_DELIMITER) { - return true; - } - } - false -} - fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { shared_target_mutation_block_reason( audit_target_specs(), @@ -160,85 +144,7 @@ fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> { - let target_type = params - .get("target_type") - .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?; - if target_service_name(audit_target_specs(), target_type).is_none() { - return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type)); - } - let target_name = params - .get("target_name") - .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?; - Ok((target_type, target_name)) -} - -async fn load_server_config_from_store() -> S3Result { - let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { - return Ok(Config::new()); - }; - - rustfs_ecstore::config::com::read_config_without_migrate(store) - .await - .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e)) -} - -async fn apply_audit_runtime_config(config: Config) -> S3Result<()> { - let has_targets = has_any_audit_targets(&config); - - if let Some(system) = audit_system() { - match system.get_state().await { - AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { - if has_targets { - system - .reload_config(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?; - } else { - system - .close() - .await - .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?; - } - } - AuditSystemState::Stopped | AuditSystemState::Stopping => { - if has_targets { - system - .start(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; - } - } - } - } else if has_targets { - start_global_audit_system(config) - .await - .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; - } - - Ok(()) -} - -async fn update_audit_config_and_reload(mut modifier: F) -> S3Result<()> -where - F: FnMut(&mut Config) -> bool, -{ - let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { - return Err(s3_error!(InternalError, "server storage not initialized")); - }; - - let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) - .await - .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; - - if !modifier(&mut config) { - return Ok(()); - } - - rustfs_ecstore::config::com::save_server_config(store, &config) - .await - .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; - - apply_audit_runtime_config(config).await + extract_supported_target_params(audit_target_specs(), params, "audit") } pub struct AuditTargetConfig {} @@ -269,28 +175,16 @@ impl Operation for AuditTargetConfig { .map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?; let specs = audit_target_specs(); - let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type); - - let kv_map = shared_collect_validated_key_values( + let kvs = build_enabled_target_kvs( + specs, audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())), - &allowed_keys, target_type, + AUDIT_DEFAULT_DIR, "audit target", - )?; + ) + .await?; - let spec = target_spec(specs, target_type) - .ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?; - timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR)) - .await - .map_err(|_| s3_error!(InvalidArgument, "audit target validation timed out"))??; - - let mut kvs = rustfs_ecstore::config::KVS::new(); - for (key, value) in kv_map { - kvs.insert(key, value); - } - kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); - - update_audit_config_and_reload(|config| { + update_audit_config_and_reload(audit_target_specs(), |config| { config .0 .entry(target_type.to_lowercase()) @@ -315,25 +209,7 @@ impl Operation for ListAuditTargets { let mut runtime_statuses = HashMap::new(); if let Some(system) = audit_system() { - let targets = system.get_target_values().await; - let semaphore = Arc::new(Semaphore::new(10)); - let mut futures = FuturesUnordered::new(); - - for target in targets { - let sem = Arc::clone(&semaphore); - futures.push(async move { - let _permit = sem.acquire().await; - let status = match timeout(Duration::from_secs(3), target.is_active()).await { - Ok(Ok(true)) => "online", - _ => "offline", - }; - ((target.id().id, target.id().name), status.to_string()) - }); - } - - while let Some((key, status)) = futures.next().await { - runtime_statuses.insert(key, status); - } + runtime_statuses = collect_runtime_statuses(system.get_target_values().await).await; } let config = load_server_config_from_store().await?; @@ -363,7 +239,7 @@ impl Operation for RemoveAuditTarget { return Err(s3_error!(InvalidRequest, "{reason}")); } - update_audit_config_and_reload(|config| { + update_audit_config_and_reload(audit_target_specs(), |config| { let mut changed = false; if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) { if targets.remove(&target_name.to_lowercase()).is_some() { @@ -384,9 +260,10 @@ impl Operation for RemoveAuditTarget { #[cfg(test)] mod tests { use super::*; + use crate::admin::handlers::target_descriptor::collect_validated_key_values as shared_collect_validated_key_values; use matchit::Router; - use rustfs_config::ENV_PREFIX; use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS}; + use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX}; use rustfs_ecstore::config::{KV, KVS}; use serial_test::serial; use std::collections::{HashMap, HashSet}; diff --git a/rustfs/src/admin/handlers/audit_runtime_config.rs b/rustfs/src/admin/handlers/audit_runtime_config.rs new file mode 100644 index 000000000..1d9dbeeea --- /dev/null +++ b/rustfs/src/admin/handlers/audit_runtime_config.rs @@ -0,0 +1,130 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::handlers::target_descriptor::AdminTargetSpec; +use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState}; +use rustfs_config::DEFAULT_DELIMITER; +use rustfs_ecstore::config::Config; +use s3s::{S3Result, s3_error}; + +pub(crate) async fn load_server_config_from_store() -> S3Result { + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { + return Ok(Config::new()); + }; + + rustfs_ecstore::config::com::read_config_without_migrate(store) + .await + .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e)) +} + +fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool { + specs.iter().any(|spec| { + config + .0 + .get(spec.subsystem) + .is_some_and(|targets| targets.keys().any(|key| key != DEFAULT_DELIMITER)) + }) +} + +pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> { + let has_targets = has_any_audit_targets(specs, &config); + + if let Some(system) = audit_system() { + match system.get_state().await { + AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { + if has_targets { + system + .reload_config(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?; + } else { + system + .close() + .await + .map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?; + } + } + AuditSystemState::Stopped | AuditSystemState::Stopping => { + if has_targets { + system + .start(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + } + } + } + } else if has_targets { + start_global_audit_system(config) + .await + .map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?; + } + + Ok(()) +} + +pub(crate) async fn update_audit_config_and_reload(specs: &[AdminTargetSpec], mut modifier: F) -> S3Result<()> +where + F: FnMut(&mut Config) -> bool, +{ + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { + return Err(s3_error!(InternalError, "server storage not initialized")); + }; + + let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) + .await + .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; + + if !modifier(&mut config) { + return Ok(()); + } + + rustfs_ecstore::config::com::save_server_config(store, &config) + .await + .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; + + apply_audit_runtime_config(specs, config).await +} + +pub(crate) async fn set_audit_target_config( + specs: &[AdminTargetSpec], + subsystem: &str, + target_name: &str, + kvs: rustfs_ecstore::config::KVS, +) -> S3Result<()> { + update_audit_config_and_reload(specs, |config| { + config + .0 + .entry(subsystem.to_lowercase()) + .or_default() + .insert(target_name.to_lowercase(), kvs.clone()); + true + }) + .await +} + +pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> { + update_audit_config_and_reload(specs, |config| { + let mut changed = false; + if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) { + if targets.remove(&target_name.to_lowercase()).is_some() { + changed = true; + } + if targets.is_empty() { + config.0.remove(&subsystem.to_lowercase()); + } + } + changed + }) + .await +} diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index b2efde467..c78673ead 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -14,12 +14,12 @@ use crate::admin::{ auth::validate_admin_request, + handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot}, handlers::target_descriptor::{ - AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys, - build_json_response, collect_validated_key_values as shared_collect_validated_key_values, + AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs, + build_json_response, collect_runtime_statuses, extract_supported_target_params, merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason, - target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec, - validate_target_request, + target_mutation_block_reason as shared_target_mutation_block_reason, }, router::{AdminOperation, Operation, S3Router}, }; @@ -28,22 +28,18 @@ use crate::server::{ ADMIN_PREFIX, RemoteAddr, is_notify_module_enabled, refresh_notify_module_enabled, refresh_persisted_module_switches_from_store, }; -use futures::stream::{FuturesUnordered, StreamExt}; use http::StatusCode; use hyper::Method; use matchit::Params; use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; -use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_config::{EVENT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE}; use rustfs_ecstore::config::Config; -use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors; use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_targets::catalog::builtin::builtin_notify_target_admin_descriptors; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::collections::HashMap; use std::sync::LazyLock; -use tokio::sync::Semaphore; -use tokio::time::{Duration, timeout}; use tracing::{Span, info, warn}; pub fn register_notification_target_route(r: &mut S3Router) -> std::io::Result<()> { @@ -99,7 +95,7 @@ struct NotificationEndpointsResponse { } static NOTIFICATION_TARGET_SPECS: LazyLock> = LazyLock::new(|| { - builtin_notification_target_descriptors() + builtin_notify_target_admin_descriptors() .into_iter() .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) .collect() @@ -121,10 +117,6 @@ async fn authorize_notification_admin_request(req: &S3Request, action: Adm validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await } -fn get_notification_system() -> S3Result> { - rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized")) -} - fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option { shared_target_mutation_block_reason( notification_target_specs(), @@ -180,8 +172,7 @@ impl Operation for NotificationTarget { if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await { return Err(s3_error!(InvalidRequest, "{reason}")); } - let ns = get_notification_system()?; - let config_snapshot = ns.config.read().await.clone(); + let (ns, config_snapshot) = load_notification_config_snapshot().await?; if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { return Err(s3_error!(InvalidRequest, "{reason}")); } @@ -196,28 +187,17 @@ impl Operation for NotificationTarget { .map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?; let specs = notification_target_specs(); - let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type); - - let kv_map = shared_collect_validated_key_values( + let kvs = build_enabled_target_kvs( + specs, notification_body .key_values .iter() .map(|kv| (kv.key.as_str(), kv.value.as_str())), - &allowed_keys, target_type, + EVENT_DEFAULT_DIR, "target", - )?; - let spec = target_spec(specs, target_type) - .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?; - timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR)) - .await - .map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??; - - let mut kvs = rustfs_ecstore::config::KVS::new(); - for (key, value) in kv_map { - kvs.insert(key, value); - } - kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + ) + .await?; info!("Setting target config for type '{}', name '{}'", target_type, target_name); ns.set_target_config(target_type, target_name, kvs) @@ -235,29 +215,8 @@ impl Operation for ListNotificationTargets { let span = Span::current(); let _enter = span.enter(); authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?; - let ns = get_notification_system()?; - - let targets = ns.get_target_values().await; - let semaphore = Arc::new(Semaphore::new(10)); - let mut futures = FuturesUnordered::new(); - - for target in targets { - let sem = Arc::clone(&semaphore); - futures.push(async move { - let _permit = sem.acquire().await; - let status = match timeout(Duration::from_secs(3), target.is_active()).await { - Ok(Ok(true)) => "online", - _ => "offline", - }; - ((target.id().id, target.id().name), status.to_string()) - }); - } - - let mut runtime_statuses = HashMap::new(); - while let Some((key, status)) = futures.next().await { - runtime_statuses.insert(key, status); - } - let config = ns.config.read().await.clone(); + let (ns, config) = load_notification_config_snapshot().await?; + let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await; let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses); let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints }) @@ -283,30 +242,15 @@ impl Operation for ListTargetsArns { } let ns = get_notification_system()?; - let targets = ns.get_target_values().await; let region = req .region .clone() .ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?; - let semaphore = Arc::new(Semaphore::new(10)); - let mut futures = FuturesUnordered::new(); - - for target in targets { - let sem = Arc::clone(&semaphore); - futures.push(async move { - let _permit = sem.acquire().await; - let status = match timeout(Duration::from_secs(3), target.is_active()).await { - Ok(Ok(true)) => "online", - _ => "offline", - }; - (target.id(), status.to_string()) - }); - } - - let mut target_statuses = Vec::new(); - while let Some(target_status) = futures.next().await { - target_statuses.push(target_status); - } + let target_statuses = collect_runtime_statuses(ns.get_target_values().await) + .await + .into_iter() + .map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status)) + .collect(); let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses); @@ -329,8 +273,7 @@ impl Operation for RemoveNotificationTarget { if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await { return Err(s3_error!(InvalidRequest, "{reason}")); } - let ns = get_notification_system()?; - let config_snapshot = ns.config.read().await.clone(); + let (ns, config_snapshot) = load_notification_config_snapshot().await?; if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) { return Err(s3_error!(InvalidRequest, "{reason}")); } @@ -344,27 +287,19 @@ impl Operation for RemoveNotificationTarget { } } -fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str> { - params - .get(key) - .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: '{}'", key)) -} - fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> { - let target_type = extract_param(params, "target_type")?; - if target_service_name(notification_target_specs(), target_type).is_none() { - return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type)); - } - let target_name = extract_param(params, "target_name")?; - Ok((target_type, target_name)) + extract_supported_target_params(notification_target_specs(), params, "notification") } #[cfg(test)] mod tests { use super::*; + use crate::admin::handlers::target_descriptor::{ + allowed_target_keys, collect_validated_key_values as shared_collect_validated_key_values, + }; use matchit::Router; - use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS}; + use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY}; use rustfs_ecstore::config::{KV, KVS}; use rustfs_targets::arn::TargetID; use serial_test::serial; diff --git a/rustfs/src/admin/handlers/mod.rs b/rustfs/src/admin/handlers/mod.rs index 93372f4f1..e5285ab0f 100644 --- a/rustfs/src/admin/handlers/mod.rs +++ b/rustfs/src/admin/handlers/mod.rs @@ -14,6 +14,7 @@ pub mod account_info; pub mod audit; +mod audit_runtime_config; pub mod bucket_meta; pub mod event; pub mod group; @@ -27,7 +28,10 @@ pub mod kms_keys; pub mod kms_management; pub mod metrics; pub mod module_switch; +mod notify_runtime_access; pub mod oidc; +pub mod plugins_catalog; +pub mod plugins_instances; pub mod policies; pub mod pools; pub mod profile; @@ -57,6 +61,11 @@ mod tests { let _account_handler = account_info::AccountInfoHandler {}; let _list_audit_targets = audit::ListAuditTargets {}; let _get_module_switches = module_switch::GetModuleSwitchesHandler {}; + let _get_plugin_catalog = plugins_catalog::GetPluginCatalogHandler {}; + let _list_plugin_instances = plugins_instances::ListPluginInstancesHandler {}; + let _get_plugin_instance = plugins_instances::GetPluginInstanceHandler {}; + let _put_plugin_instance = plugins_instances::PutPluginInstanceHandler {}; + let _delete_plugin_instance = plugins_instances::DeletePluginInstanceHandler {}; let _update_module_switches = module_switch::UpdateModuleSwitchesHandler {}; let _service_handler = system::ServiceHandle {}; let _server_info_handler = system::ServerInfoHandler {}; diff --git a/rustfs/src/admin/handlers/notify_runtime_access.rs b/rustfs/src/admin/handlers/notify_runtime_access.rs new file mode 100644 index 000000000..c108ff255 --- /dev/null +++ b/rustfs/src/admin/handlers/notify_runtime_access.rs @@ -0,0 +1,47 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::config::Config; +use s3s::{S3Result, s3_error}; +use std::sync::Arc; + +pub(crate) fn get_notification_system() -> S3Result> { + rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized")) +} + +pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc, Config)> { + let system = get_notification_system()?; + let config = system.config.read().await.clone(); + Ok((system, config)) +} + +pub(crate) async fn set_notification_target_config( + subsystem: &str, + target_name: &str, + kvs: rustfs_ecstore::config::KVS, +) -> S3Result<()> { + let system = get_notification_system()?; + system + .set_target_config(subsystem, target_name, kvs) + .await + .map_err(|e| s3_error!(InternalError, "failed to set notification target config: {}", e)) +} + +pub(crate) async fn remove_notification_target_config(subsystem: &str, target_name: &str) -> S3Result<()> { + let system = get_notification_system()?; + system + .remove_target_config(subsystem, target_name) + .await + .map_err(|e| s3_error!(InternalError, "failed to remove notification target config: {}", e)) +} diff --git a/rustfs/src/admin/handlers/plugins_catalog.rs b/rustfs/src/admin/handlers/plugins_catalog.rs new file mode 100644 index 000000000..5a1fb7890 --- /dev/null +++ b/rustfs/src/admin/handlers/plugins_catalog.rs @@ -0,0 +1,265 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::{ + auth::validate_admin_request, + plugin_contract::{ + PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind, + PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract, + }, + router::{AdminOperation, Operation, S3Router}, +}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use http::{HeaderMap, HeaderValue, StatusCode}; +use hyper::Method; +use matchit::Params; +use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_targets::catalog::{ + builtin::builtin_audit_target_admin_descriptors, builtin::builtin_notify_target_admin_descriptors, +}; +use rustfs_targets::{ + BuiltinTargetAdminDescriptor, builtin_target_marketplace_manifest, builtin_target_plugin_installation, + catalog::example_external_webhook_plugin, +}; +use s3s::header::CONTENT_TYPE; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; +use serde::Serialize; +use std::collections::HashMap; + +pub fn register_plugin_catalog_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v4/plugins/catalog").as_str(), + AdminOperation(&GetPluginCatalogHandler {}), + )?; + + Ok(()) +} + +fn target_domain_name_from_subsystem(subsystem: &str) -> PluginContractDomain { + if subsystem.starts_with("audit_") { + PluginContractDomain::Audit + } else { + PluginContractDomain::Notify + } +} + +fn build_catalog_response() -> PluginCatalogResponse { + let mut plugins: HashMap<&'static str, PluginCatalogEntry> = HashMap::new(); + + for descriptor in builtin_notify_target_admin_descriptors() + .into_iter() + .chain(builtin_audit_target_admin_descriptors()) + { + merge_catalog_descriptor(&mut plugins, &descriptor); + } + + let mut plugins = plugins.into_values().collect::>(); + plugins.push(example_external_webhook_plugin_entry()); + plugins.sort_by(|a, b| a.target_type.cmp(&b.target_type)); + for plugin in &mut plugins { + plugin.supported_domains.sort(); + plugin.domain_configs.sort_by_key(|a| a.domain); + } + + PluginCatalogResponse { plugins } +} + +fn example_external_webhook_plugin_entry() -> PluginCatalogEntry { + let example = example_external_webhook_plugin(); + let manifest = example.manifest; + + PluginCatalogEntry { + plugin_id: manifest.plugin_id.to_string(), + target_type: manifest.target_type.to_string(), + display_name: manifest.display_name.to_string(), + provider: manifest.provider.to_string(), + version: manifest.version.to_string(), + packaging: PluginContractPackaging::from(manifest.packaging), + entrypoint_kind: PluginContractEntrypointKind::from(manifest.entrypoint_kind), + api_compatibility_version: manifest.api_compatibility_version.to_string(), + runtime_contract: PluginRuntimeContract::from(manifest.runtime_contract), + distribution: manifest.distribution.map(PluginDistributionContract::from), + supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(), + secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(), + domain_configs: vec![PluginCatalogDomainEntry { + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook".to_string(), + valid_fields: example.valid_fields, + }], + installation: Some(example.installation.into()), + } +} + +fn merge_catalog_descriptor(plugins: &mut HashMap<&'static str, PluginCatalogEntry>, descriptor: &BuiltinTargetAdminDescriptor) { + let manifest = descriptor.manifest(); + let marketplace = builtin_target_marketplace_manifest(manifest.target_type); + let domain = target_domain_name_from_subsystem(descriptor.admin_metadata().subsystem()); + let domain_entry = PluginCatalogDomainEntry { + domain, + subsystem: descriptor.admin_metadata().subsystem().to_string(), + valid_fields: descriptor.valid_fields().iter().map(|field| (*field).to_string()).collect(), + }; + + let entry = plugins.entry(manifest.plugin_id).or_insert_with(|| PluginCatalogEntry { + plugin_id: manifest.plugin_id.to_string(), + target_type: manifest.target_type.to_string(), + display_name: manifest.display_name.to_string(), + provider: manifest.provider.to_string(), + version: manifest.version.to_string(), + packaging: PluginContractPackaging::from(marketplace.packaging), + entrypoint_kind: PluginContractEntrypointKind::from(marketplace.entrypoint_kind), + api_compatibility_version: marketplace.api_compatibility_version.to_string(), + runtime_contract: PluginRuntimeContract::from(marketplace.runtime_contract), + distribution: marketplace.distribution.map(PluginDistributionContract::from), + supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(), + secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(), + domain_configs: Vec::new(), + installation: Some(builtin_target_plugin_installation(manifest).into()), + }); + + if !entry.domain_configs.iter().any(|existing| existing.domain == domain) { + entry.domain_configs.push(domain_entry); + } +} + +async fn authorize_plugin_catalog_request(req: &S3Request) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await +} + +fn build_json_response( + status: StatusCode, + body: &impl Serialize, + request_id: Option<&HeaderValue>, +) -> S3Result> { + let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + let mut header = HeaderMap::new(); + header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + if let Some(value) = request_id { + header.insert("x-request-id", value.clone()); + } + Ok(S3Response::with_headers((status, Body::from(data)), header)) +} + +pub struct GetPluginCatalogHandler {} + +#[async_trait::async_trait] +impl Operation for GetPluginCatalogHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_plugin_catalog_request(&req).await?; + build_json_response(StatusCode::OK, &build_catalog_response(), req.headers.get("x-request-id")) + } +} + +#[cfg(test)] +mod tests { + use super::build_catalog_response; + use crate::admin::plugin_contract::{ + PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginRuntimeTransport, + }; + + #[test] + fn plugin_catalog_handlers_require_admin_authorization_contract() { + let src = include_str!("plugins_catalog.rs"); + let handler_block = extract_block_between_markers(src, "impl Operation for GetPluginCatalogHandler", "#[cfg(test)]"); + + assert!( + handler_block.contains("authorize_plugin_catalog_request(&req).await?;"), + "plugin catalog GET should require admin authorization" + ); + } + + #[test] + fn plugin_catalog_contains_representative_builtin_targets() { + let response = build_catalog_response(); + + let webhook = response + .plugins + .iter() + .find(|plugin| plugin.plugin_id == "builtin:webhook") + .expect("builtin webhook plugin should be present"); + assert_eq!(webhook.target_type, "webhook"); + assert_eq!(webhook.display_name, "Webhook"); + assert_eq!(webhook.packaging, PluginContractPackaging::Builtin); + assert_eq!(webhook.entrypoint_kind, PluginContractEntrypointKind::Builtin); + assert_eq!(webhook.api_compatibility_version, "rustfs.target-plugin.v1"); + assert_eq!(webhook.runtime_contract.protocol_version, "rustfs.target-runtime.v1"); + assert_eq!(webhook.runtime_contract.transport, PluginRuntimeTransport::InProcess); + assert_eq!(webhook.distribution, None); + assert!(webhook.supported_domains.contains(&PluginContractDomain::Audit)); + assert!(webhook.supported_domains.contains(&PluginContractDomain::Notify)); + assert!( + webhook + .domain_configs + .iter() + .any(|domain| domain.subsystem == "audit_webhook") + ); + assert!( + webhook + .domain_configs + .iter() + .any(|domain| domain.subsystem == "notify_webhook") + ); + + let kafka = response + .plugins + .iter() + .find(|plugin| plugin.plugin_id == "builtin:kafka") + .expect("builtin kafka plugin should be present"); + assert_eq!(kafka.target_type, "kafka"); + assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "audit_kafka")); + assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "notify_kafka")); + } + + #[test] + fn plugin_catalog_exposes_secret_fields_only_as_metadata() { + let response = build_catalog_response(); + let webhook = response + .plugins + .iter() + .find(|plugin| plugin.plugin_id == "builtin:webhook") + .expect("builtin webhook plugin should be present"); + + assert!(webhook.secret_fields.contains(&"auth_token".to_string())); + assert!(!webhook.secret_fields.iter().any(|field| field.contains("https://"))); + assert!(!webhook.secret_fields.iter().any(|field| field.contains("password="))); + } + + fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str { + let start = src + .find(start_marker) + .unwrap_or_else(|| panic!("Expected marker `{start_marker}` in source")); + let after_start = &src[start..]; + let end = after_start + .find(end_marker) + .unwrap_or_else(|| panic!("Expected end marker `{end_marker}` in source")); + &after_start[..end] + } +} diff --git a/rustfs/src/admin/handlers/plugins_instances.rs b/rustfs/src/admin/handlers/plugins_instances.rs new file mode 100644 index 000000000..aa2797dc8 --- /dev/null +++ b/rustfs/src/admin/handlers/plugins_instances.rs @@ -0,0 +1,1630 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::admin::{ + auth::validate_admin_request, + handlers::audit_runtime_config::{load_server_config_from_store, remove_audit_target_config, set_audit_target_config}, + handlers::notify_runtime_access::{ + load_notification_config_snapshot, remove_notification_target_config, set_notification_target_config, + }, + handlers::target_descriptor::{ + AdminTargetSpec, TargetEndpointSource, TargetInstanceReadModel, admin_target_spec_from_builtin, build_enabled_target_kvs, + build_json_response, collect_runtime_statuses, collect_target_instances, find_target_instance, + target_module_disabled_reason, target_mutation_block_reason as shared_target_mutation_block_reason, + }, + plugin_contract::{ + PluginContractDomain, PluginInstanceDetail, PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, + PluginInstanceDiagnosticCount, PluginInstanceEntry, PluginInstanceSource, PluginInstancesResponse, + }, + router::{AdminOperation, Operation, S3Router}, +}; +use crate::auth::{check_key_valid, get_session_token}; +use crate::server::{ + ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, is_notify_module_enabled, refresh_audit_module_enabled, + refresh_notify_module_enabled, refresh_persisted_module_switches_from_store, +}; +use hyper::{Method, StatusCode}; +use matchit::Params; +use rustfs_audit::audit_system; +use rustfs_config::audit::AUDIT_ROUTE_PREFIX; +use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; +use rustfs_config::{AUDIT_DEFAULT_DIR, EVENT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE}; +use rustfs_ecstore::config::{Config, KVS}; +use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_targets::catalog::builtin::{builtin_audit_target_admin_descriptors, builtin_notify_target_admin_descriptors}; +use rustfs_targets::manifest::builtin_target_manifest; +use rustfs_targets::{builtin_target_plugin_operational_state, runtime_state_from_status_label}; +use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::LazyLock; +use tracing::warn; +use url::form_urlencoded; + +pub fn register_plugin_instance_route(r: &mut S3Router) -> std::io::Result<()> { + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v4/plugins/instances").as_str(), + AdminOperation(&ListPluginInstancesHandler {}), + )?; + r.insert( + Method::GET, + format!("{}{}", ADMIN_PREFIX, "/v4/plugins/instances/{id}").as_str(), + AdminOperation(&GetPluginInstanceHandler {}), + )?; + r.insert( + Method::PUT, + format!("{}{}", ADMIN_PREFIX, "/v4/plugins/instances/{id}").as_str(), + AdminOperation(&PutPluginInstanceHandler {}), + )?; + r.insert( + Method::DELETE, + format!("{}{}", ADMIN_PREFIX, "/v4/plugins/instances/{id}").as_str(), + AdminOperation(&DeletePluginInstanceHandler {}), + )?; + + Ok(()) +} + +static NOTIFICATION_TARGET_SPECS: LazyLock> = LazyLock::new(|| { + builtin_notify_target_admin_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() +}); + +static AUDIT_TARGET_SPECS: LazyLock> = LazyLock::new(|| { + builtin_audit_target_admin_descriptors() + .into_iter() + .map(|descriptor| admin_target_spec_from_builtin(&descriptor)) + .collect() +}); + +fn notification_target_specs() -> &'static [AdminTargetSpec] { + &NOTIFICATION_TARGET_SPECS +} + +fn audit_target_specs() -> &'static [AdminTargetSpec] { + &AUDIT_TARGET_SPECS +} + +const REDACTED_SECRET_VALUE: &str = "***redacted***"; + +fn builtin_secret_fields_for_service(plugin_id: &str, service: &str) -> &'static [&'static str] { + if !plugin_id.starts_with("builtin:") { + return &[]; + } + + match service.to_ascii_lowercase().as_str() { + "webhook" => builtin_target_manifest("webhook").secret_fields, + "mqtt" => builtin_target_manifest("mqtt").secret_fields, + "kafka" => builtin_target_manifest("kafka").secret_fields, + "amqp" => builtin_target_manifest("amqp").secret_fields, + "nats" => builtin_target_manifest("nats").secret_fields, + "pulsar" => builtin_target_manifest("pulsar").secret_fields, + "mysql" => builtin_target_manifest("mysql").secret_fields, + "redis" => builtin_target_manifest("redis").secret_fields, + "postgres" => builtin_target_manifest("postgres").secret_fields, + _ => &[], + } +} + +fn map_instance_config(config: KVS, plugin_id: &str, service: &str) -> HashMap { + let secret_fields = builtin_secret_fields_for_service(plugin_id, service); + config + .0 + .into_iter() + .map(|kv| { + let should_redact = secret_fields.iter().any(|field| field.eq_ignore_ascii_case(&kv.key)); + let value = if should_redact && !kv.value.is_empty() { + REDACTED_SECRET_VALUE.to_string() + } else { + kv.value + }; + (kv.key, value) + }) + .collect() +} + +#[derive(Debug, Clone, Copy)] +struct PluginInstanceDomainContext { + domain: PluginContractDomain, + specs: &'static [AdminTargetSpec], + route_prefix: &'static str, + default_queue_dir: &'static str, +} + +fn plugin_instance_domain_context(domain: PluginContractDomain) -> PluginInstanceDomainContext { + match domain { + PluginContractDomain::Notify => PluginInstanceDomainContext { + domain, + specs: notification_target_specs(), + route_prefix: NOTIFY_ROUTE_PREFIX, + default_queue_dir: EVENT_DEFAULT_DIR, + }, + PluginContractDomain::Audit => PluginInstanceDomainContext { + domain, + specs: audit_target_specs(), + route_prefix: AUDIT_ROUTE_PREFIX, + default_queue_dir: AUDIT_DEFAULT_DIR, + }, + } +} + +fn map_instance(instance: TargetInstanceReadModel) -> PluginInstanceEntry { + let runtime_state = runtime_state_from_status_label(&instance.status); + let plugin_id = instance.plugin_id; + let service = instance.service; + let config = map_instance_config(instance.config, &plugin_id, &service); + + PluginInstanceEntry { + id: instance.canonical_id, + plugin_id, + domain: PluginContractDomain::from(instance.domain), + subsystem: instance.subsystem, + account_id: instance.account_id, + service, + status: instance.status, + source: map_instance_source(instance.source), + enabled: instance.enabled, + config, + operational_state: Some(builtin_target_plugin_operational_state(instance.enabled, runtime_state).into()), + diagnostic_codes: Vec::new(), + } +} + +fn diagnostic(code: PluginInstanceDiagnosticCode, message: impl Into) -> PluginInstanceDiagnostic { + PluginInstanceDiagnostic { + code, + message: message.into(), + } +} + +fn collect_instance_diagnostics( + instance: &TargetInstanceReadModel, + module_disabled_reason: Option, +) -> Vec { + let mut diagnostics = Vec::new(); + + if let Some(reason) = module_disabled_reason { + diagnostics.push(diagnostic(PluginInstanceDiagnosticCode::ModuleDisabled, reason)); + } + + if !instance.enabled { + diagnostics.push(diagnostic( + PluginInstanceDiagnosticCode::InstanceDisabled, + "plugin instance is disabled in its effective configuration", + )); + } + + match instance.source { + TargetEndpointSource::Env => diagnostics.push(diagnostic( + PluginInstanceDiagnosticCode::EnvironmentManaged, + "plugin instance is managed by environment variables and cannot be edited from persisted config", + )), + TargetEndpointSource::Mixed => diagnostics.push(diagnostic( + PluginInstanceDiagnosticCode::MixedSource, + "plugin instance is configured by both persisted config and environment variables", + )), + TargetEndpointSource::Config | TargetEndpointSource::Runtime => {} + } + + if instance.status.eq_ignore_ascii_case("offline") { + if instance.enabled && !instance.runtime_present { + diagnostics.push(diagnostic( + PluginInstanceDiagnosticCode::NotLoadedInRuntime, + "plugin instance is enabled in config but not currently loaded in runtime", + )); + } else if instance.runtime_present { + diagnostics.push(diagnostic( + PluginInstanceDiagnosticCode::RuntimeOffline, + "plugin instance exists in runtime but its health check is offline", + )); + } + } + + diagnostics +} + +async fn plugin_instance_detail(instance: TargetInstanceReadModel) -> PluginInstanceDetail { + let action = "reading plugin instance diagnostics"; + let context = plugin_instance_domain_context(PluginContractDomain::from(instance.domain)); + let diagnostics = collect_instance_diagnostics(&instance, plugin_instance_operation_block_reason(context, action).await); + let mut mapped = map_instance(instance); + mapped.diagnostic_codes = diagnostics.iter().map(|item| item.code.clone()).collect(); + + PluginInstanceDetail { + instance: mapped, + diagnostics, + } +} + +fn plugin_instance_list_entry(instance: TargetInstanceReadModel, module_disabled_reason: Option) -> PluginInstanceEntry { + let diagnostics = collect_instance_diagnostics(&instance, module_disabled_reason); + let mut mapped = map_instance(instance); + mapped.diagnostic_codes = diagnostics.into_iter().map(|item| item.code).collect(); + mapped +} + +fn map_instance_source(source: TargetEndpointSource) -> PluginInstanceSource { + match source { + TargetEndpointSource::Config => PluginInstanceSource::Config, + TargetEndpointSource::Env => PluginInstanceSource::Env, + TargetEndpointSource::Mixed => PluginInstanceSource::Mixed, + TargetEndpointSource::Runtime => PluginInstanceSource::Runtime, + } +} + +#[derive(Debug, Clone)] +struct ResolvedPluginInstanceTarget { + context: PluginInstanceDomainContext, + target_name: String, + target_spec: AdminTargetSpec, +} + +#[derive(Debug, serde::Deserialize)] +struct KeyValue { + key: String, + value: String, +} + +#[derive(Debug, serde::Deserialize)] +struct PluginInstanceBody { + key_values: Vec, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct PluginInstanceFilters { + domain: Option, + service: Option, + status: Option, + source: Option, + diagnostic_code: Option, + enabled: Option, + query: Option, + limit: Option, + marker: Option, +} + +fn extract_plugin_instance_filters(req: &S3Request) -> S3Result { + let mut filters = PluginInstanceFilters::default(); + + if let Some(query) = req.uri.query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + let value = value.trim(); + if value.is_empty() { + continue; + } + + match key.as_ref() { + "domain" => filters.domain = Some(parse_plugin_contract_domain(value)?), + "service" => filters.service = Some(value.to_ascii_lowercase()), + "status" => filters.status = Some(parse_instance_status(value)?), + "source" => filters.source = Some(parse_plugin_instance_source(value)?), + "diagnostic_code" => filters.diagnostic_code = Some(parse_plugin_instance_diagnostic_code(value)?), + "enabled" => filters.enabled = Some(parse_bool_filter(value)?), + "q" => filters.query = Some(value.to_ascii_lowercase()), + "limit" => filters.limit = Some(parse_limit_filter(value)?), + "marker" => filters.marker = Some(value.to_string()), + _ => {} + } + } + } + + Ok(filters) +} + +fn parse_plugin_contract_domain(value: &str) -> S3Result { + match value.to_ascii_lowercase().as_str() { + "audit" => Ok(PluginContractDomain::Audit), + "notify" => Ok(PluginContractDomain::Notify), + _ => Err(s3_error!(InvalidArgument, "invalid plugin instance domain filter: '{}'", value)), + } +} + +fn parse_instance_status(value: &str) -> S3Result { + match value.to_ascii_lowercase().as_str() { + "online" | "offline" => Ok(value.to_ascii_lowercase()), + _ => Err(s3_error!(InvalidArgument, "invalid plugin instance status filter: '{}'", value)), + } +} + +fn parse_plugin_instance_source(value: &str) -> S3Result { + match value.to_ascii_lowercase().as_str() { + "config" => Ok(PluginInstanceSource::Config), + "env" => Ok(PluginInstanceSource::Env), + "mixed" => Ok(PluginInstanceSource::Mixed), + "runtime" => Ok(PluginInstanceSource::Runtime), + _ => Err(s3_error!(InvalidArgument, "invalid plugin instance source filter: '{}'", value)), + } +} + +fn parse_plugin_instance_diagnostic_code(value: &str) -> S3Result { + match value.to_ascii_lowercase().as_str() { + "module_disabled" => Ok(PluginInstanceDiagnosticCode::ModuleDisabled), + "instance_disabled" => Ok(PluginInstanceDiagnosticCode::InstanceDisabled), + "environment_managed" => Ok(PluginInstanceDiagnosticCode::EnvironmentManaged), + "mixed_source" => Ok(PluginInstanceDiagnosticCode::MixedSource), + "not_loaded_in_runtime" => Ok(PluginInstanceDiagnosticCode::NotLoadedInRuntime), + "runtime_offline" => Ok(PluginInstanceDiagnosticCode::RuntimeOffline), + _ => Err(s3_error!(InvalidArgument, "invalid plugin instance diagnostic_code filter: '{}'", value)), + } +} + +fn parse_bool_filter(value: &str) -> S3Result { + value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "invalid plugin instance enabled filter: '{}'", value)) +} + +fn parse_limit_filter(value: &str) -> S3Result { + let limit = value + .parse::() + .map_err(|_| s3_error!(InvalidArgument, "invalid plugin instance limit filter: '{}'", value))?; + if limit == 0 { + return Err(s3_error!(InvalidArgument, "invalid plugin instance limit filter: '{}'", value)); + } + Ok(limit) +} + +fn parse_plugin_instance_id(instance_id: &str) -> S3Result<(String, PluginContractDomain, String)> { + let mut parts = instance_id.rsplitn(3, ':'); + let Some(target_name) = parts.next() else { + return Err(s3_error!(InvalidArgument, "invalid plugin instance id: '{}'", instance_id)); + }; + let Some(domain) = parts.next() else { + return Err(s3_error!(InvalidArgument, "invalid plugin instance id: '{}'", instance_id)); + }; + let Some(plugin_id) = parts.next() else { + return Err(s3_error!(InvalidArgument, "invalid plugin instance id: '{}'", instance_id)); + }; + + if target_name.is_empty() || plugin_id.is_empty() { + return Err(s3_error!(InvalidArgument, "invalid plugin instance id: '{}'", instance_id)); + } + + Ok(( + plugin_id.to_string(), + parse_plugin_contract_domain(domain)?, + target_name.to_ascii_lowercase(), + )) +} + +fn resolve_plugin_instance_target(instance_id: &str) -> S3Result { + let (plugin_id, domain, target_name) = parse_plugin_instance_id(instance_id)?; + let context = plugin_instance_domain_context(domain); + + let target_spec = context + .specs + .iter() + .find(|spec| rustfs_targets::builtin_target_marketplace_manifest(spec.service).plugin_id == plugin_id) + .cloned() + .ok_or_else(|| s3_error!(InvalidArgument, "unsupported plugin instance id: '{}'", instance_id))?; + + Ok(ResolvedPluginInstanceTarget { + context, + target_name, + target_spec, + }) +} + +fn filter_plugin_instances(mut instances: Vec, filters: &PluginInstanceFilters) -> Vec { + instances.retain(|instance| plugin_instance_matches_filters(instance, filters)); + instances +} + +fn paginate_plugin_instances( + instances: Vec, + filters: &PluginInstanceFilters, +) -> S3Result<(Vec, bool, Option)> { + let start_index = if let Some(marker) = filters.marker.as_deref() { + instances + .iter() + .position(|instance| instance.id == marker) + .map(|index| index + 1) + .ok_or_else(|| s3_error!(InvalidArgument, "invalid plugin instance marker: '{}'", marker))? + } else { + 0 + }; + + if start_index >= instances.len() { + return Ok((Vec::new(), false, None)); + } + + let remaining = &instances[start_index..]; + let limit = filters.limit.unwrap_or(remaining.len()); + let page_len = remaining.len().min(limit); + let page = remaining[..page_len].to_vec(); + let truncated = start_index + page_len < instances.len(); + let next_marker = truncated.then(|| page.last().expect("paginated page should not be empty").id.clone()); + + Ok((page, truncated, next_marker)) +} + +fn collect_diagnostic_counts(instances: &[PluginInstanceEntry]) -> Vec { + let mut counts = BTreeMap::::new(); + for instance in instances { + for code in &instance.diagnostic_codes { + *counts.entry(code.clone()).or_default() += 1; + } + } + + counts + .into_iter() + .map(|(code, count)| PluginInstanceDiagnosticCount { code, count }) + .collect() +} + +fn plugin_instance_matches_filters(instance: &PluginInstanceEntry, filters: &PluginInstanceFilters) -> bool { + if let Some(domain) = filters.domain + && instance.domain != domain + { + return false; + } + + if let Some(service) = filters.service.as_deref() + && !instance.service.eq_ignore_ascii_case(service) + { + return false; + } + + if let Some(status) = filters.status.as_deref() + && !instance.status.eq_ignore_ascii_case(status) + { + return false; + } + + if let Some(source) = filters.source + && instance.source != source + { + return false; + } + + if let Some(diagnostic_code) = &filters.diagnostic_code + && !instance.diagnostic_codes.contains(diagnostic_code) + { + return false; + } + + if let Some(enabled) = filters.enabled + && instance.enabled != enabled + { + return false; + } + + if let Some(query) = filters.query.as_deref() + && !plugin_instance_matches_query(instance, query) + { + return false; + } + + true +} + +fn plugin_instance_matches_query(instance: &PluginInstanceEntry, query: &str) -> bool { + let query = query.to_ascii_lowercase(); + [ + instance.id.as_str(), + instance.plugin_id.as_str(), + instance.subsystem.as_str(), + instance.account_id.as_str(), + instance.service.as_str(), + ] + .into_iter() + .any(|field| field.to_ascii_lowercase().contains(&query)) +} + +async fn authorize_plugin_instance_request(req: &S3Request) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::GetBucketTargetAction)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await +} + +async fn authorize_plugin_instance_write_request(req: &S3Request) -> S3Result<()> { + let Some(input_cred) = &req.credentials else { + return Err(s3_error!(InvalidRequest, "authentication required")); + }; + + let (cred, owner) = + check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; + + validate_admin_request( + &req.headers, + &cred, + owner, + false, + vec![Action::AdminAction(AdminAction::SetBucketTargetAction)], + req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), + ) + .await +} + +fn plugin_instance_mutation_block_reason( + context: PluginInstanceDomainContext, + config: &Config, + target_type: &str, + target_name: &str, + target_label: &str, +) -> Option { + shared_target_mutation_block_reason(context.specs, context.route_prefix, config, target_type, target_name, target_label) +} + +async fn plugin_instance_operation_block_reason(context: PluginInstanceDomainContext, action: &str) -> Option { + if let Err(err) = refresh_persisted_module_switches_from_store().await { + warn!( + error = %err, + "failed to reload persisted module switches before checking plugin instance operation gating" + ); + } + + match context.domain { + PluginContractDomain::Notify => { + refresh_notify_module_enabled(); + target_module_disabled_reason("notify", rustfs_config::ENV_NOTIFY_ENABLE, is_notify_module_enabled(), action) + } + PluginContractDomain::Audit => { + refresh_audit_module_enabled(); + target_module_disabled_reason("audit", rustfs_config::ENV_AUDIT_ENABLE, is_audit_module_enabled(), action) + } + } +} + +async fn plugin_instance_runtime_statuses(context: PluginInstanceDomainContext) -> S3Result> { + match context.domain { + PluginContractDomain::Notify => { + let (ns, _) = load_notification_config_snapshot().await?; + Ok(collect_runtime_statuses(ns.get_target_values().await).await) + } + PluginContractDomain::Audit => { + let mut runtime_statuses = HashMap::new(); + if let Some(system) = audit_system() { + runtime_statuses = collect_runtime_statuses(system.get_target_values().await).await; + } + Ok(runtime_statuses) + } + } +} + +async fn plugin_instance_config_snapshot(context: PluginInstanceDomainContext) -> S3Result { + match context.domain { + PluginContractDomain::Notify => load_notification_config_snapshot().await.map(|(_, config)| config), + PluginContractDomain::Audit => load_server_config_from_store().await, + } +} + +async fn collect_domain_instances(context: PluginInstanceDomainContext) -> S3Result> { + let runtime_statuses = plugin_instance_runtime_statuses(context).await?; + let config = plugin_instance_config_snapshot(context).await?; + let module_disabled_reason = plugin_instance_operation_block_reason(context, "listing plugin instances").await; + let mut entries = Vec::new(); + for instance in collect_target_instances(context.specs, context.route_prefix, &config, runtime_statuses) { + entries.push(plugin_instance_list_entry(instance, module_disabled_reason.clone())); + } + Ok(entries) +} + +async fn collect_all_instances() -> S3Result> { + let (mut notify_instances, audit_instances) = tokio::try_join!( + collect_domain_instances(plugin_instance_domain_context(PluginContractDomain::Notify)), + collect_domain_instances(plugin_instance_domain_context(PluginContractDomain::Audit)) + )?; + notify_instances.extend(audit_instances); + notify_instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); + Ok(notify_instances) +} + +async fn find_plugin_instance(instance_id: &str) -> S3Result> { + let context = plugin_instance_domain_context(parse_plugin_instance_id(instance_id)?.1); + let runtime_statuses = plugin_instance_runtime_statuses(context).await?; + let config = plugin_instance_config_snapshot(context).await?; + Ok(find_target_instance( + context.specs, + context.route_prefix, + &config, + runtime_statuses, + instance_id, + )) +} + +async fn set_plugin_instance_config( + context: PluginInstanceDomainContext, + resolved: &ResolvedPluginInstanceTarget, + kvs: KVS, +) -> S3Result<()> { + match context.domain { + PluginContractDomain::Notify => { + set_notification_target_config(resolved.target_spec.subsystem, &resolved.target_name, kvs).await + } + PluginContractDomain::Audit => { + set_audit_target_config(audit_target_specs(), resolved.target_spec.subsystem, &resolved.target_name, kvs).await + } + } +} + +async fn remove_plugin_instance_config( + context: PluginInstanceDomainContext, + resolved: &ResolvedPluginInstanceTarget, +) -> S3Result<()> { + match context.domain { + PluginContractDomain::Notify => { + remove_notification_target_config(resolved.target_spec.subsystem, &resolved.target_name).await + } + PluginContractDomain::Audit => { + remove_audit_target_config(audit_target_specs(), resolved.target_spec.subsystem, &resolved.target_name).await + } + } +} + +pub struct ListPluginInstancesHandler {} + +#[async_trait::async_trait] +impl Operation for ListPluginInstancesHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_plugin_instance_request(&req).await?; + let filters = extract_plugin_instance_filters(&req)?; + let instances = filter_plugin_instances(collect_all_instances().await?, &filters); + let diagnostic_counts = collect_diagnostic_counts(&instances); + let (instances, truncated, next_marker) = paginate_plugin_instances(instances, &filters)?; + let data = serde_json::to_vec(&PluginInstancesResponse { + instances, + diagnostic_counts, + truncated, + next_marker, + }) + .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + } +} + +pub struct GetPluginInstanceHandler {} + +#[async_trait::async_trait] +impl Operation for GetPluginInstanceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authorize_plugin_instance_request(&req).await?; + let instance_id = params + .get("id") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'id'"))?; + + let instance = find_plugin_instance(instance_id) + .await? + .ok_or_else(|| s3_error!(NoSuchKey, "plugin instance not found"))?; + + let data = serde_json::to_vec(&plugin_instance_detail(instance).await) + .map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?; + Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id"))) + } +} + +pub struct PutPluginInstanceHandler {} + +#[async_trait::async_trait] +impl Operation for PutPluginInstanceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authorize_plugin_instance_write_request(&req).await?; + let instance_id = params + .get("id") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'id'"))?; + let resolved = resolve_plugin_instance_target(instance_id)?; + let context = resolved.context; + + if let Some(reason) = plugin_instance_operation_block_reason(context, "managing plugin instances from the console").await + { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + let mut input = req.input; + let body_bytes = input + .store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE) + .await + .map_err(|_| s3_error!(InvalidRequest, "failed to read request body"))?; + let body: PluginInstanceBody = serde_json::from_slice(&body_bytes) + .map_err(|e| s3_error!(InvalidArgument, "invalid json body for plugin instance config: {}", e))?; + + match context.domain { + PluginContractDomain::Notify => { + let (_ns, config_snapshot) = load_notification_config_snapshot().await?; + if let Some(reason) = plugin_instance_mutation_block_reason( + context, + &config_snapshot, + resolved.target_spec.subsystem, + &resolved.target_name, + "plugin instance", + ) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + let kvs = build_enabled_target_kvs( + context.specs, + body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())), + resolved.target_spec.subsystem, + context.default_queue_dir, + "plugin instance", + ) + .await?; + + set_plugin_instance_config(context, &resolved, kvs).await?; + } + PluginContractDomain::Audit => { + let config_snapshot = load_server_config_from_store().await?; + if let Some(reason) = plugin_instance_mutation_block_reason( + context, + &config_snapshot, + resolved.target_spec.subsystem, + &resolved.target_name, + "plugin instance", + ) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + let kvs = build_enabled_target_kvs( + context.specs, + body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())), + resolved.target_spec.subsystem, + context.default_queue_dir, + "plugin instance", + ) + .await?; + + set_plugin_instance_config(context, &resolved, kvs).await?; + } + } + + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + } +} + +pub struct DeletePluginInstanceHandler {} + +#[async_trait::async_trait] +impl Operation for DeletePluginInstanceHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authorize_plugin_instance_write_request(&req).await?; + let instance_id = params + .get("id") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'id'"))?; + let resolved = resolve_plugin_instance_target(instance_id)?; + let context = resolved.context; + + if let Some(reason) = plugin_instance_operation_block_reason(context, "managing plugin instances from the console").await + { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + match context.domain { + PluginContractDomain::Notify => { + let (_ns, config_snapshot) = load_notification_config_snapshot().await?; + if let Some(reason) = plugin_instance_mutation_block_reason( + context, + &config_snapshot, + resolved.target_spec.subsystem, + &resolved.target_name, + "plugin instance", + ) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + remove_plugin_instance_config(context, &resolved).await?; + } + PluginContractDomain::Audit => { + let config_snapshot = load_server_config_from_store().await?; + if let Some(reason) = plugin_instance_mutation_block_reason( + context, + &config_snapshot, + resolved.target_spec.subsystem, + &resolved.target_name, + "plugin instance", + ) { + return Err(s3_error!(InvalidRequest, "{reason}")); + } + + remove_plugin_instance_config(context, &resolved).await?; + } + } + + Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id"))) + } +} + +#[cfg(test)] +mod tests { + use super::{ + PluginContractDomain, PluginInstanceFilters, collect_diagnostic_counts, collect_instance_diagnostics, + extract_plugin_instance_filters, filter_plugin_instances, map_instance, paginate_plugin_instances, parse_bool_filter, + parse_instance_status, parse_limit_filter, parse_plugin_contract_domain, parse_plugin_instance_diagnostic_code, + parse_plugin_instance_id, parse_plugin_instance_source, resolve_plugin_instance_target, + }; + use crate::admin::handlers::target_descriptor::{ + TargetEndpointSource, TargetInstanceReadModel, canonical_target_instance_id, collect_target_instances, + }; + use crate::admin::plugin_contract::{ + PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, PluginInstanceSource, + }; + use http::{Extensions, HeaderMap, Uri}; + use hyper::Method; + use rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS; + use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; + use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; + use rustfs_config::{ENABLE_KEY, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT}; + use rustfs_ecstore::config::{Config, KV, KVS}; + use rustfs_targets::TargetDomain; + use s3s::{Body, S3Request}; + use std::collections::HashMap; + + fn enabled_kvs(value: &str) -> KVS { + KVS(vec![KV { + key: ENABLE_KEY.to_string(), + value: value.to_string(), + hidden_if_empty: false, + }]) + } + + #[test] + fn plugin_instance_handlers_require_admin_authorization_contract() { + let src = include_str!("plugins_instances.rs"); + let list_block = extract_block_between_markers( + src, + "impl Operation for ListPluginInstancesHandler", + "pub struct GetPluginInstanceHandler", + ); + let detail_block = extract_block_between_markers( + src, + "impl Operation for GetPluginInstanceHandler", + "pub struct PutPluginInstanceHandler", + ); + let put_block = extract_block_between_markers( + src, + "impl Operation for PutPluginInstanceHandler", + "pub struct DeletePluginInstanceHandler", + ); + let delete_block = extract_block_between_markers(src, "impl Operation for DeletePluginInstanceHandler", "#[cfg(test)]"); + + assert!( + list_block.contains("authorize_plugin_instance_request(&req).await?;"), + "plugin instance list should require admin authorization" + ); + assert!( + detail_block.contains("authorize_plugin_instance_request(&req).await?;"), + "plugin instance detail should require admin authorization" + ); + assert!( + put_block.contains("authorize_plugin_instance_write_request(&req).await?;"), + "plugin instance writes should require SetBucketTargetAction" + ); + assert!( + delete_block.contains("authorize_plugin_instance_write_request(&req).await?;"), + "plugin instance deletion should require SetBucketTargetAction" + ); + + let read_auth_block = extract_block_between_markers( + src, + "async fn authorize_plugin_instance_request", + "async fn authorize_plugin_instance_write_request", + ); + assert!( + read_auth_block.contains("AdminAction::GetBucketTargetAction"), + "plugin instance read routes should require GetBucketTargetAction" + ); + } + + #[test] + fn configured_instance_without_runtime_appears_offline() { + let config = Config(HashMap::from([( + NOTIFY_WEBHOOK_SUB_SYS.to_string(), + HashMap::from([( + "primary".to_string(), + KVS(vec![ + KV { + key: ENABLE_KEY.to_string(), + value: "on".to_string(), + hidden_if_empty: false, + }, + KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/webhook".to_string(), + hidden_if_empty: false, + }, + ]), + )]), + )])); + + let instances = + collect_target_instances(super::notification_target_specs(), NOTIFY_ROUTE_PREFIX, &config, HashMap::new()); + let primary = instances + .into_iter() + .find(|instance| instance.account_id == "primary" && instance.service == "webhook") + .expect("configured instance should be present"); + + assert_eq!(primary.status, "offline"); + assert_eq!(primary.source, TargetEndpointSource::Config); + } + + #[test] + fn env_only_instance_appears_with_env_source() { + temp_env::with_vars( + [ + ("RUSTFS_NOTIFY_WEBHOOK_ENABLE_ENV-ONLY", Some("on")), + ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ENV-ONLY", Some("https://example.com/env")), + ], + || { + let instances = collect_target_instances( + super::notification_target_specs(), + NOTIFY_ROUTE_PREFIX, + &Config(HashMap::new()), + HashMap::new(), + ); + let env_only = instances + .into_iter() + .find(|instance| instance.account_id == "env-only") + .expect("env-only instance should be present"); + + assert_eq!(env_only.source, TargetEndpointSource::Env); + assert_eq!(env_only.status, "offline"); + }, + ); + } + + #[test] + fn runtime_only_instance_appears_with_runtime_source() { + let runtime_statuses = HashMap::from([(("runtime-only".to_string(), "webhook".to_string()), "online".to_string())]); + let instances = collect_target_instances( + super::notification_target_specs(), + NOTIFY_ROUTE_PREFIX, + &Config(HashMap::new()), + runtime_statuses, + ); + + let runtime_only = instances + .into_iter() + .find(|instance| instance.account_id == "runtime-only") + .expect("runtime-only instance should be present"); + + assert_eq!(runtime_only.source, TargetEndpointSource::Runtime); + assert_eq!(runtime_only.status, "online"); + assert_eq!(runtime_only.plugin_id, "builtin:webhook"); + assert_eq!(runtime_only.subsystem, NOTIFY_WEBHOOK_SUB_SYS); + } + + #[test] + fn detail_identity_matches_list_identity() { + let instance = TargetInstanceReadModel { + canonical_id: canonical_target_instance_id("builtin:webhook", TargetDomain::Audit, "Primary"), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Audit, + subsystem: AUDIT_WEBHOOK_SUB_SYS.to_string(), + account_id: "Primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + runtime_present: false, + source: TargetEndpointSource::Config, + enabled: true, + config: enabled_kvs("on"), + }; + + let mapped = map_instance(instance.clone()); + assert_eq!(mapped.id, instance.canonical_id); + assert_eq!(mapped.domain, PluginContractDomain::Audit); + assert!(mapped.diagnostic_codes.is_empty()); + } + + #[test] + fn map_instance_redacts_secret_config_fields() { + let instance = TargetInstanceReadModel { + canonical_id: canonical_target_instance_id("builtin:webhook", TargetDomain::Notify, "primary"), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Notify, + subsystem: NOTIFY_WEBHOOK_SUB_SYS.to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "online".to_string(), + runtime_present: true, + source: TargetEndpointSource::Config, + enabled: true, + config: KVS(vec![ + KV { + key: WEBHOOK_ENDPOINT.to_string(), + value: "https://example.com/webhook".to_string(), + hidden_if_empty: false, + }, + KV { + key: WEBHOOK_AUTH_TOKEN.to_string(), + value: "super-secret-token".to_string(), + hidden_if_empty: false, + }, + ]), + }; + + let mapped = map_instance(instance); + assert_eq!( + mapped.config.get(WEBHOOK_ENDPOINT).map(String::as_str), + Some("https://example.com/webhook") + ); + assert_eq!( + mapped.config.get(WEBHOOK_AUTH_TOKEN).map(String::as_str), + Some(super::REDACTED_SECRET_VALUE) + ); + } + + #[test] + fn canonical_id_is_stable_and_lowercases_instance_segment() { + assert_eq!( + canonical_target_instance_id("builtin:webhook", TargetDomain::Notify, "PrimaryCase"), + "builtin:webhook:notify:primarycase" + ); + } + + #[test] + fn parse_plugin_instance_id_extracts_plugin_domain_and_name() { + let (plugin_id, domain, target_name) = + parse_plugin_instance_id("builtin:webhook:notify:PrimaryCase").expect("instance id should parse"); + + assert_eq!(plugin_id, "builtin:webhook"); + assert_eq!(domain, PluginContractDomain::Notify); + assert_eq!(target_name, "primarycase"); + } + + #[test] + fn parse_plugin_instance_id_rejects_invalid_shape() { + let err = parse_plugin_instance_id("builtin:webhook").expect_err("truncated id should fail"); + assert!(err.to_string().contains("invalid plugin instance id")); + } + + #[test] + fn resolve_plugin_instance_target_uses_shared_specs() { + let resolved = + resolve_plugin_instance_target("builtin:webhook:audit:Primary").expect("builtin audit instance should resolve"); + + assert_eq!(resolved.context.domain, PluginContractDomain::Audit); + assert_eq!(resolved.target_name, "primary"); + assert_eq!(resolved.target_spec.service, "webhook"); + assert_eq!(resolved.target_spec.subsystem, AUDIT_WEBHOOK_SUB_SYS); + } + + #[test] + fn extract_plugin_instance_filters_parses_supported_query_fields() { + let req = build_plugin_instances_request( + "/rustfs/admin/v4/plugins/instances?domain=notify&service=webhook&status=offline&source=env&diagnostic_code=not_loaded_in_runtime&enabled=true&q=Primary&limit=25&marker=builtin:webhook:notify:seed", + ); + + let filters = extract_plugin_instance_filters(&req).expect("query should parse"); + assert_eq!( + filters, + PluginInstanceFilters { + domain: Some(PluginContractDomain::Notify), + service: Some("webhook".to_string()), + status: Some("offline".to_string()), + source: Some(PluginInstanceSource::Env), + diagnostic_code: Some(PluginInstanceDiagnosticCode::NotLoadedInRuntime), + enabled: Some(true), + query: Some("primary".to_string()), + limit: Some(25), + marker: Some("builtin:webhook:notify:seed".to_string()), + } + ); + } + + #[test] + fn extract_plugin_instance_filters_rejects_invalid_enum_values() { + let err = parse_plugin_contract_domain("invalid").expect_err("invalid domain should fail"); + assert!(err.to_string().contains("invalid plugin instance domain filter")); + + let err = parse_plugin_instance_source("weird").expect_err("invalid source should fail"); + assert!(err.to_string().contains("invalid plugin instance source filter")); + + let err = parse_plugin_instance_diagnostic_code("mystery").expect_err("invalid diagnostic code should fail"); + assert!(err.to_string().contains("invalid plugin instance diagnostic_code filter")); + + let err = parse_instance_status("unknown").expect_err("invalid status should fail"); + assert!(err.to_string().contains("invalid plugin instance status filter")); + + let err = parse_bool_filter("maybe").expect_err("invalid bool should fail"); + assert!(err.to_string().contains("invalid plugin instance enabled filter")); + + let err = parse_limit_filter("0").expect_err("zero limit should fail"); + assert!(err.to_string().contains("invalid plugin instance limit filter")); + } + + #[test] + fn filter_plugin_instances_applies_all_supported_filters() { + let matched = sample_instance(SampleInstance { + id: "builtin:webhook:notify:primary", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "primary", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Env, + enabled: true, + }); + let filtered = filter_plugin_instances( + vec![ + matched.clone(), + sample_instance(SampleInstance { + id: "builtin:webhook:audit:primary", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Audit, + subsystem: "audit_webhook", + account_id: "primary", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Env, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:kafka:notify:secondary", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "secondary", + service: "kafka", + status: "online", + source: PluginInstanceSource::Config, + enabled: false, + }), + ], + &PluginInstanceFilters { + domain: Some(PluginContractDomain::Notify), + service: Some("webhook".to_string()), + status: Some("offline".to_string()), + source: Some(PluginInstanceSource::Env), + diagnostic_code: None, + enabled: Some(true), + query: Some("primary".to_string()), + limit: None, + marker: None, + }, + ); + + assert_eq!(filtered, vec![matched]); + } + + #[test] + fn filter_plugin_instances_search_matches_multiple_identity_fields_case_insensitively() { + let instances = vec![ + sample_instance(SampleInstance { + id: "builtin:webhook:notify:primary", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "Primary", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Config, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:kafka:notify:secondary", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "secondary", + service: "kafka", + status: "online", + source: PluginInstanceSource::Runtime, + enabled: true, + }), + ]; + + let filtered = filter_plugin_instances( + instances, + &PluginInstanceFilters { + query: Some("NOTIFY_KAFKA".to_string().to_ascii_lowercase()), + ..PluginInstanceFilters::default() + }, + ); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].plugin_id, "builtin:kafka"); + } + + #[test] + fn filter_plugin_instances_can_match_diagnostic_code_summary() { + let mut matched = sample_instance(SampleInstance { + id: "builtin:webhook:notify:primary", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "primary", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Config, + enabled: true, + }); + matched.diagnostic_codes = vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime]; + + let mut other = sample_instance(SampleInstance { + id: "builtin:kafka:notify:secondary", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "secondary", + service: "kafka", + status: "offline", + source: PluginInstanceSource::Runtime, + enabled: true, + }); + other.diagnostic_codes = vec![PluginInstanceDiagnosticCode::RuntimeOffline]; + + let filtered = filter_plugin_instances( + vec![matched.clone(), other], + &PluginInstanceFilters { + diagnostic_code: Some(PluginInstanceDiagnosticCode::NotLoadedInRuntime), + ..PluginInstanceFilters::default() + }, + ); + + assert_eq!(filtered, vec![matched]); + } + + #[test] + fn collect_diagnostic_counts_aggregates_filtered_instance_summaries() { + let mut first = sample_instance(SampleInstance { + id: "builtin:webhook:notify:primary", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "primary", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Config, + enabled: true, + }); + first.diagnostic_codes = vec![ + PluginInstanceDiagnosticCode::ModuleDisabled, + PluginInstanceDiagnosticCode::NotLoadedInRuntime, + ]; + + let mut second = sample_instance(SampleInstance { + id: "builtin:kafka:notify:secondary", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "secondary", + service: "kafka", + status: "offline", + source: PluginInstanceSource::Runtime, + enabled: true, + }); + second.diagnostic_codes = vec![PluginInstanceDiagnosticCode::RuntimeOffline]; + + let counts = collect_diagnostic_counts(&[first, second]); + assert_eq!( + counts, + vec![ + PluginInstanceDiagnosticCount { + code: PluginInstanceDiagnosticCode::ModuleDisabled, + count: 1, + }, + PluginInstanceDiagnosticCount { + code: PluginInstanceDiagnosticCode::NotLoadedInRuntime, + count: 1, + }, + PluginInstanceDiagnosticCount { + code: PluginInstanceDiagnosticCode::RuntimeOffline, + count: 1, + }, + ] + ); + } + + #[test] + fn paginate_plugin_instances_returns_requested_page_and_next_marker() { + let instances = vec![ + sample_instance(SampleInstance { + id: "builtin:amqp:notify:a", + plugin_id: "builtin:amqp", + domain: PluginContractDomain::Notify, + subsystem: "notify_amqp", + account_id: "a", + service: "amqp", + status: "offline", + source: PluginInstanceSource::Config, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:kafka:notify:b", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "b", + service: "kafka", + status: "online", + source: PluginInstanceSource::Env, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:webhook:notify:c", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "c", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Runtime, + enabled: true, + }), + ]; + + let (page, truncated, next_marker) = paginate_plugin_instances( + instances, + &PluginInstanceFilters { + limit: Some(2), + ..PluginInstanceFilters::default() + }, + ) + .expect("pagination should succeed"); + + assert_eq!(page.len(), 2); + assert!(truncated); + assert_eq!(next_marker.as_deref(), Some("builtin:kafka:notify:b")); + } + + #[test] + fn diagnostics_include_not_loaded_in_runtime_for_enabled_config_instance() { + let instance = TargetInstanceReadModel { + canonical_id: "builtin:webhook:notify:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Notify, + subsystem: NOTIFY_WEBHOOK_SUB_SYS.to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + runtime_present: false, + source: TargetEndpointSource::Config, + enabled: true, + config: enabled_kvs("on"), + }; + + let diagnostics = collect_instance_diagnostics(&instance, None); + assert!( + diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::NotLoadedInRuntime) + ); + } + + #[test] + fn diagnostics_include_runtime_offline_when_runtime_presence_is_known() { + let instance = TargetInstanceReadModel { + canonical_id: "builtin:webhook:notify:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Notify, + subsystem: NOTIFY_WEBHOOK_SUB_SYS.to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + runtime_present: true, + source: TargetEndpointSource::Runtime, + enabled: true, + config: KVS::new(), + }; + + let diagnostics = collect_instance_diagnostics(&instance, None); + assert!( + diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::RuntimeOffline) + ); + } + + #[test] + fn diagnostics_include_source_and_module_reasons_without_guessing() { + let instance = TargetInstanceReadModel { + canonical_id: "builtin:webhook:audit:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Audit, + subsystem: AUDIT_WEBHOOK_SUB_SYS.to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + runtime_present: false, + source: TargetEndpointSource::Mixed, + enabled: false, + config: enabled_kvs("off"), + }; + + let diagnostics = + collect_instance_diagnostics(&instance, Some("audit module is disabled; enable the audit module first".to_string())); + + assert!( + diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::ModuleDisabled) + ); + assert!( + diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::InstanceDisabled) + ); + assert!( + diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::MixedSource) + ); + assert!( + !diagnostics + .iter() + .any(|item| item.code == PluginInstanceDiagnosticCode::NotLoadedInRuntime) + ); + } + + #[tokio::test] + async fn list_entry_exposes_diagnostic_code_summary() { + let instance = TargetInstanceReadModel { + canonical_id: "builtin:webhook:notify:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: TargetDomain::Notify, + subsystem: NOTIFY_WEBHOOK_SUB_SYS.to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + runtime_present: false, + source: TargetEndpointSource::Config, + enabled: true, + config: enabled_kvs("on"), + }; + + let entry = super::plugin_instance_list_entry(instance, None); + assert!( + entry + .diagnostic_codes + .contains(&PluginInstanceDiagnosticCode::NotLoadedInRuntime), + "list entry should include the offline diagnostic summary" + ); + } + + #[test] + fn paginate_plugin_instances_respects_marker_after_filtered_results() { + let instances = vec![ + sample_instance(SampleInstance { + id: "builtin:amqp:notify:a", + plugin_id: "builtin:amqp", + domain: PluginContractDomain::Notify, + subsystem: "notify_amqp", + account_id: "a", + service: "amqp", + status: "offline", + source: PluginInstanceSource::Config, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:kafka:notify:b", + plugin_id: "builtin:kafka", + domain: PluginContractDomain::Notify, + subsystem: "notify_kafka", + account_id: "b", + service: "kafka", + status: "online", + source: PluginInstanceSource::Env, + enabled: true, + }), + sample_instance(SampleInstance { + id: "builtin:webhook:notify:c", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "c", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Runtime, + enabled: true, + }), + ]; + + let (page, truncated, next_marker) = paginate_plugin_instances( + instances, + &PluginInstanceFilters { + marker: Some("builtin:amqp:notify:a".to_string()), + ..PluginInstanceFilters::default() + }, + ) + .expect("pagination should succeed"); + + assert_eq!(page.len(), 2); + assert_eq!(page[0].id, "builtin:kafka:notify:b"); + assert!(!truncated); + assert_eq!(next_marker, None); + } + + #[test] + fn paginate_plugin_instances_rejects_unknown_marker() { + let err = paginate_plugin_instances( + vec![sample_instance(SampleInstance { + id: "builtin:webhook:notify:c", + plugin_id: "builtin:webhook", + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook", + account_id: "c", + service: "webhook", + status: "offline", + source: PluginInstanceSource::Runtime, + enabled: true, + })], + &PluginInstanceFilters { + marker: Some("missing".to_string()), + ..PluginInstanceFilters::default() + }, + ) + .expect_err("unknown marker should fail"); + + assert!(err.to_string().contains("invalid plugin instance marker")); + } + + struct SampleInstance<'a> { + id: &'a str, + plugin_id: &'a str, + domain: PluginContractDomain, + subsystem: &'a str, + account_id: &'a str, + service: &'a str, + status: &'a str, + source: PluginInstanceSource, + enabled: bool, + } + + fn sample_instance(input: SampleInstance<'_>) -> PluginInstanceEntry { + PluginInstanceEntry { + id: input.id.to_string(), + plugin_id: input.plugin_id.to_string(), + domain: input.domain, + subsystem: input.subsystem.to_string(), + account_id: input.account_id.to_string(), + service: input.service.to_string(), + status: input.status.to_string(), + source: input.source, + enabled: input.enabled, + config: HashMap::new(), + operational_state: None, + diagnostic_codes: Vec::new(), + } + } + + fn build_plugin_instances_request(uri: &'static str) -> S3Request { + S3Request { + input: Body::empty(), + method: Method::GET, + uri: Uri::from_static(uri), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + } + } + + fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str { + let start = src + .find(start_marker) + .unwrap_or_else(|| panic!("Expected marker `{start_marker}` in source")); + let after_start = &src[start..]; + let end = after_start + .find(end_marker) + .unwrap_or_else(|| panic!("Expected end marker `{end_marker}` in source")); + &after_start[..end] + } +} diff --git a/rustfs/src/admin/handlers/target_descriptor.rs b/rustfs/src/admin/handlers/target_descriptor.rs index 80709d00e..b76ed80a6 100644 --- a/rustfs/src/admin/handlers/target_descriptor.rs +++ b/rustfs/src/admin/handlers/target_descriptor.rs @@ -12,22 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. +use futures::StreamExt; +use futures::future::BoxFuture; use hashbrown::HashSet as HbHashSet; use http::{HeaderMap, HeaderValue, StatusCode}; use rustfs_config::{ - AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA, - MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, - MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR, + AMQP_QUEUE_DIR, ENABLE_KEY, EnableState, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, + MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, + MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR, }; -use rustfs_ecstore::config::Config; +use rustfs_ecstore::config::{Config, KVS}; +use rustfs_targets::SharedTarget; use rustfs_targets::{ - BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available, - check_mqtt_broker_available_with_tls, check_mysql_server_available, check_nats_server_available, - check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available, + BuiltinTargetAdminDescriptor, TargetAdminMetadata, TargetDomain, TargetError, TargetRequestValidator, + check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available_with_tls, + check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available, + check_redis_server_available, config::{ - build_amqp_args, build_kafka_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args, - build_redis_args, collect_env_target_instance_ids, validate_redis_config, + TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args, + build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances, + validate_redis_config, }, + manifest::builtin_target_manifest, target::{TargetType, mqtt::MQTTTlsConfig}, }; use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; @@ -36,12 +42,14 @@ use std::collections::{HashMap, HashSet}; use std::io::{Error, ErrorKind}; use std::path::Path; use std::sync::Arc; -use tokio::time::{Duration, sleep}; +use tokio::sync::Semaphore; +use tokio::time::{Duration, sleep, timeout}; use url::Url; pub(crate) type EndpointKey = (String, String); type AdminRequestValidatorFn = - Arc, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>; + Arc Fn(&'a HashMap, &'a str) -> BoxFuture<'a, S3Result<()>> + Send + Sync>; +type DomainScopedValidatorFn = for<'a> fn(&'a HashMap, &'a str, TargetDomain) -> BoxFuture<'a, S3Result<()>>; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] @@ -59,28 +67,26 @@ pub(crate) struct MergedTargetEndpoint { pub source: TargetEndpointSource, } -#[derive(Clone, Copy, Debug)] -pub(crate) enum TargetDomain { - Notify, - Audit, +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TargetInstanceReadModel { + pub canonical_id: String, + pub plugin_id: String, + pub domain: TargetDomain, + pub subsystem: String, + pub account_id: String, + pub service: String, + pub status: String, + pub runtime_present: bool, + pub source: TargetEndpointSource, + pub enabled: bool, + pub config: KVS, } -impl TargetDomain { - pub(crate) fn runtime_target_type(self) -> TargetType { - match self { - TargetDomain::Notify => TargetType::NotifyEvent, - TargetDomain::Audit => TargetType::AuditLog, - } - } -} - -impl From for TargetDomain { - fn from(value: TargetType) -> Self { - match value { - TargetType::NotifyEvent => TargetDomain::Notify, - TargetType::AuditLog => TargetDomain::Audit, - } - } +struct TargetEndpointSnapshot { + normalized_instances: Vec, + configured_keys: Vec, + config_targets: HbHashSet, + env_targets: HbHashSet, } #[derive(Clone)] @@ -91,69 +97,55 @@ pub(crate) struct AdminTargetSpec { validator: AdminRequestValidatorFn, } -pub(crate) fn admin_target_spec_from_builtin(descriptor: &BuiltinTargetDescriptor) -> AdminTargetSpec -where - E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned, -{ +pub(crate) fn admin_target_spec_from_builtin(descriptor: &BuiltinTargetAdminDescriptor) -> AdminTargetSpec { + let admin = descriptor.admin_metadata(); AdminTargetSpec { - subsystem: descriptor.subsystem(), - service: descriptor.plugin().target_type(), - valid_keys: descriptor.plugin().valid_fields(), - validator: match descriptor.request_validator() { - TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry), - TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry), - TargetRequestValidator::Amqp(target_type) => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - Arc::new(validate_audit_amqp_request_entry) - } else { - Arc::new(validate_notify_amqp_request_entry) - } - } - TargetRequestValidator::Kafka(target_type) => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - Arc::new(validate_audit_kafka_request_entry) - } else { - Arc::new(validate_notify_kafka_request_entry) - } - } - TargetRequestValidator::MySql(target_type) => { - Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type)) - } - TargetRequestValidator::Nats(target_type) => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - Arc::new(validate_audit_nats_request_entry) - } else { - Arc::new(validate_notify_nats_request_entry) - } - } - TargetRequestValidator::Postgres(target_type) => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - Arc::new(validate_audit_postgres_request_entry) - } else { - Arc::new(validate_notify_postgres_request_entry) - } - } - TargetRequestValidator::Pulsar(target_type) => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - Arc::new(validate_audit_pulsar_request_entry) - } else { - Arc::new(validate_notify_pulsar_request_entry) - } - } - TargetRequestValidator::Redis { - default_channel, - target_type, - } => { - if matches!(TargetDomain::from(target_type), TargetDomain::Audit) { - validate_audit_redis_request_entry(default_channel) - } else { - validate_notify_redis_request_entry(default_channel) - } - } - }, + subsystem: admin.subsystem(), + service: descriptor.manifest().target_type, + valid_keys: descriptor.valid_fields(), + validator: validator_from_metadata(admin), } } +fn validator_from_metadata(metadata: TargetAdminMetadata) -> AdminRequestValidatorFn { + match metadata.request_validator() { + TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry), + TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry), + TargetRequestValidator::Amqp(target_type) => { + domain_request_validator(TargetDomain::from(target_type), validate_amqp_request) + } + TargetRequestValidator::Kafka(target_type) => { + domain_request_validator(TargetDomain::from(target_type), validate_kafka_request) + } + TargetRequestValidator::MySql(target_type) => { + Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type)) + } + TargetRequestValidator::Nats(target_type) => { + domain_request_validator(TargetDomain::from(target_type), validate_nats_request) + } + TargetRequestValidator::Postgres(target_type) => { + domain_request_validator(TargetDomain::from(target_type), validate_postgres_request) + } + TargetRequestValidator::Pulsar(target_type) => { + domain_request_validator(TargetDomain::from(target_type), validate_pulsar_request) + } + TargetRequestValidator::Redis { + default_channel, + target_type, + } => redis_request_validator(TargetDomain::from(target_type), default_channel), + } +} + +fn domain_request_validator(domain: TargetDomain, validator: DomainScopedValidatorFn) -> AdminRequestValidatorFn { + Arc::new(move |kv_map, default_queue_dir| validator(kv_map, default_queue_dir, domain)) +} + +fn redis_request_validator(domain: TargetDomain, default_channel: &'static str) -> AdminRequestValidatorFn { + Arc::new(move |kv_map, default_queue_dir| { + Box::pin(validate_redis_request(kv_map, default_queue_dir, domain, default_channel)) + }) +} + impl std::fmt::Debug for AdminTargetSpec { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AdminTargetSpec") @@ -182,54 +174,26 @@ pub(crate) fn target_service_name(specs: &[AdminTargetSpec], target_type: &str) target_spec(specs, target_type).map(|spec| spec.service) } -pub(crate) fn collect_configured_endpoint_keys(specs: &[AdminTargetSpec], config: &Config) -> Vec { - let mut endpoints = Vec::new(); - for spec in specs { - let Some(targets) = config.0.get(spec.subsystem) else { - continue; - }; - - for (target_name, kvs) in targets { - if target_name == rustfs_config::DEFAULT_DELIMITER { - continue; - } - let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false); - if enabled { - endpoints.push((target_name.clone(), spec.service.to_string())); - } - } +pub(crate) fn extract_supported_target_params<'a>( + specs: &[AdminTargetSpec], + params: &'a matchit::Params<'_, '_>, + unsupported_target_label: &str, +) -> S3Result<(&'a str, &'a str)> { + let target_type = params + .get("target_type") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?; + if target_service_name(specs, target_type).is_none() { + return Err(s3_error!( + InvalidArgument, + "unsupported {} target type: '{}'", + unsupported_target_label, + target_type + )); } - endpoints -} - -pub(crate) fn collect_config_entry_keys(specs: &[AdminTargetSpec], config: &Config) -> HbHashSet { - let mut endpoints = HbHashSet::new(); - for spec in specs { - let Some(targets) = config.0.get(spec.subsystem) else { - continue; - }; - - for target_name in targets.keys() { - if target_name == rustfs_config::DEFAULT_DELIMITER { - continue; - } - endpoints.insert(normalized_endpoint_key(target_name, spec.service)); - } - } - endpoints -} - -pub(crate) fn collect_env_endpoint_keys(specs: &[AdminTargetSpec], route_prefix: &str) -> HbHashSet { - let mut endpoints = HbHashSet::new(); - for spec in specs { - let valid_keys = spec.valid_keys.iter().map(|key| (*key).to_string()).collect::>(); - for instance_id in collect_env_target_instance_ids(route_prefix, spec.service, &valid_keys) { - if instance_id != rustfs_config::DEFAULT_DELIMITER && !instance_id.is_empty() { - endpoints.insert(normalized_endpoint_key(&instance_id, spec.service)); - } - } - } - endpoints + let target_name = params + .get("target_name") + .ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?; + Ok((target_type, target_name)) } pub(crate) fn classify_endpoint_source( @@ -237,7 +201,11 @@ pub(crate) fn classify_endpoint_source( env_targets: &HbHashSet, key: &EndpointKey, ) -> TargetEndpointSource { - match (config_targets.contains(key), env_targets.contains(key)) { + classify_endpoint_source_flags(config_targets.contains(key), env_targets.contains(key)) +} + +fn classify_endpoint_source_flags(has_config_source: bool, has_env_source: bool) -> TargetEndpointSource { + match (has_config_source, has_env_source) { (true, true) => TargetEndpointSource::Mixed, (true, false) => TargetEndpointSource::Config, (false, true) => TargetEndpointSource::Env, @@ -252,11 +220,10 @@ pub(crate) fn endpoint_source( target_type: &str, target_name: &str, ) -> TargetEndpointSource { - let config_targets = collect_config_entry_keys(specs, config); - let env_targets = collect_env_endpoint_keys(specs, route_prefix); + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); let service = target_service_name(specs, target_type).unwrap_or_default(); let key = normalized_endpoint_key(target_name, service); - classify_endpoint_source(&config_targets, &env_targets, &key) + classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key) } pub(crate) fn target_mutation_block_reason( @@ -301,6 +268,33 @@ pub(crate) fn build_json_response( S3Response::with_headers((status, body), header) } +pub(crate) async fn collect_runtime_statuses(targets: Vec>) -> HashMap +where + E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned, +{ + let semaphore = Arc::new(Semaphore::new(10)); + let mut futures = futures::stream::FuturesUnordered::new(); + + for target in targets { + let sem = Arc::clone(&semaphore); + futures.push(async move { + let _permit = sem.acquire().await; + let status = match tokio::time::timeout(Duration::from_secs(3), target.is_active()).await { + Ok(Ok(true)) => "online", + _ => "offline", + }; + ((target.id().id, target.id().name), status.to_string()) + }); + } + + let mut runtime_statuses = HashMap::new(); + while let Some((key, status)) = futures.next().await { + runtime_statuses.insert(key, status); + } + + runtime_statuses +} + pub(crate) fn merge_target_endpoints( specs: &[AdminTargetSpec], route_prefix: &str, @@ -309,9 +303,7 @@ pub(crate) fn merge_target_endpoints( ) -> Vec { let mut endpoints = Vec::new(); let mut seen = HashSet::new(); - let configured_keys = collect_configured_endpoint_keys(specs, config); - let config_targets = collect_config_entry_keys(specs, config); - let env_targets = collect_env_endpoint_keys(specs, route_prefix); + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); let mut normalized_runtime_statuses: HashMap = HashMap::new(); for ((account_id, service), status) in runtime_statuses { @@ -321,7 +313,7 @@ pub(crate) fn merge_target_endpoints( .or_insert((account_id, service, status)); } - for key in configured_keys { + for key in snapshot.configured_keys { let normalized = normalized_endpoint_key(&key.0, &key.1); if !seen.insert(normalized.clone()) { continue; @@ -336,7 +328,7 @@ pub(crate) fn merge_target_endpoints( account_id: key.0, service: key.1, status, - source: classify_endpoint_source(&config_targets, &env_targets, &normalized), + source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized), }); } @@ -346,12 +338,12 @@ pub(crate) fn merge_target_endpoints( account_id, service, status, - source: classify_endpoint_source(&config_targets, &env_targets, &normalized), + source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized), }); } } - for key in &env_targets { + for key in &snapshot.env_targets { if !seen.insert(key.clone()) { continue; } @@ -360,7 +352,7 @@ pub(crate) fn merge_target_endpoints( account_id: key.0.clone(), service: key.1.clone(), status: "offline".to_string(), - source: classify_endpoint_source(&config_targets, &env_targets, key), + source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, key), }); } @@ -368,6 +360,96 @@ pub(crate) fn merge_target_endpoints( endpoints } +pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String { + format!("{plugin_id}:{}:{}", canonical_domain_label(domain), instance_id.to_lowercase()) +} + +pub(crate) fn collect_target_instances( + specs: &[AdminTargetSpec], + route_prefix: &str, + config: &Config, + runtime_statuses: HashMap, +) -> Vec { + let mut instances = Vec::new(); + let mut seen = HashSet::new(); + let mut normalized_runtime_statuses: HashMap = HashMap::new(); + let domain = inferred_target_domain(route_prefix); + let snapshot = collect_endpoint_snapshot(specs, route_prefix, config); + + for ((account_id, service), status) in runtime_statuses { + let normalized = normalized_endpoint_key(&account_id, &service); + normalized_runtime_statuses + .entry(normalized) + .or_insert((account_id, service, status)); + } + + for instance in snapshot.normalized_instances { + let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type); + if !seen.insert(key.clone()) { + continue; + } + + let runtime_present = normalized_runtime_statuses.contains_key(&key); + let status = normalized_runtime_statuses + .remove(&key) + .map(|(_, _, status)| status) + .unwrap_or_else(|| "offline".to_string()); + let source = classify_endpoint_source_flags(instance_has_config_entry(&instance), instance_has_env_entry(&instance)); + + instances.push(TargetInstanceReadModel { + canonical_id: canonical_target_instance_id(&instance.plugin_id, domain, &instance.instance_id), + plugin_id: instance.plugin_id, + domain, + subsystem: instance.subsystem, + account_id: instance.instance_id, + service: instance.target_type, + status, + runtime_present, + source, + enabled: instance.enabled, + config: instance.effective_config, + }); + } + + for (normalized, (account_id, service, status)) in normalized_runtime_statuses { + if !seen.insert(normalized) { + continue; + } + + let (plugin_id, subsystem): (String, String) = target_spec_by_service(specs, &service) + .map(|spec| (builtin_target_manifest(spec.service).plugin_id.to_string(), spec.subsystem.to_string())) + .unwrap_or_else(|| ("custom:target".to_string(), format!("{}_{}", canonical_domain_label(domain), service))); + instances.push(TargetInstanceReadModel { + canonical_id: canonical_target_instance_id(&plugin_id, domain, &account_id), + plugin_id, + domain, + subsystem, + account_id, + service, + status, + runtime_present: true, + source: TargetEndpointSource::Runtime, + enabled: true, + config: KVS::new(), + }); + } + + instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id))); + instances +} + +pub(crate) fn find_target_instance( + specs: &[AdminTargetSpec], + route_prefix: &str, + config: &Config, + runtime_statuses: HashMap, + canonical_id: &str, +) -> Option { + collect_target_instances(specs, route_prefix, config, runtime_statuses) + .into_iter() + .find(|instance| instance.canonical_id == canonical_id) +} + pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> { target_spec(specs, target_type) .map(|spec| spec.valid_keys.iter().copied().collect()) @@ -435,8 +517,109 @@ pub(crate) async fn validate_target_request( spec.validate_request(kv_map, default_queue_dir).await } -fn config_enable_is_on(value: &str) -> bool { - matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1") +pub(crate) async fn build_enabled_target_kvs<'a, I>( + specs: &[AdminTargetSpec], + key_values: I, + target_type: &str, + default_queue_dir: &str, + target_label: &str, +) -> S3Result +where + I: IntoIterator, +{ + let allowed_keys = allowed_target_keys(specs, target_type); + let kv_map = collect_validated_key_values(key_values, &allowed_keys, target_type, target_label)?; + let spec = target_spec(specs, target_type) + .ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?; + timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir)) + .await + .map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??; + + let mut kvs = KVS::new(); + for (key, value) in kv_map { + kvs.insert(key, value); + } + kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string()); + Ok(kvs) +} + +fn instance_has_config_entry(instance: &TargetPluginInstanceRecord) -> bool { + instance.source_hints.has_file_instance +} + +fn instance_has_env_entry(instance: &TargetPluginInstanceRecord) -> bool { + instance.source_hints.has_env_instance +} + +fn normalized_target_instances( + specs: &[AdminTargetSpec], + route_prefix: &str, + config: &Config, +) -> Vec { + specs + .iter() + .flat_map(|spec| { + normalize_target_plugin_instances( + config, + &TargetPluginInstanceCompatDescriptor { + domain: inferred_target_domain(route_prefix), + plugin_id: builtin_target_manifest(spec.service).plugin_id, + target_type: spec.service, + subsystem: spec.subsystem, + route_prefix, + valid_fields: spec.valid_keys, + }, + ) + }) + .collect() +} + +fn inferred_target_domain(route_prefix: &str) -> TargetDomain { + match route_prefix { + rustfs_config::notify::NOTIFY_ROUTE_PREFIX => TargetDomain::Notify, + rustfs_config::audit::AUDIT_ROUTE_PREFIX => TargetDomain::Audit, + _ => TargetDomain::Notify, + } +} + +fn canonical_domain_label(domain: TargetDomain) -> &'static str { + match domain { + TargetDomain::Notify => "notify", + TargetDomain::Audit => "audit", + } +} + +fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Option<&'a AdminTargetSpec> { + specs.iter().find(|spec| spec.service == service) +} + +fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot { + let normalized_instances = normalized_target_instances(specs, route_prefix, config); + let mut configured_keys = Vec::new(); + let mut config_targets = HbHashSet::new(); + let mut env_targets = HbHashSet::new(); + + for instance in &normalized_instances { + let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type); + + if instance_has_config_entry(instance) { + config_targets.insert(key.clone()); + if instance.enabled { + configured_keys.push((instance.instance_id.clone(), instance.target_type.clone())); + } + } + + if instance_has_env_entry(instance) { + env_targets.insert(key); + } + } + + TargetEndpointSnapshot { + normalized_instances, + configured_keys, + config_targets, + env_targets, + } } async fn retry_with_backoff(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result @@ -489,12 +672,11 @@ async fn validate_webhook_request(kv_map: &HashMap) -> S3Result< Ok(()) } -fn validate_webhook_request_entry( - kv_map: &HashMap, - _default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - Box::pin(async move { validate_webhook_request(&kv_map).await }) +fn validate_webhook_request_entry<'a>( + kv_map: &'a HashMap, + _default_queue_dir: &'a str, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(validate_webhook_request(kv_map)) } async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> { @@ -541,131 +723,34 @@ async fn validate_mqtt_request(kv_map: &HashMap) -> S3Result<()> Ok(()) } -fn validate_mqtt_request_entry( - kv_map: &HashMap, - _default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - Box::pin(async move { validate_mqtt_request(&kv_map).await }) +fn validate_mqtt_request_entry<'a>( + kv_map: &'a HashMap, + _default_queue_dir: &'a str, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(validate_mqtt_request(kv_map)) } -fn validate_notify_nats_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) -} - -fn validate_audit_nats_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) -} - -fn validate_notify_kafka_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) -} - -fn validate_audit_kafka_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) -} - -fn validate_notify_amqp_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) -} - -fn validate_audit_amqp_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) -} - -fn validate_notify_pulsar_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) -} - -fn validate_audit_pulsar_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) -} - -fn validate_mysql_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, +fn validate_mysql_request_entry<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, target_type: TargetType, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir, target_type).await }) +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(validate_mysql_request(kv_map, default_queue_dir, target_type)) } -fn validate_notify_postgres_request_entry( +fn validate_nats_request<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, + domain: TargetDomain, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(async move { validate_nats_request_impl(kv_map, default_queue_dir, domain).await }) +} + +async fn validate_nats_request_impl( kv_map: &HashMap, default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await }) -} - -fn validate_audit_postgres_request_entry( - kv_map: &HashMap, - default_queue_dir: &str, -) -> futures::future::BoxFuture<'static, S3Result<()>> { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await }) -} - -fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn { - Arc::new(move |kv_map: &HashMap, default_queue_dir: &str| { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await }) - }) -} - -fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn { - Arc::new(move |kv_map: &HashMap, default_queue_dir: &str| { - let kv_map = kv_map.clone(); - let default_queue_dir = default_queue_dir.to_string(); - Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await }) - }) -} - -async fn validate_nats_request(kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> { + domain: TargetDomain, +) -> S3Result<()> { if let Some(queue_dir) = kv_map.get("queue_dir") { validate_queue_dir(queue_dir.as_str()).await?; } @@ -677,7 +762,19 @@ async fn validate_nats_request(kv_map: &HashMap, default_queue_d }) } -async fn validate_kafka_request(kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> { +fn validate_kafka_request<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, + domain: TargetDomain, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(async move { validate_kafka_request_impl(kv_map, default_queue_dir, domain).await }) +} + +async fn validate_kafka_request_impl( + kv_map: &HashMap, + default_queue_dir: &str, + domain: TargetDomain, +) -> S3Result<()> { if let Some(queue_dir) = kv_map.get(KAFKA_QUEUE_DIR) { validate_queue_dir(queue_dir.as_str()).await?; } @@ -697,7 +794,19 @@ async fn validate_kafka_request(kv_map: &HashMap, default_queue_ }) } -async fn validate_amqp_request(kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> { +fn validate_amqp_request<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, + domain: TargetDomain, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(async move { validate_amqp_request_impl(kv_map, default_queue_dir, domain).await }) +} + +async fn validate_amqp_request_impl( + kv_map: &HashMap, + default_queue_dir: &str, + domain: TargetDomain, +) -> S3Result<()> { if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) { validate_queue_dir(queue_dir.as_str()).await?; } @@ -709,7 +818,15 @@ async fn validate_amqp_request(kv_map: &HashMap, default_queue_d }) } -async fn validate_pulsar_request( +fn validate_pulsar_request<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, + domain: TargetDomain, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(async move { validate_pulsar_request_impl(kv_map, default_queue_dir, domain).await }) +} + +async fn validate_pulsar_request_impl( kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain, @@ -742,7 +859,15 @@ async fn validate_mysql_request( }) } -async fn validate_postgres_request( +fn validate_postgres_request<'a>( + kv_map: &'a HashMap, + default_queue_dir: &'a str, + domain: TargetDomain, +) -> BoxFuture<'a, S3Result<()>> { + Box::pin(async move { validate_postgres_request_impl(kv_map, default_queue_dir, domain).await }) +} + +async fn validate_postgres_request_impl( kv_map: &HashMap, default_queue_dir: &str, domain: TargetDomain, diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index 62efb8c8a..4b59a54ba 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -15,6 +15,7 @@ mod auth; pub mod console; pub mod handlers; +mod plugin_contract; pub mod router; pub mod service; pub mod site_replication_identity; @@ -26,8 +27,8 @@ mod console_test; mod route_registration_test; use handlers::{ - audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication, - site_replication, sts, system, tier, user, + audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota, + rebalance, replication, site_replication, sts, system, tier, user, }; use router::{AdminOperation, S3Router}; use s3s::route::S3Route; @@ -58,6 +59,8 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result bucket_meta::register_bucket_meta_route(&mut r)?; audit::register_audit_target_route(&mut r)?; module_switch::register_module_switch_route(&mut r)?; + plugins_catalog::register_plugin_catalog_route(&mut r)?; + plugins_instances::register_plugin_instance_route(&mut r)?; replication::register_replication_route(&mut r)?; site_replication::register_site_replication_route(&mut r)?; diff --git a/rustfs/src/admin/plugin_contract.rs b/rustfs/src/admin/plugin_contract.rs new file mode 100644 index 000000000..7c41fee8a --- /dev/null +++ b/rustfs/src/admin/plugin_contract.rs @@ -0,0 +1,577 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_targets::{ + TargetDomain, TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEnableState, + TargetPluginEntrypointKind, TargetPluginExternalRuntimeContract, TargetPluginInstallState, TargetPluginInstallation, + TargetPluginOperationalState, TargetPluginPackaging, TargetPluginRuntimeState, TargetPluginRuntimeTransport, +}; +use serde::Serialize; +use std::collections::HashMap; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginContractDomain { + Audit, + Notify, +} + +impl From for PluginContractDomain { + fn from(value: TargetDomain) -> Self { + match value { + TargetDomain::Audit => Self::Audit, + TargetDomain::Notify => Self::Notify, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginContractPackaging { + Builtin, + External, +} + +impl From for PluginContractPackaging { + fn from(value: TargetPluginPackaging) -> Self { + match value { + TargetPluginPackaging::Builtin => Self::Builtin, + TargetPluginPackaging::External => Self::External, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginContractEntrypointKind { + Builtin, + Sidecar, + Wasm, +} + +impl From for PluginContractEntrypointKind { + fn from(value: TargetPluginEntrypointKind) -> Self { + match value { + TargetPluginEntrypointKind::Builtin => Self::Builtin, + TargetPluginEntrypointKind::Sidecar => Self::Sidecar, + TargetPluginEntrypointKind::Wasm => Self::Wasm, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginRuntimeTransport { + InProcess, + Grpc, + WasmHost, +} + +impl From for PluginRuntimeTransport { + fn from(value: TargetPluginRuntimeTransport) -> Self { + match value { + TargetPluginRuntimeTransport::InProcess => Self::InProcess, + TargetPluginRuntimeTransport::Grpc => Self::Grpc, + TargetPluginRuntimeTransport::WasmHost => Self::WasmHost, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginInstallState { + NotInstalled, + Installed, + InstallFailed, +} + +impl From for PluginInstallState { + fn from(value: TargetPluginInstallState) -> Self { + match value { + TargetPluginInstallState::NotInstalled => Self::NotInstalled, + TargetPluginInstallState::Installed => Self::Installed, + TargetPluginInstallState::InstallFailed => Self::InstallFailed, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginEnableState { + Enabled, + Disabled, +} + +impl From for PluginEnableState { + fn from(value: TargetPluginEnableState) -> Self { + match value { + TargetPluginEnableState::Enabled => Self::Enabled, + TargetPluginEnableState::Disabled => Self::Disabled, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginOperationalRuntimeState { + Running, + Offline, + Error, + Unknown, +} + +impl From for PluginOperationalRuntimeState { + fn from(value: TargetPluginRuntimeState) -> Self { + match value { + TargetPluginRuntimeState::Running => Self::Running, + TargetPluginRuntimeState::Offline => Self::Offline, + TargetPluginRuntimeState::Error => Self::Error, + TargetPluginRuntimeState::Unknown => Self::Unknown, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginRevisionContract { + pub version: String, + pub digest_sha256: Option, + pub source: String, + pub installed_at: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub artifact_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginInstallationContract { + pub install_state: PluginInstallState, + pub current_revision: Option, + pub previous_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub validation_error: Option, +} + +impl From for PluginInstallationContract { + fn from(value: TargetPluginInstallation) -> Self { + Self { + install_state: PluginInstallState::from(value.install_state), + current_revision: value.current_revision.map(|revision| PluginRevisionContract { + version: revision.version, + digest_sha256: revision.digest_sha256, + source: revision.source, + installed_at: revision.installed_at, + artifact_id: revision.artifact_id, + }), + previous_revision: value.previous_revision.map(|revision| PluginRevisionContract { + version: revision.version, + digest_sha256: revision.digest_sha256, + source: revision.source, + installed_at: revision.installed_at, + artifact_id: revision.artifact_id, + }), + validation_error: value.validation_error, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginOperationalStateContract { + pub install_state: PluginInstallState, + pub enable_state: PluginEnableState, + pub runtime_state: PluginOperationalRuntimeState, +} + +impl From for PluginOperationalStateContract { + fn from(value: TargetPluginOperationalState) -> Self { + Self { + install_state: PluginInstallState::from(value.install_state), + enable_state: PluginEnableState::from(value.enable_state), + runtime_state: PluginOperationalRuntimeState::from(value.runtime_state), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginRuntimeContract { + pub protocol_version: String, + pub transport: PluginRuntimeTransport, +} + +impl From for PluginRuntimeContract { + fn from(value: TargetPluginExternalRuntimeContract) -> Self { + Self { + protocol_version: value.protocol_version.to_string(), + transport: PluginRuntimeTransport::from(value.transport), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginArtifactContract { + pub artifact_id: String, + pub target_triple: String, + pub download_uri: String, + pub digest_sha256: String, + pub size_bytes: u64, +} + +impl From for PluginArtifactContract { + fn from(value: TargetPluginArtifactManifest) -> Self { + Self { + artifact_id: value.artifact_id.to_string(), + target_triple: value.target_triple.to_string(), + download_uri: value.download_uri.to_string(), + digest_sha256: value.digest_sha256.to_string(), + size_bytes: value.size_bytes, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginDistributionContract { + pub artifacts: Vec, +} + +impl From for PluginDistributionContract { + fn from(value: TargetPluginDistributionManifest) -> Self { + Self { + artifacts: value.artifacts.iter().copied().map(PluginArtifactContract::from).collect(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginInstanceSource { + Config, + Env, + Mixed, + Runtime, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginCatalogDomainEntry { + pub domain: PluginContractDomain, + pub subsystem: String, + pub valid_fields: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginCatalogEntry { + pub plugin_id: String, + pub target_type: String, + pub display_name: String, + pub provider: String, + pub version: String, + pub packaging: PluginContractPackaging, + pub entrypoint_kind: PluginContractEntrypointKind, + pub api_compatibility_version: String, + pub runtime_contract: PluginRuntimeContract, + pub distribution: Option, + pub supported_domains: Vec, + pub secret_fields: Vec, + pub domain_configs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub installation: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct PluginCatalogResponse { + pub plugins: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginInstanceEntry { + pub id: String, + pub plugin_id: String, + pub domain: PluginContractDomain, + pub subsystem: String, + pub account_id: String, + pub service: String, + pub status: String, + pub source: PluginInstanceSource, + pub enabled: bool, + pub config: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub operational_state: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub diagnostic_codes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum PluginInstanceDiagnosticCode { + ModuleDisabled, + InstanceDisabled, + EnvironmentManaged, + MixedSource, + NotLoadedInRuntime, + RuntimeOffline, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginInstanceDiagnostic { + pub code: PluginInstanceDiagnosticCode, + pub message: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) struct PluginInstanceDiagnosticCount { + pub code: PluginInstanceDiagnosticCode, + pub count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct PluginInstanceDetail { + #[serde(flatten)] + pub instance: PluginInstanceEntry, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub diagnostics: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(crate) struct PluginInstancesResponse { + pub instances: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub diagnostic_counts: Vec, + pub truncated: bool, + pub next_marker: Option, +} + +#[cfg(test)] +mod tests { + use super::{ + PluginArtifactContract, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, + PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginInstanceDetail, + PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, + PluginInstanceSource, PluginInstancesResponse, PluginRuntimeContract, PluginRuntimeTransport, + }; + use serde_json::json; + use std::collections::HashMap; + + #[test] + fn plugin_catalog_contract_serializes_stable_json_shape() { + let response = PluginCatalogResponse { + plugins: vec![PluginCatalogEntry { + plugin_id: "builtin:webhook".to_string(), + target_type: "webhook".to_string(), + display_name: "Webhook".to_string(), + provider: "rustfs".to_string(), + version: "1.0.0".to_string(), + packaging: PluginContractPackaging::Builtin, + entrypoint_kind: PluginContractEntrypointKind::Builtin, + api_compatibility_version: "rustfs.target-plugin.v1".to_string(), + runtime_contract: PluginRuntimeContract { + protocol_version: "rustfs.target-runtime.v1".to_string(), + transport: PluginRuntimeTransport::InProcess, + }, + distribution: None, + supported_domains: vec![PluginContractDomain::Audit, PluginContractDomain::Notify], + secret_fields: vec!["auth_token".to_string()], + domain_configs: vec![PluginCatalogDomainEntry { + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook".to_string(), + valid_fields: vec!["endpoint".to_string(), "auth_token".to_string()], + }], + installation: None, + }], + }; + + let value = serde_json::to_value(response).expect("catalog response should serialize"); + assert_eq!( + value, + json!({ + "plugins": [{ + "plugin_id": "builtin:webhook", + "target_type": "webhook", + "display_name": "Webhook", + "provider": "rustfs", + "version": "1.0.0", + "packaging": "builtin", + "entrypoint_kind": "builtin", + "api_compatibility_version": "rustfs.target-plugin.v1", + "runtime_contract": { + "protocol_version": "rustfs.target-runtime.v1", + "transport": "in_process" + }, + "distribution": null, + "supported_domains": ["audit", "notify"], + "secret_fields": ["auth_token"], + "domain_configs": [{ + "domain": "notify", + "subsystem": "notify_webhook", + "valid_fields": ["endpoint", "auth_token"] + }] + }] + }) + ); + } + + #[test] + fn plugin_instance_contract_serializes_stable_json_shape() { + let response = PluginInstancesResponse { + instances: vec![PluginInstanceEntry { + id: "builtin:webhook:notify:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook".to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + source: PluginInstanceSource::Config, + enabled: true, + config: HashMap::from([ + ("enable".to_string(), "on".to_string()), + ("endpoint".to_string(), "https://example.com/hook".to_string()), + ]), + operational_state: None, + diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime], + }], + diagnostic_counts: vec![PluginInstanceDiagnosticCount { + code: PluginInstanceDiagnosticCode::NotLoadedInRuntime, + count: 1, + }], + truncated: false, + next_marker: None, + }; + + let value = serde_json::to_value(response).expect("instance response should serialize"); + assert_eq!( + value, + json!({ + "instances": [{ + "id": "builtin:webhook:notify:primary", + "plugin_id": "builtin:webhook", + "domain": "notify", + "subsystem": "notify_webhook", + "account_id": "primary", + "service": "webhook", + "status": "offline", + "source": "config", + "enabled": true, + "config": { + "enable": "on", + "endpoint": "https://example.com/hook" + }, + "diagnostic_codes": ["not_loaded_in_runtime"] + }], + "diagnostic_counts": [{ + "code": "not_loaded_in_runtime", + "count": 1 + }], + "truncated": false, + "next_marker": null + }) + ); + } + + #[test] + fn plugin_instance_detail_contract_serializes_diagnostics_when_present() { + let detail = PluginInstanceDetail { + instance: PluginInstanceEntry { + id: "builtin:webhook:notify:primary".to_string(), + plugin_id: "builtin:webhook".to_string(), + domain: PluginContractDomain::Notify, + subsystem: "notify_webhook".to_string(), + account_id: "primary".to_string(), + service: "webhook".to_string(), + status: "offline".to_string(), + source: PluginInstanceSource::Config, + enabled: true, + config: HashMap::from([("endpoint".to_string(), "https://example.com/hook".to_string())]), + operational_state: None, + diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime], + }, + diagnostics: vec![PluginInstanceDiagnostic { + code: PluginInstanceDiagnosticCode::NotLoadedInRuntime, + message: "plugin instance is enabled in config but not currently loaded in runtime".to_string(), + }], + }; + + let value = serde_json::to_value(detail).expect("instance detail should serialize"); + assert_eq!( + value, + json!({ + "id": "builtin:webhook:notify:primary", + "plugin_id": "builtin:webhook", + "domain": "notify", + "subsystem": "notify_webhook", + "account_id": "primary", + "service": "webhook", + "status": "offline", + "source": "config", + "enabled": true, + "config": { + "endpoint": "https://example.com/hook" + }, + "diagnostic_codes": ["not_loaded_in_runtime"], + "diagnostics": [{ + "code": "not_loaded_in_runtime", + "message": "plugin instance is enabled in config but not currently loaded in runtime" + }] + }) + ); + } + + #[test] + fn plugin_catalog_distribution_contract_serializes_when_present() { + let entry = PluginCatalogEntry { + plugin_id: "external:webhook".to_string(), + target_type: "webhook".to_string(), + display_name: "Webhook+".to_string(), + provider: "example".to_string(), + version: "1.2.3".to_string(), + packaging: PluginContractPackaging::Builtin, + entrypoint_kind: PluginContractEntrypointKind::Sidecar, + api_compatibility_version: "rustfs.target-plugin.v1".to_string(), + runtime_contract: PluginRuntimeContract { + protocol_version: "rustfs.target-runtime.v1".to_string(), + transport: PluginRuntimeTransport::Grpc, + }, + distribution: Some(PluginDistributionContract { + artifacts: vec![PluginArtifactContract { + artifact_id: "sidecar-linux-amd64".to_string(), + target_triple: "x86_64-unknown-linux-gnu".to_string(), + download_uri: "https://plugins.example.test/webhook.tar.zst".to_string(), + digest_sha256: "0123456789abcdef".to_string(), + size_bytes: 4096, + }], + }), + supported_domains: vec![PluginContractDomain::Notify], + secret_fields: Vec::new(), + domain_configs: Vec::new(), + installation: None, + }; + + let value = serde_json::to_value(entry).expect("catalog entry should serialize"); + assert_eq!(value["distribution"]["artifacts"][0]["artifact_id"], "sidecar-linux-amd64"); + assert_eq!(value["distribution"]["artifacts"][0]["target_triple"], "x86_64-unknown-linux-gnu"); + assert_eq!( + value["distribution"]["artifacts"][0]["download_uri"], + "https://plugins.example.test/webhook.tar.zst" + ); + assert_eq!(value["distribution"]["artifacts"][0]["digest_sha256"], "0123456789abcdef"); + assert_eq!(value["distribution"]["artifacts"][0]["size_bytes"], 4096); + } +} diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 3374d7894..8d444b377 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -14,8 +14,8 @@ use crate::admin::{ handlers::{ - audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication, - site_replication, sts, system, tier, user, + audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, + quota, rebalance, replication, site_replication, sts, system, tier, user, }, router::{AdminOperation, S3Router}, }; @@ -54,6 +54,8 @@ fn register_admin_routes(router: &mut S3Router) { bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route"); audit::register_audit_target_route(router).expect("register audit target route"); module_switch::register_module_switch_route(router).expect("register module switch route"); + plugins_catalog::register_plugin_catalog_route(router).expect("register plugin catalog route"); + plugins_instances::register_plugin_instance_route(router).expect("register plugin instances route"); replication::register_replication_route(router).expect("register replication route"); site_replication::register_site_replication_route(router).expect("register site replication route"); profile_admin::register_profiling_route(router).expect("register profile route"); @@ -101,6 +103,11 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list")); assert_route(&router, Method::GET, &admin_path("/v3/module-switches")); assert_route(&router, Method::PUT, &admin_path("/v3/module-switches")); + assert_route(&router, Method::GET, &admin_path("/v4/plugins/catalog")); + assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances")); + assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances/example-id")); + assert_route(&router, Method::PUT, &admin_path("/v4/plugins/instances/example-id")); + assert_route(&router, Method::DELETE, &admin_path("/v4/plugins/instances/example-id")); assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit")); assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset")); assert_route(&router, Method::GET, &admin_path("/v3/accountinfo")); diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 6130147a3..d3e6a7201 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -510,7 +510,7 @@ fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: }) .transpose()?; let version_id = query_value_exact(&filtered_uri, "versionId").filter(|value| !value.is_empty()); - let range = parse_optional_header(&req.headers, http::header::RANGE)? + let range = parse_optional_header(&req.headers, header::RANGE)? .map(|value| Range::parse(&value).map_err(|_| s3_error!(InvalidArgument, "Range header is invalid"))) .transpose()?; @@ -520,13 +520,10 @@ fn build_object_lambda_get_request(req: &S3Request, bucket: &str, object: .part_number(part_number) .version_id(version_id) .range(range) - .if_match(parse_optional_etag_condition_header::(&req.headers, http::header::IF_MATCH)?) - .if_none_match(parse_optional_etag_condition_header::( - &req.headers, - http::header::IF_NONE_MATCH, - )?) - .if_modified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_MODIFIED_SINCE)?) - .if_unmodified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_UNMODIFIED_SINCE)?); + .if_match(parse_optional_etag_condition_header::(&req.headers, header::IF_MATCH)?) + .if_none_match(parse_optional_etag_condition_header::(&req.headers, header::IF_NONE_MATCH)?) + .if_modified_since(parse_optional_timestamp_header(&req.headers, header::IF_MODIFIED_SINCE)?) + .if_unmodified_since(parse_optional_timestamp_header(&req.headers, header::IF_UNMODIFIED_SINCE)?); builder = builder.sse_customer_algorithm(parse_optional_header( &req.headers, diff --git a/rustfs/src/server/audit.rs b/rustfs/src/server/audit.rs index 75f8a89a1..ceb42b65a 100644 --- a/rustfs/src/server/audit.rs +++ b/rustfs/src/server/audit.rs @@ -72,13 +72,7 @@ pub async fn start_audit_system() -> AuditResult<()> { // 1. Get the global configuration loaded by ecstore let server_config = match server_config_from_context() { - Some(config) => { - info!( - target: "rustfs::main::start_audit_system", - "Global server configuration loads successfully: {:?}", config - ); - config - } + Some(config) => config, None => { warn!( target: "rustfs::main::start_audit_system", @@ -92,9 +86,8 @@ pub async fn start_audit_system() -> AuditResult<()> { target: "rustfs::main::start_audit_system", "The global server configuration is loaded" ); - // 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't - let has_targets = has_any_audit_targets(&server_config); - if !has_targets { + // 2. Check if the audit subsystem exists in the configuration, and skip initialization if it doesn't + if !has_any_audit_targets(&server_config) { info!( target: "rustfs::main::start_audit_system", "Audit subsystem targets are not configured, and audit system initialization is skipped." @@ -107,35 +100,16 @@ pub async fn start_audit_system() -> AuditResult<()> { "Audit subsystem configuration detected and started initializing the audit system." ); - if let Some(system) = audit_system() { - match system.get_state().await { - AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { - // Match notify behavior: prefer reloading the existing singleton - // instead of constructing a second lifecycle path on re-enable. - match system.reload_config(server_config).await { - Ok(()) => { - info!( - target: "rustfs::main::start_audit_system", - "Audit system reloaded successfully with time: {}.", - jiff::Zoned::now() - ); - Ok(()) - } - Err(e) => { - warn!( - target: "rustfs::main::start_audit_system", - "Audit system reload failed: {:?}", - e - ); - Err(e) - } - } - } - AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await { + let system = audit_system().unwrap_or_else(init_audit_system); + match system.get_state().await { + AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => { + // Match notify behavior: prefer reloading the existing singleton + // instead of constructing a second lifecycle path on re-enable. + match system.reload_config(server_config).await { Ok(()) => { info!( target: "rustfs::main::start_audit_system", - "Audit system started successfully with time: {}.", + "Audit system reloaded successfully with time: {}.", jiff::Zoned::now() ); Ok(()) @@ -143,16 +117,14 @@ pub async fn start_audit_system() -> AuditResult<()> { Err(e) => { warn!( target: "rustfs::main::start_audit_system", - "Audit system startup failed: {:?}", + "Audit system reload failed: {:?}", e ); Err(e) } - }, + } } - } else { - let system = init_audit_system(); - match system.start(server_config).await { + AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await { Ok(()) => { info!( target: "rustfs::main::start_audit_system", @@ -169,7 +141,7 @@ pub async fn start_audit_system() -> AuditResult<()> { ); Err(e) } - } + }, } } diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index 1de42c696..6443ec934 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -17,6 +17,7 @@ use crate::app::context::resolve_server_config; use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook}; use rustfs_notify::EventArgs as NotifyEventArgs; use rustfs_s3_common::EventName; +use std::net::SocketAddr; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::spawn; use tracing::{error, info, instrument, warn}; @@ -39,13 +40,7 @@ pub fn is_notify_module_enabled() -> bool { fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs { let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default(); - let (host, port) = match args.host.rsplit_once(':') { - Some((host, port)) => match port.parse::() { - Ok(port) => (host.to_string(), port), - Err(_) => (args.host, 0), - }, - None => (args.host, 0), - }; + let (host, port) = parse_host_and_port(args.host); let req_params = args.req_params.into_iter().collect(); let resp_elements = args.resp_elements.into_iter().collect(); @@ -62,6 +57,24 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs { } } +fn parse_host_and_port(host: String) -> (String, u16) { + if let Ok(addr) = host.parse::() { + return (addr.ip().to_string(), addr.port()); + } + + if host.chars().filter(|&c| c == ':').count() != 1 { + return (host, 0); + } + + match host.split_once(':') { + Some((base, port)) if !base.is_empty() => match port.parse::() { + Ok(port) => (base.to_string(), port), + Err(_) => (host, 0), + }, + _ => (host, 0), + } +} + fn install_ecstore_event_dispatch_hook() { let installed = register_event_dispatch_hook(|args| { let notify_args = convert_ecstore_event_args(args); @@ -75,6 +88,23 @@ fn install_ecstore_event_dispatch_hook() { } } +fn ensure_live_events_initialized() -> bool { + if rustfs_notify::notification_system().is_some() { + return true; + } + + match rustfs_notify::initialize_live_events() { + Ok(()) => { + install_ecstore_event_dispatch_hook(); + true + } + Err(e) => { + error!("Failed to initialize live event stream support: {}", e); + false + } + } +} + /// Shuts down the event notifier system gracefully pub async fn shutdown_event_notifier() { info!("Shutting down event notifier system..."); @@ -110,17 +140,11 @@ pub async fn init_event_notifier() { "Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.", rustfs_config::ENV_NOTIFY_ENABLE ); - if rustfs_notify::notification_system().is_none() { - match rustfs_notify::initialize_live_events() { - Ok(()) => { - install_ecstore_event_dispatch_hook(); - info!( - target: "rustfs::main::init_event_notifier", - "Live event stream support initialized successfully." - ); - } - Err(e) => error!("Failed to initialize live event stream support: {}", e), - } + if ensure_live_events_initialized() { + info!( + target: "rustfs::main::init_event_notifier", + "Live event stream support initialized successfully." + ); } return; } @@ -155,13 +179,49 @@ pub async fn init_event_notifier() { "Event notifier system reloaded successfully." ); } - } else if let Err(e) = rustfs_notify::initialize(server_config).await { - error!("Failed to initialize event notifier system: {}", e); } else { - install_ecstore_event_dispatch_hook(); - info!( - target: "rustfs::main::init_event_notifier", - "Event notifier system initialized successfully." - ); + match rustfs_notify::initialize(server_config).await { + Ok(()) => { + install_ecstore_event_dispatch_hook(); + info!( + target: "rustfs::main::init_event_notifier", + "Event notifier system initialized successfully." + ); + } + Err(e) => error!("Failed to initialize event notifier system: {}", e), + } + } +} + +#[cfg(test)] +mod tests { + use super::parse_host_and_port; + + #[test] + fn parse_host_and_port_with_ipv4_and_port() { + let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string()); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9000); + } + + #[test] + fn parse_host_and_port_with_bracketed_ipv6_and_port() { + let (host, port) = parse_host_and_port("[::1]:9000".to_string()); + assert_eq!(host, "::1"); + assert_eq!(port, 9000); + } + + #[test] + fn parse_host_and_port_with_ipv6_without_port() { + let (host, port) = parse_host_and_port("::1".to_string()); + assert_eq!(host, "::1"); + assert_eq!(port, 0); + } + + #[test] + fn parse_host_and_port_with_hostname_and_port() { + let (host, port) = parse_host_and_port("localhost:9001".to_string()); + assert_eq!(host, "localhost"); + assert_eq!(port, 9001); } }