gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-28 09:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(targets): add AMQP support for notify and audit (#2879)

Browse Source 
Co-authored-by: Hyesook Yun <74169420+suk13574@users.noreply.github.com>
This commit is contained in:
houseme
2026-05-09 09:56:26 +08:00
committed by GitHub
parent 1582f216fe
commit 81ad48dac2
29 changed files with 3280 additions and 830 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+526 -110
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+2 -1
View File
@@ -126,6 +126,7 @@ futures-core = "0.3.32"
futures-util = "0.3.32"
pollster = "0.4.0"
pulsar = { version = "6.7.2", default-features = false, features = ["tokio-rustls-runtime"] }
lapin = { version = "4.7.2", default-features = false, features = ["tokio", "rustls", "rustls--aws_lc_rs"] }
hyper = { version = "1.9.0", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
@@ -135,7 +136,7 @@ http-body-util = "0.1.3"
reqwest = { version = "0.13.3", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
rustfs-kafka-async = { version = "1.2.0" }
socket2 = { version = "0.6.3", features = ["all"] }
tokio = { version = "1.52.2", features = ["fs", "rt-multi-thread"] }
tokio = { version = "1.52.3", features = ["fs", "rt-multi-thread"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
tokio-stream = { version = "0.1.18" }
tokio-test = "0.4.5"
crates/audit/src/factory.rs
+173 -170
View File
@@ -13,189 +13,192 @@
// limitations under the License.
use crate::AuditEntry;
use async_trait::async_trait;
use rustfs_config::AUDIT_DEFAULT_DIR;
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, AUDIT_PULSAR_KEYS,
AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS,
AUDIT_AMQP_KEYS, AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS,
AUDIT_PULSAR_KEYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS,
};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
use rustfs_targets::config::{
build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
validate_webhook_config,
};
use std::collections::HashSet;
use rustfs_targets::target::{ChannelTargetType, TargetType};
use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target};
/// Trait for creating targets from configuration
#[async_trait]
pub trait TargetFactory: Send + Sync {
/// Creates a target from configuration
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError>;
/// Validates target configuration
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>;
/// Returns a set of valid configuration field names for this target type.
/// This is used to filter environment variables.
fn get_valid_fields(&self) -> HashSet<String>;
pub fn builtin_target_descriptors() -> Vec<BuiltinTargetDescriptor<AuditEntry>> {
vec![
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_AMQP_SUB_SYS,
TargetRequestValidator::Amqp(TargetType::AuditLog),
TargetPluginDescriptor::new(
ChannelTargetType::Amqp.as_str(),
AUDIT_AMQP_KEYS,
|config| validate_amqp_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_amqp_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
TargetRequestValidator::Webhook,
TargetPluginDescriptor::new(
ChannelTargetType::Webhook.as_str(),
AUDIT_WEBHOOK_KEYS,
|config| validate_webhook_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
TargetRequestValidator::Mqtt,
TargetPluginDescriptor::new(ChannelTargetType::Mqtt.as_str(), AUDIT_MQTT_KEYS, validate_mqtt_config, |id, config| {
let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?))
}),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_NATS_SUB_SYS,
TargetRequestValidator::Nats(TargetType::AuditLog),
TargetPluginDescriptor::new(
ChannelTargetType::Nats.as_str(),
AUDIT_NATS_KEYS,
|config| validate_nats_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_PULSAR_SUB_SYS,
TargetRequestValidator::Pulsar(TargetType::AuditLog),
TargetPluginDescriptor::new(
ChannelTargetType::Pulsar.as_str(),
AUDIT_PULSAR_KEYS,
|config| validate_pulsar_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_KAFKA_SUB_SYS,
TargetRequestValidator::Kafka(TargetType::AuditLog),
TargetPluginDescriptor::new(
ChannelTargetType::Kafka.as_str(),
AUDIT_KAFKA_KEYS,
|config| validate_kafka_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_REDIS_SUB_SYS,
TargetRequestValidator::Redis {
default_channel: AUDIT_REDIS_DEFAULT_CHANNEL,
target_type: TargetType::AuditLog,
},
TargetPluginDescriptor::new(
ChannelTargetType::Redis.as_str(),
AUDIT_REDIS_KEYS,
|config| validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL),
|id, config| {
let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_MYSQL_SUB_SYS,
TargetRequestValidator::MySql,
TargetPluginDescriptor::new(
ChannelTargetType::MySql.as_str(),
AUDIT_MYSQL_KEYS,
|config| validate_mysql_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
rustfs_config::audit::AUDIT_POSTGRES_SUB_SYS,
TargetRequestValidator::Postgres(TargetType::AuditLog),
TargetPluginDescriptor::new(
ChannelTargetType::Postgres.as_str(),
AUDIT_POSTGRES_KEYS,
|config| validate_postgres_config(config, AUDIT_DEFAULT_DIR),
|id, config| {
let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?))
},
),
),
]
}
/// Factory for creating Webhook targets
pub struct WebhookTargetFactory;
#[async_trait]
impl TargetFactory for WebhookTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_webhook_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_webhook_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
}
pub fn builtin_target_plugins() -> Vec<TargetPluginDescriptor<AuditEntry>> {
builtin_target_descriptors()
.into_iter()
.map(|descriptor| descriptor.plugin().clone())
.collect()
}
/// Factory for creating MQTT targets
pub struct MQTTTargetFactory;
#[cfg(test)]
mod tests {
use super::builtin_target_descriptors;
use rustfs_config::audit::AUDIT_AMQP_KEYS;
use rustfs_config::{AMQP_EXCHANGE, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_URL};
use rustfs_ecstore::config::KVS;
use rustfs_targets::target::ChannelTargetType;
#[async_trait]
impl TargetFactory for MQTTTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_mqtt_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
Ok(Box::new(target))
fn amqp_base_config() -> KVS {
let mut config = KVS::new();
config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string());
config.insert(AMQP_EXCHANGE.to_string(), "rustfs.audit".to_string());
config.insert(AMQP_ROUTING_KEY.to_string(), "audit".to_string());
config.insert(AMQP_QUEUE_DIR.to_string(), String::new());
config
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mqtt_config(config)
#[test]
fn builtin_plugins_include_amqp_descriptor() {
let plugin = builtin_target_descriptors()
.into_iter()
.find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str())
.expect("amqp plugin should exist");
assert!(plugin.plugin().valid_fields().contains(&AMQP_URL));
assert!(plugin.plugin().valid_fields().contains(&AMQP_EXCHANGE));
assert!(plugin.plugin().valid_fields().contains(&AMQP_ROUTING_KEY));
assert_eq!(plugin.plugin().valid_fields().len(), AUDIT_AMQP_KEYS.len());
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct NATSTargetFactory;
#[async_trait]
impl TargetFactory for NATSTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_nats_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_nats_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_NATS_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct PulsarTargetFactory;
#[async_trait]
impl TargetFactory for PulsarTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_pulsar_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_pulsar_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct KafkaTargetFactory;
#[async_trait]
impl TargetFactory for KafkaTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_kafka_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct RedisTargetFactory;
#[async_trait]
impl TargetFactory for RedisTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?;
let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_REDIS_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct MySqlTargetFactory;
#[async_trait]
impl TargetFactory for MySqlTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mysql_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_MYSQL_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct PostgresTargetFactory;
#[async_trait]
impl TargetFactory for PostgresTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_postgres_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect()
#[test]
fn builtin_plugins_create_audit_amqp_target() {
let plugin = builtin_target_descriptors()
.into_iter()
.find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str())
.expect("amqp plugin should exist");
let target = plugin
.plugin()
.create_target("primary".to_string(), &amqp_base_config())
.expect("AMQP audit target should be created");
let target_id = target.id();
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "amqp");
assert!(target.store().is_none());
}
}
crates/audit/src/registry.rs
+30 -76
View File
@@ -12,29 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
AuditEntry, AuditError, AuditResult,
factory::{
KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory,
RedisTargetFactory, TargetFactory, WebhookTargetFactory,
},
};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use crate::{AuditEntry, AuditError, AuditResult, factory::builtin_target_plugins};
use hashbrown::HashMap;
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
use rustfs_ecstore::config::{Config, KVS};
use rustfs_targets::arn::TargetID;
use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
use std::sync::Arc;
use rustfs_targets::{Target, TargetError, TargetPluginRegistry};
use tracing::{error, info};
/// Registry for managing audit targets
pub struct AuditRegistry {
/// Storage for created targets
targets: HashMap<String, Box<dyn Target<AuditEntry> + Send + Sync>>,
/// Factories for creating targets
factories: HashMap<String, Box<dyn TargetFactory>>,
/// Registered plugins for creating targets
plugins: TargetPluginRegistry<AuditEntry>,
}
impl Default for AuditRegistry {
@@ -46,31 +37,17 @@ impl Default for AuditRegistry {
impl AuditRegistry {
/// Creates a new AuditRegistry
pub fn new() -> Self {
let mut registry = AuditRegistry {
factories: HashMap::new(),
let mut plugins = TargetPluginRegistry::new();
plugins.register_all(builtin_target_plugins());
AuditRegistry {
targets: HashMap::new(),
};
// Register built-in factories
registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory));
registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory));
registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory));
registry
plugins,
}
}
/// Registers a new factory for a target type
///
/// # Arguments
/// * `target_type` - The type of the target (e.g., "webhook", "mqtt").
/// * `factory` - The factory instance to create targets of this type.
pub fn register(&mut self, target_type: &str, factory: Box<dyn TargetFactory>) {
self.factories.insert(target_type.to_string(), factory);
pub fn supports_target_type(&self, target_type: &str) -> bool {
self.plugins.supports_target_type(target_type)
}
/// Creates a target of the specified type with the given ID and configuration
@@ -88,16 +65,7 @@ impl AuditRegistry {
id: String,
config: &KVS,
) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let factory = self
.factories
.get(target_type)
.ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?;
// Validate configuration before creating target
factory.validate_config(&id, config)?;
// Create target
factory.create_target(id, config).await
self.plugins.create_target(target_type, id, config)
}
/// Creates all targets from a configuration
@@ -113,37 +81,10 @@ impl AuditRegistry {
&self,
config: &Config,
) -> AuditResult<Vec<Box<dyn Target<AuditEntry> + Send + Sync>>> {
let mut tasks = FuturesUnordered::new();
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
info!("Start working on target types...");
let valid_fields = factory.get_valid_fields();
for (id, merged_config) in collect_target_configs(config, AUDIT_ROUTE_PREFIX, target_type, &valid_fields) {
info!(instance_id = %id, "Target is enabled, ready to create a task");
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(tid, result)
});
}
}
let mut successful_targets = Vec::new();
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(target_type = %target.id().name, instance_id = %id, "Create a target successfully");
successful_targets.push(target);
}
Err(e) => {
error!(instance_id = %id, error = %e, "Failed to create a target");
}
}
}
info!(count = successful_targets.len(), "All target processing completed");
Ok(successful_targets)
self.plugins
.create_targets_from_config(config, AUDIT_ROUTE_PREFIX)
.await
.map_err(AuditError::from)
}
/// Adds a target to the registry
@@ -287,3 +228,16 @@ impl AuditRegistry {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::AuditRegistry;
use rustfs_targets::target::ChannelTargetType;
#[test]
fn registry_registers_amqp_factory() {
let registry = AuditRegistry::new();
assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str()));
}
}
crates/config/src/audit/amqp.rs
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub const AUDIT_AMQP_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::AMQP_URL,
crate::AMQP_EXCHANGE,
crate::AMQP_ROUTING_KEY,
crate::AMQP_MANDATORY,
crate::AMQP_PERSISTENT,
crate::AMQP_USERNAME,
crate::AMQP_PASSWORD,
crate::AMQP_TLS_CA,
crate::AMQP_TLS_CLIENT_CERT,
crate::AMQP_TLS_CLIENT_KEY,
crate::AMQP_QUEUE_DIR,
crate::AMQP_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
pub const ENV_AUDIT_AMQP_ENABLE: &str = "RUSTFS_AUDIT_AMQP_ENABLE";
pub const ENV_AUDIT_AMQP_URL: &str = "RUSTFS_AUDIT_AMQP_URL";
pub const ENV_AUDIT_AMQP_EXCHANGE: &str = "RUSTFS_AUDIT_AMQP_EXCHANGE";
pub const ENV_AUDIT_AMQP_ROUTING_KEY: &str = "RUSTFS_AUDIT_AMQP_ROUTING_KEY";
pub const ENV_AUDIT_AMQP_MANDATORY: &str = "RUSTFS_AUDIT_AMQP_MANDATORY";
pub const ENV_AUDIT_AMQP_PERSISTENT: &str = "RUSTFS_AUDIT_AMQP_PERSISTENT";
pub const ENV_AUDIT_AMQP_USERNAME: &str = "RUSTFS_AUDIT_AMQP_USERNAME";
pub const ENV_AUDIT_AMQP_PASSWORD: &str = "RUSTFS_AUDIT_AMQP_PASSWORD";
pub const ENV_AUDIT_AMQP_TLS_CA: &str = "RUSTFS_AUDIT_AMQP_TLS_CA";
pub const ENV_AUDIT_AMQP_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_AMQP_TLS_CLIENT_CERT";
pub const ENV_AUDIT_AMQP_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_AMQP_TLS_CLIENT_KEY";
pub const ENV_AUDIT_AMQP_QUEUE_DIR: &str = "RUSTFS_AUDIT_AMQP_QUEUE_DIR";
pub const ENV_AUDIT_AMQP_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_AMQP_QUEUE_LIMIT";
pub const ENV_AUDIT_AMQP_KEYS: &[&str; 13] = &[
ENV_AUDIT_AMQP_ENABLE,
ENV_AUDIT_AMQP_URL,
ENV_AUDIT_AMQP_EXCHANGE,
ENV_AUDIT_AMQP_ROUTING_KEY,
ENV_AUDIT_AMQP_MANDATORY,
ENV_AUDIT_AMQP_PERSISTENT,
ENV_AUDIT_AMQP_USERNAME,
ENV_AUDIT_AMQP_PASSWORD,
ENV_AUDIT_AMQP_TLS_CA,
ENV_AUDIT_AMQP_TLS_CLIENT_CERT,
ENV_AUDIT_AMQP_TLS_CLIENT_KEY,
ENV_AUDIT_AMQP_QUEUE_DIR,
ENV_AUDIT_AMQP_QUEUE_LIMIT,
];
crates/config/src/audit/mod.rs
+4 -2
View File
@@ -16,6 +16,7 @@
//! This module defines the configuration for audit systems, including
//! webhook and MQTT audit-related settings.
mod amqp;
mod kafka;
mod mqtt;
mod mysql;
@@ -25,6 +26,7 @@ mod pulsar;
mod redis;
mod webhook;
pub use amqp::*;
pub use kafka::*;
pub use mqtt::*;
pub use mysql::*;
@@ -41,6 +43,7 @@ pub const AUDIT_PREFIX: &str = "audit";
pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
pub const AUDIT_AMQP_SUB_SYS: &str = "audit_amqp";
pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
pub const AUDIT_MYSQL_SUB_SYS: &str = "audit_mysql";
@@ -49,10 +52,9 @@ pub const AUDIT_POSTGRES_SUB_SYS: &str = "audit_postgres";
pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
pub const AUDIT_REDIS_SUB_SYS: &str = "audit_redis";
pub const AUDIT_REDIS_DEFAULT_CHANNEL: &str = "rustfs_audit_channel";
pub const AUDIT_STORE_EXTENSION: &str = ".audit";
#[allow(dead_code)]
pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
AUDIT_AMQP_SUB_SYS,
AUDIT_KAFKA_SUB_SYS,
AUDIT_MQTT_SUB_SYS,
AUDIT_MYSQL_SUB_SYS,
crates/config/src/constants/targets.rs
+13
View File
@@ -50,6 +50,19 @@ pub const KAFKA_TLS_CA: &str = "tls_ca";
pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const AMQP_URL: &str = "url";
pub const AMQP_EXCHANGE: &str = "exchange";
pub const AMQP_ROUTING_KEY: &str = "routing_key";
pub const AMQP_MANDATORY: &str = "mandatory";
pub const AMQP_PERSISTENT: &str = "persistent";
pub const AMQP_USERNAME: &str = "username";
pub const AMQP_PASSWORD: &str = "password";
pub const AMQP_TLS_CA: &str = "tls_ca";
pub const AMQP_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const AMQP_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const AMQP_QUEUE_DIR: &str = "queue_dir";
pub const AMQP_QUEUE_LIMIT: &str = "queue_limit";
pub const NATS_ADDRESS: &str = "address";
pub const NATS_SUBJECT: &str = "subject";
pub const NATS_USERNAME: &str = "username";
crates/config/src/notify/amqp.rs
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub const NOTIFY_AMQP_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::AMQP_URL,
crate::AMQP_EXCHANGE,
crate::AMQP_ROUTING_KEY,
crate::AMQP_MANDATORY,
crate::AMQP_PERSISTENT,
crate::AMQP_USERNAME,
crate::AMQP_PASSWORD,
crate::AMQP_TLS_CA,
crate::AMQP_TLS_CLIENT_CERT,
crate::AMQP_TLS_CLIENT_KEY,
crate::AMQP_QUEUE_DIR,
crate::AMQP_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
pub const ENV_NOTIFY_AMQP_ENABLE: &str = "RUSTFS_NOTIFY_AMQP_ENABLE";
pub const ENV_NOTIFY_AMQP_URL: &str = "RUSTFS_NOTIFY_AMQP_URL";
pub const ENV_NOTIFY_AMQP_EXCHANGE: &str = "RUSTFS_NOTIFY_AMQP_EXCHANGE";
pub const ENV_NOTIFY_AMQP_ROUTING_KEY: &str = "RUSTFS_NOTIFY_AMQP_ROUTING_KEY";
pub const ENV_NOTIFY_AMQP_MANDATORY: &str = "RUSTFS_NOTIFY_AMQP_MANDATORY";
pub const ENV_NOTIFY_AMQP_PERSISTENT: &str = "RUSTFS_NOTIFY_AMQP_PERSISTENT";
pub const ENV_NOTIFY_AMQP_USERNAME: &str = "RUSTFS_NOTIFY_AMQP_USERNAME";
pub const ENV_NOTIFY_AMQP_PASSWORD: &str = "RUSTFS_NOTIFY_AMQP_PASSWORD";
pub const ENV_NOTIFY_AMQP_TLS_CA: &str = "RUSTFS_NOTIFY_AMQP_TLS_CA";
pub const ENV_NOTIFY_AMQP_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_AMQP_TLS_CLIENT_CERT";
pub const ENV_NOTIFY_AMQP_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_AMQP_TLS_CLIENT_KEY";
pub const ENV_NOTIFY_AMQP_QUEUE_DIR: &str = "RUSTFS_NOTIFY_AMQP_QUEUE_DIR";
pub const ENV_NOTIFY_AMQP_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_AMQP_QUEUE_LIMIT";
pub const ENV_NOTIFY_AMQP_KEYS: &[&str; 13] = &[
ENV_NOTIFY_AMQP_ENABLE,
ENV_NOTIFY_AMQP_URL,
ENV_NOTIFY_AMQP_EXCHANGE,
ENV_NOTIFY_AMQP_ROUTING_KEY,
ENV_NOTIFY_AMQP_MANDATORY,
ENV_NOTIFY_AMQP_PERSISTENT,
ENV_NOTIFY_AMQP_USERNAME,
ENV_NOTIFY_AMQP_PASSWORD,
ENV_NOTIFY_AMQP_TLS_CA,
ENV_NOTIFY_AMQP_TLS_CLIENT_CERT,
ENV_NOTIFY_AMQP_TLS_CLIENT_KEY,
ENV_NOTIFY_AMQP_QUEUE_DIR,
ENV_NOTIFY_AMQP_QUEUE_LIMIT,
];
crates/config/src/notify/mod.rs
+3 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod amqp;
mod arn;
mod kafka;
mod mqtt;
@@ -23,6 +24,7 @@ mod redis;
mod store;
mod webhook;
pub use amqp::*;
pub use arn::*;
pub use kafka::*;
pub use mqtt::*;
@@ -76,6 +78,7 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
NOTIFY_AMQP_SUB_SYS,
NOTIFY_KAFKA_SUB_SYS,
NOTIFY_MQTT_SUB_SYS,
NOTIFY_MYSQL_SUB_SYS,
@@ -95,7 +98,6 @@ pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
#[allow(dead_code)]
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
#[allow(dead_code)]
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
#[allow(dead_code)]
crates/ecstore/src/config/audit.rs
+85 -8
View File
@@ -15,14 +15,16 @@
use crate::config::{KV, KVS};
use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA,
KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL,
MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT,
MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST,
MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
@@ -206,6 +208,81 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
])
});
pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_URL.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_EXCHANGE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_ROUTING_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_MANDATORY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_PERSISTENT.to_owned(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
crates/ecstore/src/config/com.rs
+194 -18
View File
@@ -19,14 +19,15 @@ use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS,
AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS,
NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS,
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
@@ -60,7 +61,7 @@ struct TargetConfigDescriptor {
valid_keys: &'static [&'static str],
}
fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] {
fn notify_target_descriptors() -> [TargetConfigDescriptor; 9] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -68,6 +69,12 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] {
default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
valid_keys: NOTIFY_WEBHOOK_KEYS,
},
TargetConfigDescriptor {
external_key: "amqp",
subsystem_key: NOTIFY_AMQP_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_AMQP_KVS,
valid_keys: NOTIFY_AMQP_KEYS,
},
TargetConfigDescriptor {
external_key: "kafka",
subsystem_key: NOTIFY_KAFKA_SUB_SYS,
@@ -113,7 +120,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] {
]
}
fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] {
fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -121,6 +128,12 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] {
default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
valid_keys: AUDIT_WEBHOOK_KEYS,
},
TargetConfigDescriptor {
external_key: "amqp",
subsystem_key: AUDIT_AMQP_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_AMQP_KVS,
valid_keys: AUDIT_AMQP_KEYS,
},
TargetConfigDescriptor {
external_key: "kafka",
subsystem_key: AUDIT_KAFKA_SUB_SYS,
@@ -717,6 +730,8 @@ fn is_target_bool_key(key: &str) -> bool {
matches!(
key,
ENABLE_KEY
| rustfs_config::AMQP_MANDATORY
| rustfs_config::AMQP_PERSISTENT
| rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
| rustfs_config::KAFKA_TLS_ENABLE
| rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
@@ -726,14 +741,31 @@ fn is_target_bool_key(key: &str) -> bool {
)
}
fn parse_target_bool_scalar(value: &str) -> Option<bool> {
if let Ok(state) = value.parse::<EnableState>() {
return Some(state.is_enabled());
}
if let Ok(boolean) = value.parse::<bool>() {
return Some(boolean);
}
None
}
fn target_scalar_values_equal(key: &str, lhs: &str, rhs: &str) -> bool {
if is_target_bool_key(key)
&& let (Some(lhs), Some(rhs)) = (parse_target_bool_scalar(lhs), parse_target_bool_scalar(rhs))
{
return lhs == rhs;
}
lhs == rhs
}
fn encode_target_scalar_value(key: &str, value: &str) -> Value {
if is_target_bool_key(key) {
if let Ok(state) = value.parse::<EnableState>() {
return Value::Bool(state.is_enabled());
}
if let Ok(boolean) = value.parse::<bool>() {
return Value::Bool(boolean);
}
if is_target_bool_key(key)
&& let Some(boolean) = parse_target_bool_scalar(value)
{
return Value::Bool(boolean);
}
Value::String(value.to_string())
@@ -759,7 +791,7 @@ fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
let baseline_value = baseline.lookup(key).unwrap_or_default();
let effective_value = kvs.lookup(key).unwrap_or_else(|| baseline_value.clone());
if effective_value == baseline_value {
if target_scalar_values_equal(key, &effective_value, &baseline_value) {
continue;
}
@@ -1169,8 +1201,10 @@ mod tests {
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use rustfs_config::{
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
@@ -1816,6 +1850,15 @@ mod tests {
"tls_enable":true
}
},
"amqp":{
"primary":{
"enable":true,
"url":"amqp://127.0.0.1:5672/%2f",
"exchange":"rustfs.events",
"routing_key":"objects",
"persistent":true
}
},
"mysql":{
"primary":{
"enable":true,
@@ -1861,6 +1904,15 @@ mod tests {
assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
let amqp = cfg
.get_value(NOTIFY_AMQP_SUB_SYS, "primary")
.expect("amqp target should be decoded");
assert_eq!(amqp.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(amqp.get(rustfs_config::AMQP_URL), "amqp://127.0.0.1:5672/%2f");
assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE), "rustfs.events");
assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY), "objects");
assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT), "true");
let mysql = cfg
.get_value(NOTIFY_MYSQL_SUB_SYS, "primary")
.expect("mysql target should be decoded");
@@ -1928,6 +1980,15 @@ mod tests {
"queue_dir":"/tmp/audit-queue"
}
},
"amqp":{
"primary":{
"enable":true,
"url":"amqp://127.0.0.1:5672/%2f",
"exchange":"rustfs.audit",
"routing_key":"audit",
"persistent":true
}
},
"mqtt":{
"default":{
"enable":true,
@@ -1959,6 +2020,15 @@ mod tests {
assert_eq!(webhook.get(rustfs_config::WEBHOOK_ENDPOINT), "https://example.com/audit-hook");
assert_eq!(webhook.get(rustfs_config::WEBHOOK_QUEUE_DIR), "/tmp/audit-queue");
let amqp = cfg
.get_value(AUDIT_AMQP_SUB_SYS, "primary")
.expect("audit amqp target should be decoded");
assert_eq!(amqp.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(amqp.get(rustfs_config::AMQP_URL), "amqp://127.0.0.1:5672/%2f");
assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE), "rustfs.audit");
assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY), "audit");
assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT), "true");
let mqtt_default = cfg
.get_value(AUDIT_MQTT_SUB_SYS, DEFAULT_DELIMITER)
.expect("audit mqtt default should be decoded");
@@ -2127,6 +2197,44 @@ mod tests {
);
cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
let mut amqp_section = std::collections::HashMap::new();
amqp_section.insert(
"primary".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_URL.to_string(),
value: "amqp://127.0.0.1:5672/%2f".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_EXCHANGE.to_string(),
value: "rustfs.events".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_ROUTING_KEY.to_string(),
value: "objects".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_MANDATORY.to_string(),
value: "false".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_PERSISTENT.to_string(),
value: "false".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(NOTIFY_AMQP_SUB_SYS.to_string(), amqp_section);
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
let notify = v
@@ -2175,6 +2283,21 @@ mod tests {
);
assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
let amqp = notify
.get("amqp")
.and_then(Value::as_object)
.and_then(|targets| targets.get("primary"))
.and_then(Value::as_object)
.expect("amqp target should be encoded");
assert_eq!(
amqp.get(rustfs_config::AMQP_URL).and_then(Value::as_str),
Some("amqp://127.0.0.1:5672/%2f")
);
assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE).and_then(Value::as_str), Some("rustfs.events"));
assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY).and_then(Value::as_str), Some("objects"));
assert!(!amqp.contains_key(rustfs_config::AMQP_MANDATORY));
assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT).and_then(Value::as_bool), Some(false));
}
#[test]
@@ -2204,6 +2327,44 @@ mod tests {
);
cfg.0.insert(AUDIT_WEBHOOK_SUB_SYS.to_string(), webhook_section);
let mut amqp_section = std::collections::HashMap::new();
amqp_section.insert(
"primary".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_URL.to_string(),
value: "amqp://127.0.0.1:5672/%2f".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_EXCHANGE.to_string(),
value: "rustfs.audit".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_ROUTING_KEY.to_string(),
value: "audit".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_MANDATORY.to_string(),
value: "false".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::AMQP_PERSISTENT.to_string(),
value: "false".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(AUDIT_AMQP_SUB_SYS.to_string(), amqp_section);
let mut mqtt_default = audit::DEFAULT_AUDIT_MQTT_KVS.clone();
mqtt_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
mqtt_default.insert(rustfs_config::MQTT_TOPIC.to_string(), "audit-events".to_string());
@@ -2266,6 +2427,21 @@ mod tests {
);
assert_eq!(webhook.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
let amqp = logger
.get("amqp")
.and_then(Value::as_object)
.and_then(|targets| targets.get("primary"))
.and_then(Value::as_object)
.expect("audit amqp target should be encoded");
assert_eq!(
amqp.get(rustfs_config::AMQP_URL).and_then(Value::as_str),
Some("amqp://127.0.0.1:5672/%2f")
);
assert_eq!(amqp.get(rustfs_config::AMQP_EXCHANGE).and_then(Value::as_str), Some("rustfs.audit"));
assert_eq!(amqp.get(rustfs_config::AMQP_ROUTING_KEY).and_then(Value::as_str), Some("audit"));
assert!(!amqp.contains_key(rustfs_config::AMQP_MANDATORY));
assert_eq!(amqp.get(rustfs_config::AMQP_PERSISTENT).and_then(Value::as_bool), Some(false));
let mqtt_default = logger
.get("mqtt")
.and_then(Value::as_object)
crates/ecstore/src/config/mod.rs
+5 -3
View File
@@ -26,12 +26,12 @@ use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use rustfs_config::COMMENT_KEY;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::audit::{
AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS,
AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS,
AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use serde::{Deserialize, Serialize};
@@ -243,6 +243,8 @@ pub fn init() {
kvs.insert(AUDIT_WEBHOOK_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_WEBHOOK_KVS.clone());
kvs.insert(NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MQTT_KVS.clone());
kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
kvs.insert(NOTIFY_AMQP_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_AMQP_KVS.clone());
kvs.insert(AUDIT_AMQP_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_AMQP_KVS.clone());
kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
kvs.insert(NOTIFY_REDIS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_REDIS_KVS.clone());
crates/ecstore/src/config/notify.rs
+85 -8
View File
@@ -15,14 +15,16 @@
use crate::config::{KV, KVS};
use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA,
KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL,
MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT,
MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST,
MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
@@ -184,6 +186,81 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
])
});
pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_URL.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_EXCHANGE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_ROUTING_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_MANDATORY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_PERSISTENT.to_owned(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
KV {
key: AMQP_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: AMQP_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: AMQP_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
crates/notify/src/factory.rs
+181 -170
View File
@@ -13,189 +13,200 @@
// limitations under the License.
use crate::Event;
use async_trait::async_trait;
use rustfs_config::EVENT_DEFAULT_DIR;
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_MYSQL_KEYS, NOTIFY_NATS_KEYS, NOTIFY_POSTGRES_KEYS, NOTIFY_PULSAR_KEYS,
NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_WEBHOOK_KEYS,
NOTIFY_AMQP_KEYS, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS,
NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS,
NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
use rustfs_targets::config::{
build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
validate_webhook_config,
};
use std::collections::HashSet;
use rustfs_targets::target::{ChannelTargetType, TargetType};
use rustfs_targets::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetRequestValidator, boxed_target};
/// Trait for creating targets from configuration
#[async_trait]
pub trait TargetFactory: Send + Sync {
/// Creates a target from configuration
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError>;
/// Validates target configuration
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError>;
/// Returns a set of valid configuration field names for this target type.
/// This is used to filter environment variables.
fn get_valid_fields(&self) -> HashSet<String>;
pub fn builtin_target_descriptors() -> Vec<BuiltinTargetDescriptor<Event>> {
vec![
BuiltinTargetDescriptor::new(
NOTIFY_WEBHOOK_SUB_SYS,
TargetRequestValidator::Webhook,
TargetPluginDescriptor::new(
ChannelTargetType::Webhook.as_str(),
NOTIFY_WEBHOOK_KEYS,
|config| validate_webhook_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::webhook::WebhookTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_AMQP_SUB_SYS,
TargetRequestValidator::Amqp(TargetType::NotifyEvent),
TargetPluginDescriptor::new(
ChannelTargetType::Amqp.as_str(),
NOTIFY_AMQP_KEYS,
|config| validate_amqp_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_amqp_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::amqp::AMQPTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_KAFKA_SUB_SYS,
TargetRequestValidator::Kafka(TargetType::NotifyEvent),
TargetPluginDescriptor::new(
ChannelTargetType::Kafka.as_str(),
NOTIFY_KAFKA_KEYS,
|config| validate_kafka_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::kafka::KafkaTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_MQTT_SUB_SYS,
TargetRequestValidator::Mqtt,
TargetPluginDescriptor::new(
ChannelTargetType::Mqtt.as_str(),
NOTIFY_MQTT_KEYS,
validate_mqtt_config,
|id, config| {
let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_MYSQL_SUB_SYS,
TargetRequestValidator::MySql,
TargetPluginDescriptor::new(
ChannelTargetType::MySql.as_str(),
NOTIFY_MYSQL_KEYS,
|config| validate_mysql_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::mysql::MySqlTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_NATS_SUB_SYS,
TargetRequestValidator::Nats(TargetType::NotifyEvent),
TargetPluginDescriptor::new(
ChannelTargetType::Nats.as_str(),
NOTIFY_NATS_KEYS,
|config| validate_nats_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::nats::NATSTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_POSTGRES_SUB_SYS,
TargetRequestValidator::Postgres(TargetType::NotifyEvent),
TargetPluginDescriptor::new(
ChannelTargetType::Postgres.as_str(),
NOTIFY_POSTGRES_KEYS,
|config| validate_postgres_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::postgres::PostgresTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_REDIS_SUB_SYS,
TargetRequestValidator::Redis {
default_channel: NOTIFY_REDIS_DEFAULT_CHANNEL,
target_type: TargetType::NotifyEvent,
},
TargetPluginDescriptor::new(
ChannelTargetType::Redis.as_str(),
NOTIFY_REDIS_KEYS,
|config| validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL),
|id, config| {
let args =
build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::redis::RedisTarget::new(id, args)?))
},
),
),
BuiltinTargetDescriptor::new(
NOTIFY_PULSAR_SUB_SYS,
TargetRequestValidator::Pulsar(TargetType::NotifyEvent),
TargetPluginDescriptor::new(
ChannelTargetType::Pulsar.as_str(),
NOTIFY_PULSAR_KEYS,
|config| validate_pulsar_config(config, EVENT_DEFAULT_DIR),
|id, config| {
let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
Ok(boxed_target(rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?))
},
),
),
]
}
/// Factory for creating Webhook targets
pub struct WebhookTargetFactory;
#[async_trait]
impl TargetFactory for WebhookTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_webhook_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::webhook::WebhookTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_webhook_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_WEBHOOK_KEYS.iter().map(|s| s.to_string()).collect()
}
pub fn builtin_target_plugins() -> Vec<TargetPluginDescriptor<Event>> {
builtin_target_descriptors()
.into_iter()
.map(|descriptor| descriptor.plugin().clone())
.collect()
}
/// Factory for creating MQTT targets
pub struct MQTTTargetFactory;
#[cfg(test)]
mod tests {
use super::builtin_target_descriptors;
use rustfs_config::notify::NOTIFY_AMQP_KEYS;
use rustfs_config::{AMQP_EXCHANGE, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_URL};
use rustfs_ecstore::config::KVS;
use rustfs_targets::target::ChannelTargetType;
#[async_trait]
impl TargetFactory for MQTTTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_mqtt_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::mqtt::MQTTTarget::new(id, args)?;
Ok(Box::new(target))
fn amqp_base_config() -> KVS {
let mut config = KVS::new();
config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string());
config.insert(AMQP_EXCHANGE.to_string(), "rustfs.events".to_string());
config.insert(AMQP_ROUTING_KEY.to_string(), "objects".to_string());
config.insert(AMQP_QUEUE_DIR.to_string(), String::new());
config
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mqtt_config(config)
#[test]
fn builtin_plugins_include_amqp_descriptor() {
let plugin = builtin_target_descriptors()
.into_iter()
.find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str())
.expect("amqp plugin should exist");
assert!(plugin.plugin().valid_fields().contains(&AMQP_URL));
assert!(plugin.plugin().valid_fields().contains(&AMQP_EXCHANGE));
assert!(plugin.plugin().valid_fields().contains(&AMQP_ROUTING_KEY));
assert_eq!(plugin.plugin().valid_fields().len(), NOTIFY_AMQP_KEYS.len());
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_MQTT_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct NATSTargetFactory;
#[async_trait]
impl TargetFactory for NATSTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_nats_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::nats::NATSTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_nats_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_NATS_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct PulsarTargetFactory;
#[async_trait]
impl TargetFactory for PulsarTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_pulsar_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::pulsar::PulsarTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_pulsar_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct PostgresTargetFactory;
#[async_trait]
impl TargetFactory for PostgresTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_postgres_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct KafkaTargetFactory;
#[async_trait]
impl TargetFactory for KafkaTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_kafka_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct MySqlTargetFactory;
#[async_trait]
impl TargetFactory for MySqlTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mysql_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_MYSQL_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct RedisTargetFactory;
#[async_trait]
impl TargetFactory for RedisTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_REDIS_KEYS.iter().map(|s| s.to_string()).collect()
#[test]
fn builtin_plugins_create_notify_amqp_target() {
let plugin = builtin_target_descriptors()
.into_iter()
.find(|plugin| plugin.plugin().target_type() == ChannelTargetType::Amqp.as_str())
.expect("amqp plugin should exist");
let target = plugin
.plugin()
.create_target("primary".to_string(), &amqp_base_config())
.expect("AMQP target should be created");
let target_id = target.id();
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "amqp");
assert!(target.store().is_none());
}
}
crates/notify/src/integration.rs
+10 -2
View File
@@ -25,8 +25,8 @@ use crate::{
use hashbrown::HashMap;
use rustfs_config::notify::{
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE,
ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR};
use rustfs_ecstore::config::{Config, KVS};
@@ -54,6 +54,7 @@ fn notify_configuration_hint() -> String {
fn subsystem_target_type(target_type: &str) -> &str {
match target_type {
NOTIFY_AMQP_SUB_SYS => "amqp",
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_KAFKA_SUB_SYS => "kafka",
NOTIFY_MQTT_SUB_SYS => "mqtt",
@@ -778,6 +779,13 @@ mod tests {
assert_eq!(target_id.name, "webhook");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_amqp_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_AMQP_SUB_SYS, "Primary");
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "amqp");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_mqtt_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_MQTT_SUB_SYS, "Analytics");
crates/notify/src/registry.rs
+23 -68
View File
@@ -13,21 +13,14 @@
// limitations under the License.
use crate::Event;
use crate::factory::{
KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory,
RedisTargetFactory, TargetFactory, WebhookTargetFactory,
};
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashMap;
use crate::factory::builtin_target_plugins;
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
use rustfs_ecstore::config::{Config, KVS};
use rustfs_targets::{Target, TargetError, config::collect_target_configs, target::ChannelTargetType};
use std::sync::Arc;
use tracing::{error, info};
use rustfs_targets::{Target, TargetError, TargetPluginRegistry};
/// Registry for managing target factories
pub struct TargetRegistry {
factories: HashMap<String, Box<dyn TargetFactory>>,
plugins: TargetPluginRegistry<Event>,
}
impl Default for TargetRegistry {
@@ -39,26 +32,14 @@ impl Default for TargetRegistry {
impl TargetRegistry {
/// Creates a new TargetRegistry with built-in factories
pub fn new() -> Self {
let mut registry = TargetRegistry {
factories: HashMap::new(),
};
let mut plugins = TargetPluginRegistry::new();
plugins.register_all(builtin_target_plugins());
// Register built-in factories
registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory));
registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory));
registry
TargetRegistry { plugins }
}
/// Registers a new factory for a target type
pub fn register(&mut self, target_type: &str, factory: Box<dyn TargetFactory>) {
self.factories.insert(target_type.to_string(), factory);
pub fn supports_target_type(&self, target_type: &str) -> bool {
self.plugins.supports_target_type(target_type)
}
/// Creates a target from configuration
@@ -68,16 +49,7 @@ impl TargetRegistry {
id: String,
config: &KVS,
) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let factory = self
.factories
.get(target_type)
.ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?;
// Validate configuration before creating target
factory.validate_config(&id, config)?;
// Create target
factory.create_target(id, config).await
self.plugins.create_target(target_type, id, config)
}
/// Creates all targets from a configuration
@@ -93,36 +65,19 @@ impl TargetRegistry {
&self,
config: &Config,
) -> Result<Vec<Box<dyn Target<Event> + Send + Sync>>, TargetError> {
let mut tasks = FuturesUnordered::new();
for (target_type, factory) in &self.factories {
tracing::Span::current().record("target_type", target_type.as_str());
info!("Start working on target types...");
let valid_fields = factory.get_valid_fields();
for (id, merged_config) in collect_target_configs(config, NOTIFY_ROUTE_PREFIX, target_type, &valid_fields) {
info!(instance_id = %id, "Target is enabled, ready to create a task");
let tid = id.clone();
let merged_config_arc = Arc::new(merged_config);
tasks.push(async move {
let result = factory.create_target(tid.clone(), &merged_config_arc).await;
(tid, result)
});
}
}
let mut successful_targets = Vec::new();
while let Some((id, result)) = tasks.next().await {
match result {
Ok(target) => {
info!(instance_id = %id, "Create target successfully");
successful_targets.push(target);
}
Err(e) => {
error!(instance_id = %id, error = %e, "Failed to create target");
}
}
}
info!(count = successful_targets.len(), "All target processing completed");
Ok(successful_targets)
self.plugins.create_targets_from_config(config, NOTIFY_ROUTE_PREFIX).await
}
}
#[cfg(test)]
mod tests {
use super::TargetRegistry;
use rustfs_targets::target::ChannelTargetType;
#[test]
fn registry_registers_amqp_factory() {
let registry = TargetRegistry::new();
assert!(registry.supports_target_type(ChannelTargetType::Amqp.as_str()));
}
}
crates/targets/Cargo.toml
+4 -1
View File
@@ -20,6 +20,7 @@ async-trait = { workspace = true }
async-nats = { workspace = true }
deadpool-postgres = { workspace = true }
hyper-rustls = { workspace = true }
lapin = { workspace = true }
pulsar = { workspace = true }
reqwest = { workspace = true }
rumqttc = { workspace = true }
@@ -31,7 +32,7 @@ serde = { workspace = true }
serde_json = { workspace = true }
snap = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
tokio-postgres = { workspace = true }
tokio-postgres-rustls = { workspace = true }
tracing = { workspace = true }
@@ -42,6 +43,8 @@ sysinfo = { workspace = true, features = ["multithread"] }
rustfs-kafka-async = { workspace = true }
mysql_async = { workspace = true }
chrono = { workspace = true }
parking_lot = { workspace = true }
hashbrown = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
crates/targets/src/check.rs
+20
View File
@@ -220,3 +220,23 @@ pub async fn check_redis_server_available(args: &crate::target::redis::RedisArgs
.await
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Redis connection timed out".to_string())))
}
pub async fn check_amqp_broker_available(args: &crate::target::amqp::AMQPArgs) -> Result<(), crate::TargetError> {
match tokio::time::timeout(std::time::Duration::from_secs(5), async {
let connection = crate::target::amqp::connect_amqp(args).await?;
if !connection.connection.status().connected() || !connection.channel.status().connected() {
return Err(crate::TargetError::NotConnected);
}
connection
.connection
.close(200, "OK".into())
.await
.map_err(|e| crate::TargetError::Network(format!("Failed to close AMQP check connection: {e}")))?;
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err(crate::TargetError::Timeout("AMQP connection timed out".to_string())),
}
}
crates/targets/src/config/mod.rs
+4 -3
View File
@@ -21,7 +21,8 @@ pub use loader::{
collect_target_configs_from_env,
};
pub use target_args::{
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
build_amqp_args, build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args,
build_pulsar_args, build_redis_args, build_webhook_args, validate_amqp_config, validate_kafka_config, validate_mqtt_config,
validate_mysql_config, validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config,
validate_webhook_config,
};
crates/targets/src/config/target_args.rs
+200 -23
View File
@@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v
use crate::error::TargetError;
use crate::target::{
TargetType,
amqp::AMQPArgs,
kafka::KafkaArgs,
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
mysql::MySqlArgs,
@@ -27,22 +28,23 @@ use crate::target::{
};
use rumqttc::QoS;
use rustfs_config::{
DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING,
MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS,
KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC,
MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL,
MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR,
MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL,
REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS,
REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY,
REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use rustfs_ecstore::config::KVS;
use std::path::Path;
@@ -66,6 +68,55 @@ fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
}
}
fn parse_amqp_bool_value(field: &str, config: &KVS, default: bool) -> Result<bool, TargetError> {
match config.lookup(field) {
Some(value) => parse_target_bool(Some(value.as_str()))
.ok_or_else(|| TargetError::Configuration(format!("Invalid AMQP {field} boolean value: {value}"))),
None => Ok(default),
}
}
pub fn build_amqp_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<AMQPArgs, TargetError> {
let url = config
.lookup(AMQP_URL)
.ok_or_else(|| TargetError::Configuration("Missing AMQP url".to_string()))?;
let url = parse_url(url.trim(), "AMQP URL")?;
let exchange = config
.lookup(AMQP_EXCHANGE)
.ok_or_else(|| TargetError::Configuration("Missing AMQP exchange".to_string()))?;
let routing_key = config
.lookup(AMQP_ROUTING_KEY)
.ok_or_else(|| TargetError::Configuration("Missing AMQP routing_key".to_string()))?;
let args = AMQPArgs {
enable: true,
url,
exchange,
routing_key,
mandatory: parse_amqp_bool_value(AMQP_MANDATORY, config, false)?,
persistent: parse_amqp_bool_value(AMQP_PERSISTENT, config, true)?,
username: config.lookup(AMQP_USERNAME).unwrap_or_default(),
password: config.lookup(AMQP_PASSWORD).unwrap_or_default(),
tls_ca: config.lookup(AMQP_TLS_CA).unwrap_or_default(),
tls_client_cert: config.lookup(AMQP_TLS_CLIENT_CERT).unwrap_or_default(),
tls_client_key: config.lookup(AMQP_TLS_CLIENT_KEY).unwrap_or_default(),
queue_dir: config.lookup(AMQP_QUEUE_DIR).unwrap_or_else(|| default_queue_dir.to_string()),
queue_limit: config
.lookup(AMQP_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
target_type,
};
args.validate()?;
Ok(args)
}
pub fn validate_amqp_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
let _ = build_amqp_args(config, default_queue_dir, TargetType::NotifyEvent)?;
Ok(())
}
pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<WebhookArgs, TargetError> {
let endpoint = config
.lookup(WEBHOOK_ENDPOINT)
@@ -540,19 +591,28 @@ pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<()
#[cfg(test)]
mod tests {
use super::{
build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_kafka_config, validate_mysql_config,
validate_postgres_config, validate_redis_config,
build_amqp_args, build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_amqp_config,
validate_kafka_config, validate_mysql_config, validate_postgres_config, validate_redis_config,
};
use crate::target::{TargetType, postgres::PostgresFormat};
use rustfs_config::{
KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL,
REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE,
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_TLS_CLIENT_CERT,
AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING,
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY,
POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
};
use rustfs_ecstore::config::KVS;
fn amqp_base_config() -> KVS {
let mut config = KVS::new();
config.insert(AMQP_URL.to_string(), "amqp://127.0.0.1:5672/%2f".to_string());
config.insert(AMQP_EXCHANGE.to_string(), "rustfs.events".to_string());
config.insert(AMQP_ROUTING_KEY.to_string(), "objects".to_string());
config
}
fn kafka_base_config() -> KVS {
let mut config = KVS::new();
config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string());
@@ -570,6 +630,123 @@ mod tests {
config
}
#[test]
fn build_amqp_args_accepts_valid_config() {
let args = build_amqp_args(&amqp_base_config(), "", TargetType::NotifyEvent).expect("valid AMQP args");
assert_eq!(args.url.as_str(), "amqp://127.0.0.1:5672/%2f");
assert_eq!(args.exchange, "rustfs.events");
assert_eq!(args.routing_key, "objects");
assert!(!args.mandatory);
assert!(args.persistent);
}
#[test]
fn build_amqp_args_accepts_bool_aliases() {
let mut config = amqp_base_config();
config.insert(AMQP_MANDATORY.to_string(), "on".to_string());
config.insert(AMQP_PERSISTENT.to_string(), "no".to_string());
let args = build_amqp_args(&config, "", TargetType::NotifyEvent).expect("valid AMQP bool aliases");
assert!(args.mandatory);
assert!(!args.persistent);
}
#[test]
fn validate_amqp_config_rejects_invalid_bool() {
let mut config = amqp_base_config();
config.insert(AMQP_MANDATORY.to_string(), "sometimes".to_string());
let err = validate_amqp_config(&config, "").expect_err("invalid AMQP bool should fail");
assert!(err.to_string().contains("Invalid AMQP mandatory boolean"));
}
#[test]
fn validate_amqp_config_rejects_invalid_scheme() {
let mut config = amqp_base_config();
config.insert(AMQP_URL.to_string(), "http://127.0.0.1:5672".to_string());
let err = validate_amqp_config(&config, "").expect_err("invalid AMQP scheme should fail");
assert!(err.to_string().contains("only amqp and amqps"));
}
#[test]
fn validate_amqp_config_rejects_missing_url_host() {
let mut config = amqp_base_config();
config.insert(AMQP_URL.to_string(), "amqp:///objects".to_string());
let err = validate_amqp_config(&config, "").expect_err("missing AMQP host should fail");
assert!(err.to_string().contains("missing host"));
}
#[test]
fn validate_amqp_config_rejects_missing_exchange() {
let mut config = amqp_base_config();
config.0.retain(|kv| kv.key != AMQP_EXCHANGE);
let err = validate_amqp_config(&config, "").expect_err("missing AMQP exchange should fail");
assert!(err.to_string().contains("Missing AMQP exchange"));
}
#[test]
fn validate_amqp_config_rejects_missing_routing_key() {
let mut config = amqp_base_config();
config.0.retain(|kv| kv.key != AMQP_ROUTING_KEY);
let err = validate_amqp_config(&config, "").expect_err("missing AMQP routing_key should fail");
assert!(err.to_string().contains("Missing AMQP routing_key"));
}
#[test]
fn validate_amqp_config_rejects_relative_queue_dir() {
let mut config = amqp_base_config();
config.insert(AMQP_QUEUE_DIR.to_string(), "relative-queue".to_string());
let err = validate_amqp_config(&config, "").expect_err("relative queue_dir should fail");
assert!(err.to_string().contains("absolute path"));
}
#[test]
fn validate_amqp_config_rejects_unpaired_tls_client_cert_key() {
let mut config = amqp_base_config();
config.insert(AMQP_URL.to_string(), "amqps://127.0.0.1:5671/%2f".to_string());
config.insert(AMQP_TLS_CLIENT_CERT.to_string(), "/tmp/client.crt".to_string());
let err = validate_amqp_config(&config, "").expect_err("unpaired TLS cert should fail");
assert!(err.to_string().contains("tls_client_cert and tls_client_key"));
}
#[test]
fn validate_amqp_config_rejects_tls_paths_without_amqps() {
let mut config = amqp_base_config();
config.insert(AMQP_TLS_CLIENT_CERT.to_string(), "/tmp/client.crt".to_string());
config.insert(AMQP_TLS_CLIENT_KEY.to_string(), "/tmp/client.key".to_string());
let err = validate_amqp_config(&config, "").expect_err("TLS paths without amqps should fail");
assert!(err.to_string().contains("only allowed with amqps"));
}
#[test]
fn validate_amqp_config_rejects_ambiguous_credentials() {
let mut config = amqp_base_config();
config.insert(AMQP_URL.to_string(), "amqp://guest:guest@127.0.0.1:5672/%2f".to_string());
config.insert(AMQP_USERNAME.to_string(), "user".to_string());
config.insert(AMQP_PASSWORD.to_string(), "password".to_string());
let err = validate_amqp_config(&config, "").expect_err("ambiguous credentials should fail");
assert!(err.to_string().contains("either in url or username/password"));
}
#[test]
fn build_kafka_args_accepts_all_ack_alias() {
let mut config = kafka_base_config();
crates/targets/src/lib.rs
+4 -2
View File
@@ -16,15 +16,17 @@ pub mod arn;
mod check;
pub mod config;
pub mod error;
pub mod plugin;
pub mod store;
pub mod sys;
pub mod target;
pub use check::{
check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls,
check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
};
pub use error::{StoreError, TargetError};
pub use plugin::{BuiltinTargetDescriptor, TargetPluginDescriptor, TargetPluginRegistry, TargetRequestValidator, boxed_target};
pub use rustfs_s3_common::EventName;
use serde::{Deserialize, Serialize};
pub use sys::user_agent::*;
crates/targets/src/plugin.rs
+231
View File
@@ -0,0 +1,231 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Target, TargetError, config::collect_target_configs};
use hashbrown::HashMap;
use rustfs_ecstore::config::{Config, KVS};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{error, info};
type BoxedTarget<E> = Box<dyn Target<E> + Send + Sync>;
type TargetCreateFn<E> = Arc<dyn Fn(String, &KVS) -> Result<BoxedTarget<E>, TargetError> + Send + Sync>;
type TargetValidateFn = Arc<dyn Fn(&KVS) -> Result<(), TargetError> + Send + Sync>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TargetRequestValidator {
Webhook,
Mqtt,
Amqp(crate::target::TargetType),
Kafka(crate::target::TargetType),
MySql,
Nats(crate::target::TargetType),
Postgres(crate::target::TargetType),
Pulsar(crate::target::TargetType),
Redis {
default_channel: &'static str,
target_type: crate::target::TargetType,
},
}
#[derive(Clone)]
pub struct TargetPluginDescriptor<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
create_target: TargetCreateFn<E>,
target_type: &'static str,
valid_fields: &'static [&'static str],
valid_fields_set: Arc<HashSet<String>>,
validate_config: TargetValidateFn,
}
impl<E> TargetPluginDescriptor<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
pub fn new<Create, Validate>(
target_type: &'static str,
valid_fields: &'static [&'static str],
validate_config: Validate,
create_target: Create,
) -> Self
where
Create: Fn(String, &KVS) -> Result<BoxedTarget<E>, TargetError> + Send + Sync + 'static,
Validate: Fn(&KVS) -> Result<(), TargetError> + Send + Sync + 'static,
{
Self {
create_target: Arc::new(create_target),
target_type,
valid_fields,
valid_fields_set: Arc::new(valid_fields.iter().map(|field| (*field).to_string()).collect()),
validate_config: Arc::new(validate_config),
}
}
#[inline]
pub fn target_type(&self) -> &'static str {
self.target_type
}
#[inline]
pub fn valid_fields(&self) -> &'static [&'static str] {
self.valid_fields
}
#[inline]
pub fn valid_fields_set(&self) -> &HashSet<String> {
self.valid_fields_set.as_ref()
}
#[inline]
pub fn validate_config(&self, config: &KVS) -> Result<(), TargetError> {
(self.validate_config)(config)
}
#[inline]
pub fn create_target(&self, id: String, config: &KVS) -> Result<BoxedTarget<E>, TargetError> {
(self.create_target)(id, config)
}
}
#[derive(Clone)]
pub struct BuiltinTargetDescriptor<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
plugin: TargetPluginDescriptor<E>,
request_validator: TargetRequestValidator,
subsystem: &'static str,
}
impl<E> BuiltinTargetDescriptor<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
pub fn new(subsystem: &'static str, request_validator: TargetRequestValidator, plugin: TargetPluginDescriptor<E>) -> Self {
Self {
plugin,
request_validator,
subsystem,
}
}
#[inline]
pub fn plugin(&self) -> &TargetPluginDescriptor<E> {
&self.plugin
}
#[inline]
pub fn request_validator(&self) -> TargetRequestValidator {
self.request_validator
}
#[inline]
pub fn subsystem(&self) -> &'static str {
self.subsystem
}
}
pub struct TargetPluginRegistry<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
plugins: HashMap<String, TargetPluginDescriptor<E>>,
}
impl<E> Default for TargetPluginRegistry<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
fn default() -> Self {
Self::new()
}
}
impl<E> TargetPluginRegistry<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
pub fn new() -> Self {
Self { plugins: HashMap::new() }
}
pub fn register(&mut self, plugin: TargetPluginDescriptor<E>) -> Option<TargetPluginDescriptor<E>> {
self.plugins.insert(plugin.target_type().to_string(), plugin)
}
pub fn register_all<I>(&mut self, plugins: I)
where
I: IntoIterator<Item = TargetPluginDescriptor<E>>,
{
for plugin in plugins {
self.register(plugin);
}
}
pub fn supports_target_type(&self, target_type: &str) -> bool {
self.plugins.contains_key(target_type)
}
pub fn registered_target_types(&self) -> Vec<String> {
self.plugins.keys().cloned().collect()
}
pub fn create_target(&self, target_type: &str, id: String, config: &KVS) -> Result<BoxedTarget<E>, TargetError> {
let plugin = self
.plugins
.get(target_type)
.ok_or_else(|| TargetError::Configuration(format!("Unknown target type: {target_type}")))?;
plugin.validate_config(config)?;
plugin.create_target(id, config)
}
pub async fn create_targets_from_config(
&self,
config: &Config,
route_prefix: &str,
) -> Result<Vec<BoxedTarget<E>>, TargetError> {
let mut successful_targets = Vec::new();
for (target_type, plugin) in &self.plugins {
info!(target_type = %target_type, "Start working on target type");
for (id, merged_config) in collect_target_configs(config, route_prefix, target_type, plugin.valid_fields_set()) {
info!(target_type = %target_type, instance_id = %id, "Target is enabled, ready to create");
match self.create_target(target_type, id.clone(), &merged_config) {
Ok(target) => {
info!(target_type = %target.id().name, instance_id = %id, "Create target successfully");
successful_targets.push(target);
}
Err(err) => {
error!(target_type = %target_type, instance_id = %id, error = %err, "Failed to create target");
}
}
}
}
info!(count = successful_targets.len(), "All target processing completed");
Ok(successful_targets)
}
}
pub fn boxed_target<E, T>(target: T) -> BoxedTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
T: Target<E> + Send + Sync + 'static,
{
Box::new(target)
}
crates/targets/src/target/amqp.rs
+706
View File
@@ -0,0 +1,706 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! AMQP 0-9-1 event notification target.
//!
//! Publishes S3 events to RabbitMQ-compatible AMQP 0-9-1 brokers via `lapin`.
//! Queue-store mode uses the shared target store and replays the same raw JSON
//! body through `send_raw_from_store`.
use crate::{
StoreError, Target, TargetLog,
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
use lapin::{
BasicProperties, Channel, Confirmation, Connection, ConnectionProperties, ErrorKind as LapinErrorKind,
options::{BasicPublishOptions, ConfirmSelectOptions},
tcp::{OwnedIdentity, OwnedTLSConfig},
};
use parking_lot::Mutex;
use rustfs_config::{AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Mutex as AsyncMutex;
use tracing::{error, info, instrument, warn};
use url::Url;
#[derive(Clone)]
pub struct AMQPArgs {
pub enable: bool,
pub url: Url,
pub exchange: String,
pub routing_key: String,
pub mandatory: bool,
pub persistent: bool,
pub username: String,
pub password: String,
pub tls_ca: String,
pub tls_client_cert: String,
pub tls_client_key: String,
pub queue_dir: String,
pub queue_limit: u64,
pub target_type: TargetType,
}
impl fmt::Debug for AMQPArgs {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("AMQPArgs")
.field("enable", &self.enable)
.field("url", &redacted_amqp_url(&self.url))
.field("exchange", &self.exchange)
.field("routing_key", &self.routing_key)
.field("mandatory", &self.mandatory)
.field("persistent", &self.persistent)
.field("username", &self.username)
.field("password", if self.password.is_empty() { &"" } else { &"***REDACTED***" })
.field("tls_ca", &self.tls_ca)
.field("tls_client_cert", &self.tls_client_cert)
.field(
"tls_client_key",
if self.tls_client_key.is_empty() {
&""
} else {
&"***REDACTED***"
},
)
.field("queue_dir", &self.queue_dir)
.field("queue_limit", &self.queue_limit)
.field("target_type", &self.target_type)
.finish()
}
}
impl AMQPArgs {
pub fn validate(&self) -> Result<(), TargetError> {
if !self.enable {
return Ok(());
}
validate_amqp_url(&self.url)?;
if self.exchange.trim().is_empty() {
return Err(TargetError::Configuration("AMQP exchange cannot be empty".to_string()));
}
if self.routing_key.trim().is_empty() {
return Err(TargetError::Configuration("AMQP routing_key cannot be empty".to_string()));
}
let url_has_credentials = !self.url.username().is_empty() || self.url.password().is_some();
let config_has_credentials = !self.username.is_empty() || !self.password.is_empty();
if self.username.is_empty() != self.password.is_empty() {
return Err(TargetError::Configuration(
"AMQP username and password must be specified together".to_string(),
));
}
if url_has_credentials && config_has_credentials {
return Err(TargetError::Configuration(
"AMQP credentials must be specified either in url or username/password, not both".to_string(),
));
}
validate_amqp_tls_paths(self)?;
if !self.queue_dir.is_empty() && !Path::new(&self.queue_dir).is_absolute() {
return Err(TargetError::Configuration("AMQP queue directory must be an absolute path".to_string()));
}
Ok(())
}
}
fn redacted_amqp_url(url: &Url) -> String {
if url.password().is_none() {
return url.to_string();
}
let mut redacted = url.clone();
let _ = redacted.set_password(Some("***REDACTED***"));
redacted.to_string()
}
pub fn validate_amqp_url(url: &Url) -> Result<(), TargetError> {
match url.scheme() {
"amqp" | "amqps" => {
if url.host_str().is_none() {
return Err(TargetError::Configuration("AMQP URL is missing host".to_string()));
}
Ok(())
}
scheme => Err(TargetError::Configuration(format!(
"Unsupported AMQP URL scheme: {scheme} (only amqp and amqps are allowed)"
))),
}
}
fn validate_amqp_tls_paths(args: &AMQPArgs) -> Result<(), TargetError> {
let has_tls_settings = !args.tls_ca.is_empty() || !args.tls_client_cert.is_empty() || !args.tls_client_key.is_empty();
if has_tls_settings && args.url.scheme() != "amqps" {
return Err(TargetError::Configuration(
"AMQP TLS settings are only allowed with amqps URLs".to_string(),
));
}
if args.tls_client_cert.is_empty() != args.tls_client_key.is_empty() {
return Err(TargetError::Configuration(
"AMQP tls_client_cert and tls_client_key must be specified together".to_string(),
));
}
if !args.tls_ca.is_empty() && !Path::new(&args.tls_ca).is_absolute() {
return Err(TargetError::Configuration(format!("{AMQP_TLS_CA} must be an absolute path")));
}
if !args.tls_client_cert.is_empty() && !Path::new(&args.tls_client_cert).is_absolute() {
return Err(TargetError::Configuration(format!("{AMQP_TLS_CLIENT_CERT} must be an absolute path")));
}
if !args.tls_client_key.is_empty() && !Path::new(&args.tls_client_key).is_absolute() {
return Err(TargetError::Configuration(format!("{AMQP_TLS_CLIENT_KEY} must be an absolute path")));
}
Ok(())
}
fn connection_url(args: &AMQPArgs) -> Result<String, TargetError> {
let mut url = args.url.clone();
if !args.username.is_empty() {
url.set_username(&args.username)
.map_err(|_| TargetError::Configuration("AMQP username cannot be set on URL".to_string()))?;
url.set_password(Some(&args.password))
.map_err(|_| TargetError::Configuration("AMQP password cannot be set on URL".to_string()))?;
}
Ok(url.to_string())
}
async fn build_tls_config(args: &AMQPArgs) -> Result<OwnedTLSConfig, TargetError> {
let cert_chain = if args.tls_ca.is_empty() {
None
} else {
Some(
tokio::fs::read_to_string(&args.tls_ca)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CA}: {e}")))?,
)
};
let identity = if args.tls_client_cert.is_empty() {
None
} else {
let pem = tokio::fs::read(&args.tls_client_cert)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_CERT}: {e}")))?;
let key = tokio::fs::read(&args.tls_client_key)
.await
.map_err(|e| TargetError::Configuration(format!("Failed to read {AMQP_TLS_CLIENT_KEY}: {e}")))?;
Some(OwnedIdentity::PKCS8 { pem, key })
};
Ok(OwnedTLSConfig { identity, cert_chain })
}
fn build_publish_properties(args: &AMQPArgs) -> BasicProperties {
let mut properties = BasicProperties::default().with_content_type("application/json".into());
if args.persistent {
properties = properties.with_delivery_mode(2);
}
properties
}
fn map_lapin_error(err: lapin::Error, context: &str) -> TargetError {
let message = format!("{context}: {err}");
match err.kind() {
LapinErrorKind::IOError(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut => TargetError::Timeout(message),
LapinErrorKind::IOError(_)
| LapinErrorKind::InvalidConnectionState(_)
| LapinErrorKind::InvalidChannelState(..)
| LapinErrorKind::MissingHeartbeatError
| LapinErrorKind::ProtocolError(_)
if err.can_be_recovered() =>
{
TargetError::NotConnected
}
_ => TargetError::Network(message),
}
}
pub async fn connect_amqp(args: &AMQPArgs) -> Result<AMQPConnection, TargetError> {
args.validate()?;
tokio::time::timeout(Duration::from_secs(5), async {
let url = connection_url(args)?;
// Reconnect explicitly so every new channel enables publisher confirms below.
let properties = ConnectionProperties::default();
let connection = if args.url.scheme() == "amqps" && (!args.tls_ca.is_empty() || !args.tls_client_cert.is_empty()) {
Connection::connect_with_config(
&url,
properties,
build_tls_config(args).await?,
lapin::runtime::default_runtime()
.map_err(|e| TargetError::Initialization(format!("Failed to create AMQP runtime: {e}")))?,
)
.await
} else {
Connection::connect(&url, properties).await
}
.map_err(|e| map_lapin_error(e, "Failed to connect to AMQP broker"))?;
let channel = connection
.create_channel()
.await
.map_err(|e| map_lapin_error(e, "Failed to create AMQP channel"))?;
channel
.confirm_select(ConfirmSelectOptions::default())
.await
.map_err(|e| map_lapin_error(e, "Failed to enable AMQP publisher confirms"))?;
Ok(AMQPConnection { connection, channel })
})
.await
.unwrap_or_else(|_| Err(TargetError::Timeout("AMQP connection timed out".to_string())))
}
pub struct AMQPConnection {
pub(crate) connection: Connection,
pub(crate) channel: Channel,
}
pub struct AMQPTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
id: TargetID,
args: AMQPArgs,
connection: Arc<Mutex<Option<Arc<AMQPConnection>>>>,
connect_lock: Arc<AsyncMutex<()>>,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: std::marker::PhantomData<E>,
}
impl<E> AMQPTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(AMQPTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
connection: Arc::clone(&self.connection),
connect_lock: Arc::clone(&self.connect_lock),
store: self.store.as_ref().map(|s| s.boxed_clone()),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: std::marker::PhantomData,
})
}
#[instrument(skip(args), fields(target_id_as_string = %id))]
pub fn new(id: String, args: AMQPArgs) -> Result<Self, TargetError> {
args.validate()?;
let target_id = TargetID::new(id, ChannelTargetType::Amqp.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let base_path = PathBuf::from(&args.queue_dir);
let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Amqp.as_str(), &target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<QueuedPayload>::new(specific_queue_path, args.queue_limit, extension);
if let Err(e) = store.open() {
error!(target_id = %target_id, error = %e, "Failed to open store for AMQP target");
return Err(TargetError::Storage(format!("{e}")));
}
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
Ok(Self {
id: target_id,
args,
connection: Arc::new(Mutex::new(None)),
connect_lock: Arc::new(AsyncMutex::new(())),
store: queue_store,
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: std::marker::PhantomData,
})
}
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
let log = TargetLog {
event_name: event.event_name,
key,
records: vec![event.clone()],
};
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
async fn get_or_connect(&self) -> Result<Arc<AMQPConnection>, TargetError> {
if let Some(connection) = self.connection.lock().clone()
&& connection.connection.status().connected()
&& connection.channel.status().connected()
{
return Ok(connection);
}
let _guard = self.connect_lock.lock().await;
if let Some(connection) = self.connection.lock().clone()
&& connection.connection.status().connected()
&& connection.channel.status().connected()
{
return Ok(connection);
}
let connection = Arc::new(connect_amqp(&self.args).await?);
let mut guard = self.connection.lock();
*guard = Some(Arc::clone(&connection));
Ok(connection)
}
fn clear_connection(&self) {
*self.connection.lock() = None;
}
async fn send_body(&self, body: &[u8]) -> Result<(), TargetError> {
let connection = self.get_or_connect().await?;
let publish = connection
.channel
.basic_publish(
self.args.exchange.clone().into(),
self.args.routing_key.clone().into(),
BasicPublishOptions {
mandatory: self.args.mandatory,
..BasicPublishOptions::default()
},
body,
build_publish_properties(&self.args),
)
.await;
let confirm = match publish {
Ok(confirm) => confirm.await,
Err(err) => {
self.clear_connection();
return Err(map_lapin_error(err, "Failed to publish AMQP message"));
}
};
match confirm {
Ok(Confirmation::Ack(None) | Confirmation::NotRequested) => {
self.delivery_counters.record_success();
Ok(())
}
Ok(Confirmation::Ack(Some(returned)) | Confirmation::Nack(Some(returned))) => {
Err(TargetError::Request(format!("AMQP broker returned message: {}", returned.reply_text)))
}
Ok(Confirmation::Nack(None)) => Err(TargetError::Request("AMQP broker negatively acknowledged message".to_string())),
Err(err) => {
self.clear_connection();
Err(map_lapin_error(err, "Failed to confirm AMQP publish"))
}
}
}
}
#[async_trait]
impl<E> Target<E> for AMQPTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
fn id(&self) -> TargetID {
self.id.clone()
}
async fn is_active(&self) -> Result<bool, TargetError> {
let connection = self.get_or_connect().await?;
Ok(connection.connection.status().connected() && connection.channel.status().connected())
}
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = match self.build_queued_payload(&event) {
Ok(queued) => queued,
Err(err) => {
self.delivery_counters.record_final_failure();
return Err(err);
}
};
if let Some(store) = &self.store {
let encoded = match queued.encode() {
Ok(encoded) => encoded,
Err(err) => {
self.delivery_counters.record_final_failure();
return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
}
};
if let Err(e) = store.put_raw(&encoded) {
self.delivery_counters.record_final_failure();
return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
}
Ok(())
} else {
if let Err(err) = self.send_body(&queued.body).await {
self.delivery_counters.record_final_failure();
return Err(err);
}
Ok(())
}
}
async fn send_raw_from_store(&self, _key: Key, body: Vec<u8>, _meta: QueuedPayloadMeta) -> Result<(), TargetError> {
self.send_body(&body).await
}
async fn close(&self) -> Result<(), TargetError> {
let connection = self.connection.lock().take();
if let Some(connection) = connection {
connection
.connection
.close(200, "OK".into())
.await
.map_err(|e| map_lapin_error(e, "Failed to close AMQP connection"))?;
}
info!(target_id = %self.id, "AMQP target closed");
Ok(())
}
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
self.store.as_deref()
}
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
self.clone_box()
}
async fn init(&self) -> Result<(), TargetError> {
if !self.is_enabled() {
return Ok(());
}
match self.get_or_connect().await {
Ok(_) => Ok(()),
Err(err)
if self.store.is_some()
&& matches!(err, TargetError::Network(_) | TargetError::Timeout(_) | TargetError::NotConnected) =>
{
warn!(target_id = %self.id, error = %err, "AMQP init failed; events will buffer in store");
Ok(())
}
Err(err) => Err(err),
}
}
fn is_enabled(&self) -> bool {
self.args.enable
}
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
self.delivery_counters
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
}
fn record_final_failure(&self) {
self.delivery_counters.record_final_failure();
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_s3_common::EventName;
use serde_json::json;
use std::sync::Arc;
use uuid::Uuid;
fn valid_args() -> AMQPArgs {
AMQPArgs {
enable: true,
url: Url::parse("amqp://127.0.0.1:5672/%2f").unwrap(),
exchange: "rustfs.events".to_string(),
routing_key: "objects".to_string(),
mandatory: false,
persistent: true,
username: String::new(),
password: String::new(),
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
queue_dir: String::new(),
queue_limit: 10,
target_type: TargetType::NotifyEvent,
}
}
fn unreachable_args() -> AMQPArgs {
AMQPArgs {
url: Url::parse("amqp://127.0.0.1:1/%2f").unwrap(),
..valid_args()
}
}
fn test_event() -> Arc<EntityTarget<serde_json::Value>> {
Arc::new(EntityTarget {
object_name: "object.txt".to_string(),
bucket_name: "bucket".to_string(),
event_name: EventName::ObjectCreatedPut,
data: json!({"ok": true}),
})
}
fn temp_store_dir(name: &str) -> PathBuf {
std::env::temp_dir().join(format!("rustfs-amqp-target-{name}-{}", Uuid::new_v4()))
}
fn assert_connect_failure(err: &TargetError) {
assert!(
matches!(err, TargetError::NotConnected | TargetError::Timeout(_)),
"unexpected error: {err}"
);
}
#[test]
fn new_rejects_invalid_args() {
let mut args = valid_args();
args.exchange.clear();
let err = match AMQPTarget::<serde_json::Value>::new("primary".to_string(), args) {
Ok(_) => panic!("invalid args should fail"),
Err(err) => err,
};
assert!(err.to_string().contains("exchange cannot be empty"));
}
#[test]
fn new_accepts_queue_mode() {
let mut args = valid_args();
args.queue_dir = temp_store_dir("queue-mode").to_string_lossy().to_string();
let target =
AMQPTarget::<serde_json::Value>::new("primary".to_string(), args.clone()).expect("queue mode should be supported");
assert!(target.store().is_some());
let _ = std::fs::remove_dir_all(args.queue_dir);
}
#[tokio::test]
async fn save_with_store_queues_event_without_broker() {
let mut args = unreachable_args();
args.queue_dir = temp_store_dir("save-store").to_string_lossy().to_string();
let target = AMQPTarget::<serde_json::Value>::new("primary".to_string(), args.clone()).expect("target should build");
target
.save(test_event())
.await
.expect("store-backed save should queue without broker");
assert_eq!(target.delivery_snapshot().queue_length, 1);
assert_eq!(target.delivery_snapshot().failed_messages, 0);
let _ = std::fs::remove_dir_all(args.queue_dir);
}
#[tokio::test]
async fn save_without_store_returns_connection_error() {
let target =
AMQPTarget::<serde_json::Value>::new("primary".to_string(), unreachable_args()).expect("target should build");
let err = target
.save(test_event())
.await
.expect_err("direct publish should fail without broker");
assert_connect_failure(&err);
assert_eq!(target.delivery_snapshot().failed_messages, 1);
}
#[tokio::test]
async fn init_with_store_allows_broker_to_recover_later() {
let mut args = unreachable_args();
args.queue_dir = temp_store_dir("init-store").to_string_lossy().to_string();
let target = AMQPTarget::<serde_json::Value>::new("primary".to_string(), args.clone()).expect("target should build");
target.init().await.expect("store-backed init should tolerate broker failure");
let _ = std::fs::remove_dir_all(args.queue_dir);
}
#[tokio::test]
async fn init_without_store_returns_connection_error() {
let target =
AMQPTarget::<serde_json::Value>::new("primary".to_string(), unreachable_args()).expect("target should build");
let err = target
.init()
.await
.expect_err("init should fail without broker when no store exists");
assert_connect_failure(&err);
}
#[tokio::test]
async fn send_raw_from_store_returns_connection_error() {
let target =
AMQPTarget::<serde_json::Value>::new("primary".to_string(), unreachable_args()).expect("target should build");
let key = Key {
name: "queued".to_string(),
extension: ".event".to_string(),
item_count: 1,
compress: false,
};
let meta = QueuedPayloadMeta::new(
EventName::ObjectCreatedPut,
"bucket".to_string(),
"object.txt".to_string(),
"application/json",
2,
);
let err = target
.send_raw_from_store(key, b"{}".to_vec(), meta)
.await
.expect_err("queue replay should fail without broker");
assert_connect_failure(&err);
}
#[test]
fn debug_masks_secret_values() {
let args = AMQPArgs {
url: Url::parse("amqp://guest:secret@127.0.0.1:5672/%2f").unwrap(),
password: "secret".to_string(),
tls_client_key: "/tmp/client.key".to_string(),
..valid_args()
};
let rendered = format!("{args:?}");
assert!(!rendered.contains("guest:secret"));
assert!(!rendered.contains("password: \"secret\""));
assert!(!rendered.contains("tls_client_key: \"/tmp/client.key\""));
assert!(rendered.contains("***REDACTED***"));
}
}
crates/targets/src/target/mod.rs
+14 -1
View File
@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::warn;
pub mod amqp;
pub mod kafka;
pub mod mqtt;
pub mod mysql;
@@ -268,9 +269,12 @@ impl QueuedPayload {
/// used in the notification system.
///
/// It includes:
/// - `Amqp`: Represents an AMQP 0-9-1 target for sending notifications to a broker.
/// - `Webhook`: Represents a webhook target for sending notifications via HTTP requests.
/// - `Kafka`: Represents a Kafka target for sending notifications to a Kafka topic.
/// - `Mqtt`: Represents an MQTT target for sending notifications via MQTT protocol.
/// - `Nats`: Represents a NATS target for sending notifications to a subject.
/// - `Pulsar`: Represents a Pulsar target for sending notifications to a topic.
///
/// Each variant has an associated string representation that can be used for serialization
/// or logging purposes.
@@ -289,6 +293,7 @@ impl QueuedPayload {
/// example output:
/// Target type: webhook
pub enum ChannelTargetType {
Amqp,
Webhook,
Kafka,
Mqtt,
@@ -302,6 +307,7 @@ pub enum ChannelTargetType {
impl ChannelTargetType {
pub fn as_str(&self) -> &'static str {
match self {
ChannelTargetType::Amqp => "amqp",
ChannelTargetType::Webhook => "webhook",
ChannelTargetType::Kafka => "kafka",
ChannelTargetType::Mqtt => "mqtt",
@@ -317,6 +323,7 @@ impl ChannelTargetType {
impl std::fmt::Display for ChannelTargetType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
ChannelTargetType::Amqp => write!(f, "amqp"),
ChannelTargetType::Webhook => write!(f, "webhook"),
ChannelTargetType::Kafka => write!(f, "kafka"),
ChannelTargetType::Mqtt => write!(f, "mqtt"),
@@ -330,7 +337,7 @@ impl std::fmt::Display for ChannelTargetType {
}
/// `TargetType` enum represents the type of target in the notification system.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TargetType {
AuditLog,
NotifyEvent,
@@ -437,6 +444,12 @@ pub(crate) fn delete_stored_payload(
mod tests {
use super::*;
#[test]
fn channel_target_type_amqp_uses_runtime_name() {
assert_eq!(ChannelTargetType::Amqp.as_str(), "amqp");
assert_eq!(ChannelTargetType::Amqp.to_string(), "amqp");
}
#[test]
fn queued_payload_round_trips_meta_and_body() {
let meta = QueuedPayloadMeta::new(
crates/targets/tests/amqp_integration.rs
+221
View File
@@ -0,0 +1,221 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Integration tests for the AMQP notification target.
//!
//! These tests are ignored because they require a running RabbitMQ-compatible
//! AMQP 0-9-1 broker. To run locally:
//!
//! ```bash
//! docker run -d --name rustfs-rabbitmq -p 5672:5672 rabbitmq:3
//! cargo test -p rustfs-targets --test amqp_integration -- --ignored
//! ```
//!
//! Override the broker URL with `RUSTFS_TEST_AMQP_URL`.
use lapin::{
BasicProperties, Connection, ConnectionProperties,
options::{BasicAckOptions, BasicGetOptions, QueueBindOptions, QueueDeclareOptions, QueueDeleteOptions},
types::FieldTable,
};
use rustfs_s3_common::EventName;
use rustfs_targets::Target;
use rustfs_targets::check_amqp_broker_available;
use rustfs_targets::target::EntityTarget;
use rustfs_targets::target::TargetType;
use rustfs_targets::target::amqp::{AMQPArgs, AMQPTarget};
use serde_json::Value;
use std::sync::Arc;
use uuid::Uuid;
fn broker_url() -> String {
std::env::var("RUSTFS_TEST_AMQP_URL").unwrap_or_else(|_| "amqp://guest:guest@127.0.0.1:5672/%2f".to_string())
}
fn test_args(routing_key: &str) -> AMQPArgs {
AMQPArgs {
enable: true,
url: broker_url().parse().expect("valid AMQP URL"),
exchange: "amq.topic".to_string(),
routing_key: routing_key.to_string(),
mandatory: true,
persistent: true,
username: String::new(),
password: String::new(),
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
queue_dir: String::new(),
queue_limit: 100_000,
target_type: TargetType::NotifyEvent,
}
}
fn entity_for(bucket: &str, object: &str) -> Arc<EntityTarget<serde_json::Value>> {
Arc::new(EntityTarget {
bucket_name: bucket.to_string(),
object_name: object.to_string(),
event_name: EventName::ObjectCreatedPut,
data: serde_json::json!({"bucket": bucket, "object": object}),
})
}
async fn bind_queue(queue: &str, routing_key: &str) -> lapin::Channel {
let conn = Connection::connect(&broker_url(), ConnectionProperties::default())
.await
.expect("connect to AMQP broker");
let channel = conn.create_channel().await.expect("create channel");
channel
.queue_declare(
queue.into(),
QueueDeclareOptions {
durable: false,
exclusive: true,
auto_delete: true,
..QueueDeclareOptions::default()
},
FieldTable::default(),
)
.await
.expect("declare queue");
channel
.queue_bind(
queue.into(),
"amq.topic".into(),
routing_key.into(),
QueueBindOptions::default(),
FieldTable::default(),
)
.await
.expect("bind queue");
channel
}
async fn read_one(channel: &lapin::Channel, queue: &str) -> (Value, BasicProperties) {
let msg = tokio::time::timeout(std::time::Duration::from_secs(5), async {
loop {
if let Some(msg) = channel
.basic_get(queue.into(), BasicGetOptions::default())
.await
.expect("basic_get")
{
msg.ack(BasicAckOptions::default()).await.expect("ack message");
break msg;
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
})
.await
.expect("message should arrive");
let properties = msg.properties.clone();
let payload = serde_json::from_slice(&msg.data).expect("message payload should be JSON");
(payload, properties)
}
#[tokio::test]
#[ignore = "requires running RabbitMQ-compatible AMQP broker"]
async fn test_check_amqp_broker_available() {
check_amqp_broker_available(&test_args("rustfs.check"))
.await
.expect("broker check should succeed");
}
#[tokio::test]
#[ignore = "requires running RabbitMQ-compatible AMQP broker"]
async fn test_direct_publish_delivers_json_payload() {
let routing_key = format!("rustfs.test.{}", Uuid::new_v4().simple());
let queue = format!("rustfs-test-{}", Uuid::new_v4().simple());
let channel = bind_queue(&queue, &routing_key).await;
let target = AMQPTarget::new("direct".to_string(), test_args(&routing_key)).expect("construct AMQP target");
target
.save(entity_for("bucket1", "object-A"))
.await
.expect("publish should succeed");
let (payload, properties) = read_one(&channel, &queue).await;
assert_eq!(payload["Key"], "bucket1/object-A");
assert_eq!(payload["Records"][0]["data"]["bucket"], "bucket1");
assert_eq!(properties.content_type().as_ref().map(|s| s.as_str()), Some("application/json"));
assert_eq!(*properties.delivery_mode(), Some(2));
channel
.queue_delete(queue.into(), QueueDeleteOptions::default())
.await
.expect("delete queue");
}
#[tokio::test]
#[ignore = "requires running RabbitMQ-compatible AMQP broker"]
async fn test_publish_reconnects_after_close() {
let routing_key = format!("rustfs.reconnect.{}", Uuid::new_v4().simple());
let queue = format!("rustfs-test-{}", Uuid::new_v4().simple());
let channel = bind_queue(&queue, &routing_key).await;
let target = AMQPTarget::new("reconnect".to_string(), test_args(&routing_key)).expect("construct AMQP target");
target
.save(entity_for("bucket1", "object-before-close"))
.await
.expect("initial publish should succeed");
let (payload, _) = read_one(&channel, &queue).await;
assert_eq!(payload["Key"], "bucket1/object-before-close");
target.close().await.expect("close cached AMQP connection");
target
.save(entity_for("bucket1", "object-after-close"))
.await
.expect("publish should reconnect after close");
let (payload, _) = read_one(&channel, &queue).await;
assert_eq!(payload["Key"], "bucket1/object-after-close");
channel
.queue_delete(queue.into(), QueueDeleteOptions::default())
.await
.expect("delete queue");
}
#[tokio::test]
#[ignore = "requires running RabbitMQ-compatible AMQP broker"]
async fn test_queue_replay_delivers_and_removes_stored_payload() {
let routing_key = format!("rustfs.replay.{}", Uuid::new_v4().simple());
let queue = format!("rustfs-test-{}", Uuid::new_v4().simple());
let channel = bind_queue(&queue, &routing_key).await;
let queue_dir = std::env::temp_dir().join(format!("rustfs-amqp-integration-{}", Uuid::new_v4()));
let mut args = test_args(&routing_key);
args.queue_dir = queue_dir.to_string_lossy().to_string();
let target = AMQPTarget::new("queued".to_string(), args.clone()).expect("construct AMQP target");
target
.save(entity_for("bucket1", "object-B"))
.await
.expect("store-backed save should queue");
assert_eq!(target.delivery_snapshot().queue_length, 1);
let key = target.store().expect("store configured").list()[0].clone();
target.send_from_store(key).await.expect("replay should publish and delete");
let (payload, properties) = read_one(&channel, &queue).await;
assert_eq!(payload["Key"], "bucket1/object-B");
assert_eq!(properties.content_type().as_ref().map(|s| s.as_str()), Some("application/json"));
assert_eq!(*properties.delivery_mode(), Some(2));
assert_eq!(target.delivery_snapshot().queue_length, 0);
channel
.queue_delete(queue.into(), QueueDeleteOptions::default())
.await
.expect("delete queue");
let _ = std::fs::remove_dir_all(args.queue_dir);
}
rustfs/src/admin/handlers/audit.rs
+75 -63
View File
@@ -15,7 +15,7 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
@@ -31,13 +31,9 @@ use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS,
AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_policy::policy::action::{Action, AdminAction};
@@ -45,6 +41,7 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, warn};
@@ -95,57 +92,15 @@ struct AuditEndpointsResponse {
audit_endpoints: Vec<AuditEndpoint>,
}
fn audit_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: AUDIT_WEBHOOK_SUB_SYS,
service: "webhook",
valid_keys: AUDIT_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: AUDIT_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: AUDIT_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_MQTT_SUB_SYS,
service: "mqtt",
valid_keys: AUDIT_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: AUDIT_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: AUDIT_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: AUDIT_NATS_SUB_SYS,
service: "nats",
valid_keys: AUDIT_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: AUDIT_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_PULSAR_SUB_SYS,
service: "pulsar",
valid_keys: AUDIT_PULSAR_KEYS,
validator: AdminTargetValidator::Pulsar(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_REDIS_SUB_SYS,
service: "redis",
valid_keys: AUDIT_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL),
},
]
static AUDIT_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_audit_target_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
});
fn audit_target_specs() -> &'static [AdminTargetSpec] {
&AUDIT_TARGET_SPECS
}
async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<()> {
@@ -172,7 +127,7 @@ fn has_any_audit_targets(config: &Config) -> bool {
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
&audit_target_specs(),
audit_target_specs(),
AUDIT_ROUTE_PREFIX,
config,
target_type,
@@ -193,7 +148,7 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
}
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
shared_merge_target_endpoints(&audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| AuditEndpoint {
account_id: endpoint.account_id,
@@ -208,7 +163,7 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &
let target_type = params
.get("target_type")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
if target_service_name(&audit_target_specs(), target_type).is_none() {
if target_service_name(audit_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
}
let target_name = params
@@ -314,7 +269,7 @@ impl Operation for AuditTargetConfig {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
let specs = audit_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
@@ -323,7 +278,7 @@ impl Operation for AuditTargetConfig {
"audit target",
)?;
let spec = target_spec(&specs, target_type)
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR))
.await
@@ -431,7 +386,9 @@ mod tests {
use super::*;
use matchit::Router;
use rustfs_config::ENV_PREFIX;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_ecstore::config::{KV, KVS};
use serial_test::serial;
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars, with_vars_unset};
@@ -469,6 +426,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_config_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -513,6 +471,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_KAFKA_SUB_SYS.to_string(),
@@ -549,6 +508,44 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_amqp_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_AMQP_SUB_SYS.to_string(),
HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_AUDIT_AMQP_ENABLE_MIXED-AMQP", Some("on")),
("RUSTFS_AUDIT_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
("RUSTFS_AUDIT_AMQP_ENABLE_ENV-AMQP", Some("on")),
("RUSTFS_AUDIT_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
],
|| {
let runtime = HashMap::from([
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp")
.expect("mixed amqp target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp")
.expect("env amqp target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
@@ -565,6 +562,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_operation_block_reason_requires_audit_module_enable() {
with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("false"), || {
let reason =
@@ -575,6 +573,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_operation_block_reason_allows_when_audit_module_enabled() {
with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("true"), || {
assert!(
@@ -585,6 +584,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
@@ -598,6 +598,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -622,6 +623,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
@@ -710,6 +712,14 @@ mod tests {
assert_eq!(target_type, AUDIT_KAFKA_SUB_SYS);
assert_eq!(target_name, "primary");
let supported_amqp_params = full_router
.at("/v3/audit/target/audit_amqp/primary")
.expect("route should match");
let (target_type, target_name) =
extract_target_params(&supported_amqp_params.params).expect("audit amqp target should be supported");
assert_eq!(target_type, AUDIT_AMQP_SUB_SYS);
assert_eq!(target_name, "primary");
let mut partial_router = Router::new();
partial_router
.insert("/v3/audit/target/{target_type}", ())
@@ -722,6 +732,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -746,6 +757,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
rustfs/src/admin/handlers/event.rs
+105 -63
View File
@@ -15,7 +15,7 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
@@ -32,19 +32,16 @@ use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX,
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, info, warn};
@@ -101,57 +98,15 @@ struct NotificationEndpointsResponse {
notification_endpoints: Vec<NotificationEndpoint>,
}
fn notification_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: NOTIFY_WEBHOOK_SUB_SYS,
service: "webhook",
valid_keys: NOTIFY_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: NOTIFY_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: NOTIFY_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_MQTT_SUB_SYS,
service: "mqtt",
valid_keys: NOTIFY_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: NOTIFY_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: NOTIFY_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: NOTIFY_NATS_SUB_SYS,
service: "nats",
valid_keys: NOTIFY_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: NOTIFY_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_REDIS_SUB_SYS,
service: "redis",
valid_keys: NOTIFY_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL),
},
AdminTargetSpec {
subsystem: NOTIFY_PULSAR_SUB_SYS,
service: "pulsar",
valid_keys: NOTIFY_PULSAR_KEYS,
validator: AdminTargetValidator::Pulsar(TargetDomain::Notify),
},
]
static NOTIFICATION_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_notification_target_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
});
fn notification_target_specs() -> &'static [AdminTargetSpec] {
&NOTIFICATION_TARGET_SPECS
}
// --- Helper Functions ---
@@ -172,7 +127,7 @@ fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>>
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
&notification_target_specs(),
notification_target_specs(),
NOTIFY_ROUTE_PREFIX,
config,
target_type,
@@ -193,7 +148,7 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
}
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
shared_merge_target_endpoints(&notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| NotificationEndpoint {
account_id: endpoint.account_id,
@@ -241,7 +196,7 @@ impl Operation for NotificationTarget {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?;
let specs = notification_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
notification_body
@@ -252,7 +207,7 @@ impl Operation for NotificationTarget {
target_type,
"target",
)?;
let spec = target_spec(&specs, target_type)
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR))
.await
@@ -397,7 +352,7 @@ fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str>
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
let target_type = extract_param(params, "target_type")?;
if target_service_name(&notification_target_specs(), target_type).is_none() {
if target_service_name(notification_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type));
}
let target_name = extract_param(params, "target_name")?;
@@ -409,8 +364,10 @@ mod tests {
use super::*;
use matchit::Router;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_ecstore::config::{KV, KVS};
use rustfs_targets::arn::TargetID;
use serial_test::serial;
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars};
@@ -483,6 +440,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_env_and_mixed_sources() {
let config = Config(HashMap::from([
(
@@ -530,6 +488,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
NOTIFY_KAFKA_SUB_SYS.to_string(),
@@ -566,6 +525,44 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_amqp_env_and_mixed_sources() {
let config = Config(HashMap::from([(
NOTIFY_AMQP_SUB_SYS.to_string(),
HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_AMQP_ENABLE_MIXED-AMQP", Some("on")),
("RUSTFS_NOTIFY_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
("RUSTFS_NOTIFY_AMQP_ENABLE_ENV-AMQP", Some("on")),
("RUSTFS_NOTIFY_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
],
|| {
let runtime = HashMap::from([
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp")
.expect("mixed amqp target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp")
.expect("env amqp target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
#[test]
#[serial]
fn target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
@@ -582,6 +579,7 @@ mod tests {
}
#[test]
#[serial]
fn notification_target_operation_block_reason_requires_notify_module_enable() {
with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"), || {
let reason = futures::executor::block_on(notification_target_operation_block_reason(
@@ -593,6 +591,7 @@ mod tests {
}
#[test]
#[serial]
fn notification_target_operation_block_reason_allows_when_notify_module_enabled() {
with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || {
assert!(
@@ -605,6 +604,7 @@ mod tests {
}
#[test]
#[serial]
fn target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
@@ -628,6 +628,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -652,6 +653,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
@@ -697,6 +699,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -734,6 +737,7 @@ mod tests {
}
#[test]
#[serial]
fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -811,6 +815,44 @@ mod tests {
assert_eq!(target_name, "streaming");
}
#[test]
fn extract_target_params_accepts_amqp_target_type() {
let mut router = Router::new();
router
.insert("/v3/target/{target_type}/{target_name}", ())
.expect("route should insert");
let params = router
.at("/v3/target/notify_amqp/rabbitmq")
.expect("route should match")
.params;
let (target_type, target_name) = extract_target_params(&params).expect("amqp target type should be accepted");
assert_eq!(target_type, NOTIFY_AMQP_SUB_SYS);
assert_eq!(target_name, "rabbitmq");
}
#[test]
fn collect_validated_key_values_accepts_amqp_keys() {
let specs = notification_target_specs();
let allowed_keys = allowed_target_keys(specs, NOTIFY_AMQP_SUB_SYS);
let kv_map = shared_collect_validated_key_values(
[
(rustfs_config::AMQP_URL, "amqp://127.0.0.1:5672/%2f"),
(rustfs_config::AMQP_EXCHANGE, "rustfs.events"),
(rustfs_config::AMQP_ROUTING_KEY, "objects"),
],
&allowed_keys,
NOTIFY_AMQP_SUB_SYS,
"target",
)
.expect("amqp keys should be accepted");
assert_eq!(kv_map.get(rustfs_config::AMQP_URL).map(String::as_str), Some("amqp://127.0.0.1:5672/%2f"));
assert!(allowed_keys.contains(rustfs_config::AMQP_MANDATORY));
assert!(allowed_keys.contains(rustfs_config::AMQP_PERSISTENT));
}
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
let start = src
.find(start_marker)
rustfs/src/admin/handlers/target_descriptor.rs
+240 -30
View File
@@ -15,16 +15,17 @@
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, HeaderValue, StatusCode};
use rustfs_config::{
ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
};
use rustfs_ecstore::config::Config;
use rustfs_targets::{
TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
check_mqtt_broker_available_with_tls, check_nats_server_available, check_postgres_server_available,
check_pulsar_broker_available, check_redis_server_available,
config::{
build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
build_amqp_args, build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
collect_env_target_instance_ids, validate_mysql_config, validate_redis_config,
},
target::{TargetType, mqtt::MQTTTlsConfig},
@@ -34,10 +35,13 @@ use serde::Serialize;
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::Path;
use std::sync::Arc;
use tokio::time::{Duration, sleep};
use url::Url;
pub(crate) type EndpointKey = (String, String);
type AdminRequestValidatorFn =
Arc<dyn Fn(&HashMap<String, String>, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
@@ -55,14 +59,14 @@ pub(crate) struct MergedTargetEndpoint {
pub source: TargetEndpointSource,
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Debug)]
pub(crate) enum TargetDomain {
Notify,
Audit,
}
impl TargetDomain {
fn runtime_target_type(self) -> TargetType {
pub(crate) fn runtime_target_type(self) -> TargetType {
match self {
TargetDomain::Notify => TargetType::NotifyEvent,
TargetDomain::Audit => TargetType::AuditLog,
@@ -70,24 +74,98 @@ impl TargetDomain {
}
}
#[derive(Clone, Copy)]
pub(crate) enum AdminTargetValidator {
Webhook,
Mqtt,
Kafka(TargetDomain),
MySql,
Nats(TargetDomain),
Postgres(TargetDomain),
Pulsar(TargetDomain),
Redis(TargetDomain, &'static str),
impl From<TargetType> for TargetDomain {
fn from(value: TargetType) -> Self {
match value {
TargetType::NotifyEvent => TargetDomain::Notify,
TargetType::AuditLog => TargetDomain::Audit,
}
}
}
#[derive(Clone, Copy)]
#[derive(Clone)]
pub(crate) struct AdminTargetSpec {
pub subsystem: &'static str,
pub service: &'static str,
pub valid_keys: &'static [&'static str],
pub validator: AdminTargetValidator,
validator: AdminRequestValidatorFn,
}
pub(crate) fn admin_target_spec_from_builtin<E>(descriptor: &BuiltinTargetDescriptor<E>) -> AdminTargetSpec
where
E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
{
AdminTargetSpec {
subsystem: descriptor.subsystem(),
service: descriptor.plugin().target_type(),
valid_keys: descriptor.plugin().valid_fields(),
validator: match descriptor.request_validator() {
TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
TargetRequestValidator::Amqp(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_amqp_request_entry)
} else {
Arc::new(validate_notify_amqp_request_entry)
}
}
TargetRequestValidator::Kafka(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_kafka_request_entry)
} else {
Arc::new(validate_notify_kafka_request_entry)
}
}
TargetRequestValidator::MySql => Arc::new(validate_mysql_request_entry),
TargetRequestValidator::Nats(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_nats_request_entry)
} else {
Arc::new(validate_notify_nats_request_entry)
}
}
TargetRequestValidator::Postgres(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_postgres_request_entry)
} else {
Arc::new(validate_notify_postgres_request_entry)
}
}
TargetRequestValidator::Pulsar(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_pulsar_request_entry)
} else {
Arc::new(validate_notify_pulsar_request_entry)
}
}
TargetRequestValidator::Redis {
default_channel,
target_type,
} => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
validate_audit_redis_request_entry(default_channel)
} else {
validate_notify_redis_request_entry(default_channel)
}
}
},
}
}
impl std::fmt::Debug for AdminTargetSpec {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AdminTargetSpec")
.field("subsystem", &self.subsystem)
.field("service", &self.service)
.field("valid_keys", &self.valid_keys)
.finish_non_exhaustive()
}
}
impl AdminTargetSpec {
pub(crate) async fn validate_request(&self, kv_map: &HashMap<String, String>, default_queue_dir: &str) -> S3Result<()> {
(self.validator)(kv_map, default_queue_dir).await
}
}
pub(crate) fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
@@ -352,18 +430,7 @@ pub(crate) async fn validate_target_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> S3Result<()> {
match spec.validator {
AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await,
AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await,
AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await,
AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Redis(domain, default_channel) => {
validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await
}
}
spec.validate_request(kv_map, default_queue_dir).await
}
fn config_enable_is_on(value: &str) -> bool {
@@ -420,6 +487,14 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
Ok(())
}
fn validate_webhook_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_webhook_request(&kv_map).await })
}
async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()> {
let endpoint = kv_map
.get(MQTT_BROKER)
@@ -464,6 +539,129 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
Ok(())
}
fn validate_mqtt_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_mqtt_request(&kv_map).await })
}
fn validate_notify_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_mysql_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir).await })
}
fn validate_notify_postgres_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_postgres_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await })
})
}
fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await })
})
}
async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir.as_str()).await?;
@@ -496,6 +694,18 @@ async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_
})
}
async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
check_amqp_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e),
})
}
async fn validate_pulsar_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
rustfs/src/server/audit.rs
+2 -7
View File
@@ -35,12 +35,7 @@ pub fn is_audit_module_enabled() -> bool {
}
fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool {
for subsystem in [
rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
rustfs_config::audit::AUDIT_NATS_SUB_SYS,
rustfs_config::audit::AUDIT_PULSAR_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
] {
for &subsystem in rustfs_config::audit::AUDIT_SUB_SYSTEMS {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
@@ -102,7 +97,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
if !has_targets {
info!(
target: "rustfs::main::start_audit_system",
"Audit subsystem (Webhook/MQTT/NATS/Pulsar) is not configured, and audit system initialization is skipped."
"Audit subsystem targets are not configured, and audit system initialization is skipped."
);
return Ok(());
}