gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-10 23:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(targets): unify endpoint source/merge logic and bump rustfs-kafka-async to v1.2.0 (#2654)

Browse Source 
Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
This commit is contained in:
houseme
2026-04-23 17:14:36 +08:00
committed by GitHub
parent bc37cc4001
commit 368ef0f16c
26 changed files with 1589 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/audit/src/factory.rs
+22 -3
View File
@@ -15,13 +15,13 @@
use crate::AuditEntry;
use async_trait::async_trait;
use rustfs_config::AUDIT_DEFAULT_DIR;
use rustfs_config::audit::{AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
@@ -119,3 +119,22 @@ impl TargetFactory for PulsarTargetFactory {
AUDIT_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct KafkaTargetFactory;
#[async_trait]
impl TargetFactory for KafkaTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_kafka_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_kafka_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
crates/audit/src/registry.rs
+4 -1
View File
@@ -14,7 +14,9 @@
use crate::{
AuditEntry, AuditError, AuditResult,
factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory},
factory::{
KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
},
};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
@@ -53,6 +55,7 @@ impl AuditRegistry {
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry
}
crates/config/src/audit/kafka.rs
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Kafka Environment Variables
pub const ENV_AUDIT_KAFKA_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_ENABLE";
pub const ENV_AUDIT_KAFKA_BROKERS: &str = "RUSTFS_AUDIT_KAFKA_BROKERS";
pub const ENV_AUDIT_KAFKA_TOPIC: &str = "RUSTFS_AUDIT_KAFKA_TOPIC";
pub const ENV_AUDIT_KAFKA_ACKS: &str = "RUSTFS_AUDIT_KAFKA_ACKS";
pub const ENV_AUDIT_KAFKA_TLS_ENABLE: &str = "RUSTFS_AUDIT_KAFKA_TLS_ENABLE";
pub const ENV_AUDIT_KAFKA_TLS_CA: &str = "RUSTFS_AUDIT_KAFKA_TLS_CA";
pub const ENV_AUDIT_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_CERT";
pub const ENV_AUDIT_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_KAFKA_TLS_CLIENT_KEY";
pub const ENV_AUDIT_KAFKA_QUEUE_DIR: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_DIR";
pub const ENV_AUDIT_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_KAFKA_QUEUE_LIMIT";
pub const ENV_AUDIT_KAFKA_KEYS: &[&str; 10] = &[
ENV_AUDIT_KAFKA_ENABLE,
ENV_AUDIT_KAFKA_BROKERS,
ENV_AUDIT_KAFKA_TOPIC,
ENV_AUDIT_KAFKA_ACKS,
ENV_AUDIT_KAFKA_TLS_ENABLE,
ENV_AUDIT_KAFKA_TLS_CA,
ENV_AUDIT_KAFKA_TLS_CLIENT_CERT,
ENV_AUDIT_KAFKA_TLS_CLIENT_KEY,
ENV_AUDIT_KAFKA_QUEUE_DIR,
ENV_AUDIT_KAFKA_QUEUE_LIMIT,
];
/// A list of all valid configuration keys for a Kafka audit target.
pub const AUDIT_KAFKA_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::KAFKA_BROKERS,
crate::KAFKA_TOPIC,
crate::KAFKA_ACKS,
crate::KAFKA_TLS_ENABLE,
crate::KAFKA_TLS_CA,
crate::KAFKA_TLS_CLIENT_CERT,
crate::KAFKA_TLS_CLIENT_KEY,
crate::KAFKA_QUEUE_DIR,
crate::KAFKA_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
crates/config/src/audit/mod.rs
+4
View File
@@ -16,11 +16,13 @@
//! This module defines the configuration for audit systems, including
//! webhook and MQTT audit-related settings.
mod kafka;
mod mqtt;
mod nats;
mod pulsar;
mod webhook;
pub use kafka::*;
pub use mqtt::*;
pub use nats::*;
pub use pulsar::*;
@@ -33,6 +35,7 @@ pub const AUDIT_PREFIX: &str = "audit";
pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DELIMITER);
pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
@@ -40,6 +43,7 @@ pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
pub const AUDIT_STORE_EXTENSION: &str = ".audit";
#[allow(dead_code)]
pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
AUDIT_KAFKA_SUB_SYS,
AUDIT_MQTT_SUB_SYS,
AUDIT_NATS_SUB_SYS,
AUDIT_PULSAR_SUB_SYS,
crates/config/src/constants/targets.rs
+9
View File
@@ -40,6 +40,15 @@ pub const MQTT_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const MQTT_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const MQTT_TLS_TRUST_LEAF_AS_CA: &str = "tls_trust_leaf_as_ca";
pub const MQTT_WS_PATH_ALLOWLIST: &str = "ws_path_allowlist";
pub const KAFKA_BROKERS: &str = "brokers";
pub const KAFKA_TOPIC: &str = "topic";
pub const KAFKA_ACKS: &str = "acks";
pub const KAFKA_QUEUE_DIR: &str = "queue_dir";
pub const KAFKA_QUEUE_LIMIT: &str = "queue_limit";
pub const KAFKA_TLS_ENABLE: &str = "tls_enable";
pub const KAFKA_TLS_CA: &str = "tls_ca";
pub const KAFKA_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const KAFKA_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const NATS_ADDRESS: &str = "address";
pub const NATS_SUBJECT: &str = "subject";
crates/config/src/notify/kafka.rs
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// A list of all valid configuration keys for a Kafka target.
pub const NOTIFY_KAFKA_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::KAFKA_BROKERS,
crate::KAFKA_TOPIC,
crate::KAFKA_ACKS,
crate::KAFKA_TLS_ENABLE,
crate::KAFKA_TLS_CA,
crate::KAFKA_TLS_CLIENT_CERT,
crate::KAFKA_TLS_CLIENT_KEY,
crate::KAFKA_QUEUE_DIR,
crate::KAFKA_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
// Kafka Environment Variables
pub const ENV_NOTIFY_KAFKA_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_ENABLE";
pub const ENV_NOTIFY_KAFKA_BROKERS: &str = "RUSTFS_NOTIFY_KAFKA_BROKERS";
pub const ENV_NOTIFY_KAFKA_TOPIC: &str = "RUSTFS_NOTIFY_KAFKA_TOPIC";
pub const ENV_NOTIFY_KAFKA_ACKS: &str = "RUSTFS_NOTIFY_KAFKA_ACKS";
pub const ENV_NOTIFY_KAFKA_TLS_ENABLE: &str = "RUSTFS_NOTIFY_KAFKA_TLS_ENABLE";
pub const ENV_NOTIFY_KAFKA_TLS_CA: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CA";
pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_CERT";
pub const ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_KAFKA_TLS_CLIENT_KEY";
pub const ENV_NOTIFY_KAFKA_QUEUE_DIR: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_DIR";
pub const ENV_NOTIFY_KAFKA_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_KAFKA_QUEUE_LIMIT";
pub const ENV_NOTIFY_KAFKA_KEYS: &[&str; 10] = &[
ENV_NOTIFY_KAFKA_ENABLE,
ENV_NOTIFY_KAFKA_BROKERS,
ENV_NOTIFY_KAFKA_TOPIC,
ENV_NOTIFY_KAFKA_ACKS,
ENV_NOTIFY_KAFKA_TLS_ENABLE,
ENV_NOTIFY_KAFKA_TLS_CA,
ENV_NOTIFY_KAFKA_TLS_CLIENT_CERT,
ENV_NOTIFY_KAFKA_TLS_CLIENT_KEY,
ENV_NOTIFY_KAFKA_QUEUE_DIR,
ENV_NOTIFY_KAFKA_QUEUE_LIMIT,
];
crates/config/src/notify/mod.rs
+3 -1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
mod arn;
mod kafka;
mod mqtt;
mod nats;
mod pulsar;
@@ -20,6 +21,7 @@ mod store;
mod webhook;
pub use arn::*;
pub use kafka::*;
pub use mqtt::*;
pub use nats::*;
pub use pulsar::*;
@@ -69,13 +71,13 @@ pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
#[allow(dead_code)]
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
NOTIFY_KAFKA_SUB_SYS,
NOTIFY_MQTT_SUB_SYS,
NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_SUB_SYS,
NOTIFY_WEBHOOK_SUB_SYS,
];
#[allow(dead_code)]
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
#[allow(dead_code)]
crates/ecstore/src/config/audit.rs
+70 -9
View File
@@ -14,15 +14,16 @@
use crate::config::{KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -337,3 +338,63 @@ pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
},
])
});
pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_BROKERS.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TOPIC.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_ACKS.to_owned(),
value: "1".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TLS_ENABLE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
crates/ecstore/src/config/com.rs
+150 -31
View File
@@ -19,12 +19,12 @@ use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
@@ -58,7 +58,7 @@ struct TargetConfigDescriptor {
valid_keys: &'static [&'static str],
}
fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -66,6 +66,12 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
default_kvs: &notify::DEFAULT_NOTIFY_WEBHOOK_KVS,
valid_keys: NOTIFY_WEBHOOK_KEYS,
},
TargetConfigDescriptor {
external_key: "kafka",
subsystem_key: NOTIFY_KAFKA_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_KAFKA_KVS,
valid_keys: NOTIFY_KAFKA_KEYS,
},
TargetConfigDescriptor {
external_key: "mqtt",
subsystem_key: NOTIFY_MQTT_SUB_SYS,
@@ -87,7 +93,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 4] {
]
}
fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -95,6 +101,12 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 4] {
default_kvs: &audit::DEFAULT_AUDIT_WEBHOOK_KVS,
valid_keys: AUDIT_WEBHOOK_KEYS,
},
TargetConfigDescriptor {
external_key: "kafka",
subsystem_key: AUDIT_KAFKA_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_KAFKA_KVS,
valid_keys: AUDIT_KAFKA_KEYS,
},
TargetConfigDescriptor {
external_key: "mqtt",
subsystem_key: AUDIT_MQTT_SUB_SYS,
@@ -335,7 +347,7 @@ fn apply_external_oidc_map(cfg: &mut Config, root: &Map<String, Value>) -> bool
applied
}
fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
fn parse_target_scalar_value(key: &str, value: &Value) -> Option<String> {
match value {
Value::String(v) => Some(v.trim().to_string()),
Value::Bool(v) if key == ENABLE_KEY || key == rustfs_config::WEBHOOK_SKIP_TLS_VERIFY => Some(if *v {
@@ -350,7 +362,7 @@ fn parse_notify_scalar_value(key: &str, value: &Value) -> Option<String> {
}
}
fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
fn decode_target_instance_object(instance: &Map<String, Value>, valid_keys: &[&str]) -> KVS {
let mut kvs = KVS::new();
for (key, value) in instance {
@@ -358,7 +370,7 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
continue;
}
if let Some(parsed) = parse_notify_scalar_value(key, value) {
if let Some(parsed) = parse_target_scalar_value(key, value) {
kvs.insert(key.clone(), parsed);
}
}
@@ -366,21 +378,21 @@ fn decode_notify_instance_object(instance: &Map<String, Value>, valid_keys: &[&s
kvs
}
fn decode_notify_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
fn decode_target_instance_value(value: &Value, valid_keys: &[&str]) -> Option<KVS> {
match value {
Value::Object(instance) => Some(decode_notify_instance_object(instance, valid_keys)),
Value::Object(instance) => Some(decode_target_instance_object(instance, valid_keys)),
Value::Array(_) => serde_json::from_value::<KVS>(value.clone()).ok(),
_ => None,
}
}
fn is_notify_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
fn is_target_instance_shorthand(section: &Map<String, Value>, valid_keys: &[&str]) -> bool {
section
.iter()
.any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_notify_scalar_value(key, value).is_some())
.any(|(key, value)| valid_keys.contains(&key.as_str()) && parse_target_scalar_value(key, value).is_some())
}
fn apply_external_notify_section(
fn apply_external_target_section(
cfg: &mut Config,
notify_obj: &Map<String, Value>,
external_key: &str,
@@ -399,8 +411,8 @@ fn apply_external_notify_section(
let subsystem = cfg.0.entry(subsystem_key.to_string()).or_default();
let mut applied = false;
if is_notify_instance_shorthand(section_obj, valid_keys) {
let kvs = decode_notify_instance_object(section_obj, valid_keys);
if is_target_instance_shorthand(section_obj, valid_keys) {
let kvs = decode_target_instance_object(section_obj, valid_keys);
if !kvs.is_empty() {
let mut merged = default_kvs.clone();
merged.extend(kvs);
@@ -411,7 +423,7 @@ fn apply_external_notify_section(
}
for (raw_instance, value) in section_obj {
let Some(mut kvs) = decode_notify_instance_value(value, valid_keys) else {
let Some(mut kvs) = decode_target_instance_value(value, valid_keys) else {
continue;
};
if kvs.is_empty() {
@@ -444,7 +456,7 @@ fn apply_external_target_descriptors(
) -> bool {
let mut applied = false;
for descriptor in descriptors {
applied |= apply_external_notify_section(
applied |= apply_external_target_section(
cfg,
section_obj,
descriptor.external_key,
@@ -663,11 +675,12 @@ fn build_semantic_oidc_object(cfg: &Config) -> Map<String, Value> {
oidc_obj
}
fn is_notify_bool_key(key: &str) -> bool {
fn is_target_bool_key(key: &str) -> bool {
matches!(
key,
ENABLE_KEY
| rustfs_config::WEBHOOK_SKIP_TLS_VERIFY
| rustfs_config::KAFKA_TLS_ENABLE
| rustfs_config::MQTT_TLS_TRUST_LEAF_AS_CA
| rustfs_config::NATS_TLS_REQUIRED
| rustfs_config::PULSAR_TLS_ALLOW_INSECURE
@@ -675,8 +688,8 @@ fn is_notify_bool_key(key: &str) -> bool {
)
}
fn encode_notify_scalar_value(key: &str, value: &str) -> Value {
if is_notify_bool_key(key) {
fn encode_target_scalar_value(key: &str, value: &str) -> Value {
if is_target_bool_key(key) {
if let Ok(state) = value.parse::<EnableState>() {
return Value::Bool(state.is_enabled());
}
@@ -697,7 +710,7 @@ fn is_hidden_if_empty(default_kvs: &KVS, key: &str) -> bool {
.unwrap_or(false)
}
fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
fn build_target_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&str], default_kvs: &KVS) -> Map<String, Value> {
let mut instance = Map::new();
for key in valid_keys {
@@ -720,13 +733,13 @@ fn build_notify_instance_diff_object(kvs: &KVS, baseline: &KVS, valid_keys: &[&s
continue;
}
instance.insert((*key).to_string(), encode_notify_scalar_value(key, &effective_value));
instance.insert((*key).to_string(), encode_target_scalar_value(key, &effective_value));
}
instance
}
fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
fn merged_target_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS) -> KVS {
let mut merged = default_kvs.clone();
if let Some(kvs) = subsystem.get(DEFAULT_DELIMITER) {
merged.extend(kvs.clone());
@@ -734,7 +747,7 @@ fn merged_notify_default_kvs(subsystem: &HashMap<String, KVS>, default_kvs: &KVS
merged
}
fn build_notify_subsystem_object(
fn build_target_subsystem_object(
cfg: &Config,
subsystem_key: &str,
default_kvs: &KVS,
@@ -744,11 +757,11 @@ fn build_notify_subsystem_object(
return Map::new();
};
let effective_default = merged_notify_default_kvs(subsystem, default_kvs);
let effective_default = merged_target_default_kvs(subsystem, default_kvs);
let mut subsystem_obj = Map::new();
if let Some(default_instance) = subsystem.get(DEFAULT_DELIMITER) {
let default_obj = build_notify_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
let default_obj = build_target_instance_diff_object(default_instance, default_kvs, valid_keys, default_kvs);
if !default_obj.is_empty() {
subsystem_obj.insert("default".to_string(), Value::Object(default_obj));
}
@@ -761,7 +774,7 @@ fn build_notify_subsystem_object(
instances.sort_by_key(|(lhs, _)| *lhs);
for (instance_key, kvs) in instances {
let instance_obj = build_notify_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
let instance_obj = build_target_instance_diff_object(kvs, &effective_default, valid_keys, default_kvs);
if !instance_obj.is_empty() {
subsystem_obj.insert(instance_key.clone(), Value::Object(instance_obj));
}
@@ -774,7 +787,7 @@ fn build_target_object(cfg: &Config, descriptors: &[TargetConfigDescriptor]) ->
let mut target_obj = Map::new();
for descriptor in descriptors {
let subsystem_obj =
build_notify_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
build_target_subsystem_object(cfg, descriptor.subsystem_key, descriptor.default_kvs, descriptor.valid_keys);
if !subsystem_obj.is_empty() {
target_obj.insert(descriptor.external_key.to_string(), Value::Object(subsystem_obj));
}
@@ -1118,8 +1131,8 @@ mod tests {
ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, WalkOptions,
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
use rustfs_filemeta::FileInfo;
@@ -1753,6 +1766,15 @@ mod tests {
"topic":"events",
"queue_dir":""
}
},
"kafka":{
"streaming":{
"enable":true,
"brokers":"127.0.0.1:9092,127.0.0.1:9093",
"topic":"events-kafka",
"acks":"all",
"tls_enable":true
}
}
}
}"#;
@@ -1781,6 +1803,14 @@ mod tests {
.expect("mqtt target should be decoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR), "");
let kafka = cfg
.get_value(NOTIFY_KAFKA_SUB_SYS, "streaming")
.expect("kafka target should be decoded");
assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092,127.0.0.1:9093");
assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
}
#[test]
@@ -1850,6 +1880,14 @@ mod tests {
"broker":"tcp://127.0.0.1:1883",
"topic":"audit-events"
}
},
"kafka":{
"auditlog":{
"enable":true,
"brokers":"127.0.0.1:9092",
"topic":"audit-events-kafka",
"acks":"1"
}
}
}
}"#;
@@ -1873,6 +1911,12 @@ mod tests {
.get_value(AUDIT_MQTT_SUB_SYS, "analytics")
.expect("audit mqtt target should be decoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER), "tcp://127.0.0.1:1883");
let kafka = cfg
.get_value(AUDIT_KAFKA_SUB_SYS, "auditlog")
.expect("audit kafka target should be decoded");
assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS), "127.0.0.1:9092");
assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "audit-events-kafka");
}
#[test]
@@ -1993,6 +2037,38 @@ mod tests {
);
cfg.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_section);
let mut kafka_default = notify::DEFAULT_NOTIFY_KAFKA_KVS.clone();
kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "events-kafka".to_string());
let mut kafka_section = std::collections::HashMap::new();
kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
kafka_section.insert(
"streaming".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::KAFKA_BROKERS.to_string(),
value: "127.0.0.1:9092,127.0.0.1:9093".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::KAFKA_ACKS.to_string(),
value: "all".to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::KAFKA_TLS_ENABLE.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(NOTIFY_KAFKA_SUB_SYS.to_string(), kafka_section);
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
let notify = v
@@ -2028,6 +2104,19 @@ mod tests {
.expect("mqtt target should be encoded");
assert_eq!(mqtt.get(rustfs_config::MQTT_BROKER).and_then(Value::as_str), Some("tcp://127.0.0.1:1883"));
assert_eq!(mqtt.get(rustfs_config::MQTT_QUEUE_DIR).and_then(Value::as_str), Some(""));
let kafka = notify
.get("kafka")
.and_then(Value::as_object)
.and_then(|targets| targets.get("streaming"))
.and_then(Value::as_object)
.expect("kafka target should be encoded");
assert_eq!(
kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str),
Some("127.0.0.1:9092,127.0.0.1:9093")
);
assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS).and_then(Value::as_str), Some("all"));
assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE).and_then(Value::as_bool), Some(true));
}
#[test]
@@ -2079,6 +2168,28 @@ mod tests {
);
cfg.0.insert(AUDIT_MQTT_SUB_SYS.to_string(), mqtt_section);
let mut kafka_default = audit::DEFAULT_AUDIT_KAFKA_KVS.clone();
kafka_default.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
kafka_default.insert(rustfs_config::KAFKA_TOPIC.to_string(), "audit-events-kafka".to_string());
let mut kafka_section = std::collections::HashMap::new();
kafka_section.insert(DEFAULT_DELIMITER.to_string(), kafka_default);
kafka_section.insert(
"auditlog".to_string(),
crate::config::KVS(vec![
crate::config::KV {
key: ENABLE_KEY.to_string(),
value: EnableState::On.to_string(),
hidden_if_empty: false,
},
crate::config::KV {
key: rustfs_config::KAFKA_BROKERS.to_string(),
value: "127.0.0.1:9092".to_string(),
hidden_if_empty: false,
},
]),
);
cfg.0.insert(AUDIT_KAFKA_SUB_SYS.to_string(), kafka_section);
let out = encode_server_config_blob(&cfg, None).expect("encode should succeed");
let v: Value = serde_json::from_slice(&out).expect("output should be json");
let logger = v
@@ -2105,6 +2216,14 @@ mod tests {
.expect("audit mqtt default should be encoded");
assert_eq!(mqtt_default.get(ENABLE_KEY).and_then(Value::as_bool), Some(true));
assert_eq!(mqtt_default.get(rustfs_config::MQTT_TOPIC).and_then(Value::as_str), Some("audit-events"));
let kafka = logger
.get("kafka")
.and_then(Value::as_object)
.and_then(|targets| targets.get("auditlog"))
.and_then(Value::as_object)
.expect("audit kafka target should be encoded");
assert_eq!(kafka.get(rustfs_config::KAFKA_BROKERS).and_then(Value::as_str), Some("127.0.0.1:9092"));
}
#[test]
crates/ecstore/src/config/mod.rs
+8 -2
View File
@@ -25,8 +25,12 @@ use crate::store::ECStore;
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use rustfs_config::COMMENT_KEY;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::audit::{
AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -245,6 +249,8 @@ pub fn init() {
kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
// Register all default configurations
crates/ecstore/src/config/notify.rs
+69 -8
View File
@@ -14,14 +14,15 @@
use crate::config::{KV, KVS};
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -314,3 +315,63 @@ pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
},
])
});
pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_BROKERS.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TOPIC.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_ACKS.to_owned(),
value: "1".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TLS_ENABLE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
crates/notify/src/factory.rs
+22 -3
View File
@@ -15,13 +15,13 @@
use crate::Event;
use async_trait::async_trait;
use rustfs_config::EVENT_DEFAULT_DIR;
use rustfs_config::notify::{NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
use rustfs_config::notify::{NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
@@ -119,3 +119,22 @@ impl TargetFactory for PulsarTargetFactory {
NOTIFY_PULSAR_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct KafkaTargetFactory;
#[async_trait]
impl TargetFactory for KafkaTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_kafka_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::kafka::KafkaTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_kafka_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
crates/notify/src/integration.rs
+10 -2
View File
@@ -24,8 +24,8 @@ use crate::{
};
use hashbrown::HashMap;
use rustfs_config::notify::{
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS,
NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_ecstore::config::{Config, KVS};
use rustfs_s3_common::EventName;
@@ -45,6 +45,7 @@ const MAX_RECENT_LIVE_EVENTS: usize = 1024;
fn subsystem_target_type(target_type: &str) -> &str {
match target_type {
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_KAFKA_SUB_SYS => "kafka",
NOTIFY_MQTT_SUB_SYS => "mqtt",
NOTIFY_NATS_SUB_SYS => "nats",
NOTIFY_PULSAR_SUB_SYS => "pulsar",
@@ -765,6 +766,13 @@ mod tests {
assert_eq!(target_id.name, "mqtt");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_kafka_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_KAFKA_SUB_SYS, "EventBus");
assert_eq!(target_id.id, "eventbus");
assert_eq!(target_id.name, "kafka");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_nats_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_NATS_SUB_SYS, "Bus");
crates/notify/src/registry.rs
+4 -1
View File
@@ -13,7 +13,9 @@
// limitations under the License.
use crate::Event;
use crate::factory::{MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory};
use crate::factory::{
KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
};
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashMap;
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
@@ -45,6 +47,7 @@ impl TargetRegistry {
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry
}
crates/targets/Cargo.toml
+1
View File
@@ -33,6 +33,7 @@ url = { workspace = true }
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
sysinfo = { workspace = true, features = ["multithread"] }
rustfs-kafka-async = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
crates/targets/src/check.rs
+47
View File
@@ -128,3 +128,50 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA
Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())),
}
}
pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> {
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, RequiredAcks, SecurityConfig};
use std::time::Duration;
let map_kafka_error = |err: KafkaError, context: &str| match err {
KafkaError::Connection(ConnectionError::NoHostReachable) => crate::TargetError::NotConnected,
KafkaError::Connection(ConnectionError::Timeout(_)) => crate::TargetError::Timeout(format!("{context}: {err}")),
KafkaError::Connection(_) => crate::TargetError::Network(format!("{context}: {err}")),
KafkaError::Config(_) => crate::TargetError::Configuration(format!("{context}: {err}")),
_ => crate::TargetError::Request(format!("{context}: {err}")),
};
let acks = match args.acks {
0 => RequiredAcks::None,
1 => RequiredAcks::One,
_ => RequiredAcks::All,
};
let mut config = AsyncProducerConfig::new()
.with_ack_timeout(Duration::from_secs(5))
.with_required_acks(acks);
if args.tls_enable {
let mut security = SecurityConfig::new();
if !args.tls_ca.is_empty() {
security = security.with_ca_cert(args.tls_ca.clone());
}
if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
security = security.with_client_cert(args.tls_client_cert.clone(), args.tls_client_key.clone());
}
config = config.with_security(security);
}
match tokio::time::timeout(Duration::from_secs(5), async {
let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config)
.await
.map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?;
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())),
}
}
crates/targets/src/config/mod.rs
+2 -2
View File
@@ -21,6 +21,6 @@ pub use loader::{
collect_target_configs_from_env,
};
pub use target_args::{
build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
};
crates/targets/src/config/target_args.rs
+139 -8
View File
@@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v
use crate::error::TargetError;
use crate::target::{
TargetType,
kafka::KafkaArgs,
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
nats::{NATSArgs, validate_nats_address},
pulsar::{PulsarArgs, validate_pulsar_broker},
@@ -23,19 +24,38 @@ use crate::target::{
};
use rumqttc::QoS;
use rustfs_config::{
DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA,
MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME,
RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use rustfs_ecstore::config::KVS;
use std::path::Path;
use std::time::Duration;
fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
let Some(value) = value else {
return Ok(1);
};
let normalized = value.trim();
if normalized.is_empty() {
return Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string()));
}
match normalized.to_ascii_lowercase().as_str() {
"0" => Ok(0),
"1" => Ok(1),
"-1" | "all" => Ok(-1),
_ => Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string())),
}
}
pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<WebhookArgs, TargetError> {
let endpoint = config
.lookup(WEBHOOK_ENDPOINT)
@@ -265,3 +285,114 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
.ok_or_else(|| TargetError::Configuration("Missing Pulsar broker".to_string()))?;
validate_pulsar_broker_config(&broker, config, default_queue_dir)
}
pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
let brokers_raw = config
.lookup(KAFKA_BROKERS)
.ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
if brokers_raw.split(',').all(|s| s.trim().is_empty()) {
return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
}
let brokers: Vec<String> = brokers_raw
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect();
let topic = config
.lookup(KAFKA_TOPIC)
.ok_or_else(|| TargetError::Configuration("Missing Kafka topic".to_string()))?;
Ok(KafkaArgs {
enable: true,
brokers,
topic,
acks: parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?,
tls_enable: parse_target_bool(config.lookup(KAFKA_TLS_ENABLE).as_deref()).unwrap_or(false),
tls_ca: config.lookup(KAFKA_TLS_CA).unwrap_or_default(),
tls_client_cert: config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default(),
tls_client_key: config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default(),
queue_dir: config
.lookup(KAFKA_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string()),
queue_limit: config
.lookup(KAFKA_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
target_type,
})
}
pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
let brokers_raw = config
.lookup(KAFKA_BROKERS)
.ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
if brokers_raw.split(',').map(|s| s.trim()).all(|s| s.is_empty()) {
return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
}
if config.lookup(KAFKA_TOPIC).is_none() {
return Err(TargetError::Configuration("Missing Kafka topic".to_string()));
}
parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?;
let tls_client_cert = config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default();
let tls_client_key = config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default();
if tls_client_cert.is_empty() != tls_client_key.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_client_cert and tls_client_key must be specified together".to_string(),
));
}
let queue_dir = config
.lookup(KAFKA_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string());
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{build_kafka_args, validate_kafka_config};
use crate::target::TargetType;
use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
use rustfs_ecstore::config::KVS;
fn kafka_base_config() -> KVS {
let mut config = KVS::new();
config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string());
config.insert(KAFKA_TOPIC.to_string(), "events".to_string());
config
}
#[test]
fn build_kafka_args_accepts_all_ack_alias() {
let mut config = kafka_base_config();
config.insert(KAFKA_ACKS.to_string(), "all".to_string());
let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka args");
assert_eq!(args.acks, -1);
}
#[test]
fn build_kafka_args_rejects_invalid_acks() {
let mut config = kafka_base_config();
config.insert(KAFKA_ACKS.to_string(), "leader".to_string());
let err = build_kafka_args(&config, "", TargetType::NotifyEvent).expect_err("invalid acks should fail");
assert!(err.to_string().contains("Kafka acks must be one of"));
}
#[test]
fn validate_kafka_config_rejects_invalid_acks() {
let mut config = kafka_base_config();
config.insert(KAFKA_ACKS.to_string(), "2".to_string());
let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
assert!(err.to_string().contains("Kafka acks must be one of"));
}
}
crates/targets/src/lib.rs
+2 -1
View File
@@ -21,7 +21,8 @@ pub mod sys;
pub mod target;
pub use check::{
check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, check_pulsar_broker_available,
check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_pulsar_broker_available,
};
pub use error::{StoreError, TargetError};
pub use rustfs_s3_common::EventName;
crates/targets/src/target/kafka.rs
+437
View File
@@ -0,0 +1,437 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
StoreError, Target, TargetLog,
arn::TargetID,
error::TargetError,
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
},
};
use async_trait::async_trait;
use rustfs_config::audit::AUDIT_STORE_EXTENSION;
use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::{marker::PhantomData, path::PathBuf, sync::Arc, time::Duration};
use tokio::sync::Mutex;
use tracing::{debug, error, info, instrument, warn};
/// Arguments for configuring a Kafka target
#[derive(Debug, Clone)]
pub struct KafkaArgs {
/// Whether the target is enabled
pub enable: bool,
/// Comma-separated list of broker addresses (e.g. "localhost:9092,broker2:9092")
pub brokers: Vec<String>,
/// The topic to publish events to
pub topic: String,
/// Required acks: 0 = none, 1 = leader, -1 = all
pub acks: i16,
/// Whether to enable TLS for Kafka transport
pub tls_enable: bool,
/// Optional path to CA cert used for broker verification
pub tls_ca: String,
/// Optional path to client certificate for mTLS
pub tls_client_cert: String,
/// Optional path to client private key for mTLS
pub tls_client_key: String,
/// The directory to store events in case of failure
pub queue_dir: String,
/// The maximum number of events to store
pub queue_limit: u64,
/// The target type (audit or notify)
pub target_type: TargetType,
}
impl KafkaArgs {
/// Validates the KafkaArgs configuration
pub fn validate(&self) -> Result<(), TargetError> {
if !self.enable {
return Ok(());
}
if self.brokers.is_empty() {
return Err(TargetError::Configuration("kafka brokers cannot be empty".to_string()));
}
if self.topic.is_empty() {
return Err(TargetError::Configuration("kafka topic cannot be empty".to_string()));
}
if !matches!(self.acks, -1..=1) {
return Err(TargetError::Configuration("kafka acks must be one of: 0, 1, -1".to_string()));
}
if self.tls_client_cert.is_empty() != self.tls_client_key.is_empty() {
return Err(TargetError::Configuration(
"kafka tls_client_cert and tls_client_key must be specified together".to_string(),
));
}
if !self.queue_dir.is_empty() {
let path = std::path::Path::new(&self.queue_dir);
if !path.is_absolute() {
return Err(TargetError::Configuration("kafka queueDir path should be absolute".to_string()));
}
}
Ok(())
}
}
/// A target that sends events to an Apache Kafka topic
pub struct KafkaTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
id: TargetID,
args: KafkaArgs,
store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
delivery_counters: Arc<TargetDeliveryCounters>,
_phantom: PhantomData<E>,
}
impl<E> KafkaTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
fn map_kafka_error(err: KafkaError, context: &str) -> TargetError {
match err {
KafkaError::Connection(ConnectionError::NoHostReachable) => TargetError::NotConnected,
KafkaError::Connection(ConnectionError::Timeout(_)) => TargetError::Timeout(format!("{context}: {err}")),
KafkaError::Connection(_) => TargetError::Network(format!("{context}: {err}")),
KafkaError::Config(_) => TargetError::Configuration(format!("{context}: {err}")),
_ => TargetError::Request(format!("{context}: {err}")),
}
}
fn is_connection_error(err: &TargetError) -> bool {
matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
}
/// Creates a new KafkaTarget
#[instrument(skip(args), fields(target_id = %id))]
pub fn new(id: String, args: KafkaArgs) -> Result<Self, TargetError> {
args.validate()?;
let target_id = TargetID::new(id, ChannelTargetType::Kafka.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let queue_dir =
PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => AUDIT_STORE_EXTENSION,
TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
};
let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
if let Err(e) = store.open() {
error!("Failed to open store for Kafka target {}: {}", target_id.id, e);
return Err(TargetError::Storage(format!("{e}")));
}
Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
info!(target_id = %target_id.id, "Kafka target created");
Ok(KafkaTarget {
id: target_id,
args,
store: queue_store,
producer: Arc::new(Mutex::new(None)),
delivery_counters: Arc::new(TargetDeliveryCounters::default()),
_phantom: PhantomData,
})
}
/// Builds a Kafka producer from the current args
async fn build_producer(&self) -> Result<AsyncProducer, TargetError> {
let acks = match self.args.acks {
0 => RequiredAcks::None,
1 => RequiredAcks::One,
_ => RequiredAcks::All,
};
let mut config = AsyncProducerConfig::new()
.with_ack_timeout(Duration::from_secs(30))
.with_required_acks(acks);
if self.args.tls_enable {
let mut security = SecurityConfig::new();
if !self.args.tls_ca.is_empty() {
security = security.with_ca_cert(self.args.tls_ca.clone());
}
if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
}
config = config.with_security(security);
}
AsyncProducer::from_hosts_with_config(self.args.brokers.clone(), config)
.await
.map_err(|e| Self::map_kafka_error(e, "Failed to create Kafka producer"))
}
async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
let mut cached = self.producer.lock().await;
if let Some(producer) = cached.as_ref() {
return Ok(Arc::clone(producer));
}
let producer = Arc::new(self.build_producer().await?);
*cached = Some(Arc::clone(&producer));
Ok(producer)
}
async fn invalidate_cached_producer(&self) {
let mut cached = self.producer.lock().await;
*cached = None;
}
/// Serializes the event and builds a QueuedPayload
fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
let object_name = crate::target::decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
let log = TargetLog {
event_name: event.event_name,
key,
records: vec![event.data.clone()],
};
let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
/// Sends the raw body to Kafka
#[instrument(skip(self, body, meta), fields(target_id = %self.id))]
async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
debug!(
target = %self.id,
bucket = %meta.bucket_name,
object = %meta.object_name,
event = %meta.event_name,
payload_len = body.len(),
"Sending Kafka payload"
);
let producer = self.get_or_build_producer().await?;
if let Err(err) = producer.send(&Record::from_value(&self.args.topic, body.as_slice())).await {
let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
if Self::is_connection_error(&mapped) {
self.invalidate_cached_producer().await;
}
return Err(mapped);
}
debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
self.delivery_counters.record_success();
Ok(())
}
/// Clones this target into a boxed trait object
pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
Box::new(KafkaTarget::<E> {
id: self.id.clone(),
args: self.args.clone(),
store: self.store.as_ref().map(|s| s.boxed_clone()),
producer: Arc::clone(&self.producer),
delivery_counters: Arc::clone(&self.delivery_counters),
_phantom: PhantomData,
})
}
}
#[async_trait]
impl<E> Target<E> for KafkaTarget<E>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
fn id(&self) -> TargetID {
self.id.clone()
}
async fn is_active(&self) -> Result<bool, TargetError> {
let _ = self.get_or_build_producer().await?;
Ok(true)
}
async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
let queued = match self.build_queued_payload(&event) {
Ok(queued) => queued,
Err(err) => {
self.delivery_counters.record_final_failure();
return Err(err);
}
};
if let Some(store) = &self.store {
let encoded = match queued.encode() {
Ok(encoded) => encoded,
Err(err) => {
self.delivery_counters.record_final_failure();
return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
}
};
if let Err(e) = store.put_raw(&encoded) {
self.delivery_counters.record_final_failure();
return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
}
debug!("Event saved to store for Kafka target: {}", self.id);
Ok(())
} else {
if let Err(err) = self.send_body(queued.body, &queued.meta).await {
self.delivery_counters.record_final_failure();
return Err(err);
}
Ok(())
}
}
async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
debug!("Sending queued payload from store for Kafka target: {}, key: {}", self.id, key);
if let Err(e) = self.send_body(body, &meta).await {
if matches!(e, TargetError::NotConnected) {
warn!(target_id = %self.id, "Kafka not reachable, event remains in store.");
return Err(TargetError::NotConnected);
}
error!(target_id = %self.id, error = %e, "Failed to send event from store.");
return Err(e);
}
debug!("Event sent from store for Kafka target: {}", self.id);
Ok(())
}
async fn close(&self) -> Result<(), TargetError> {
info!("Kafka target closed: {}", self.id);
Ok(())
}
fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
self.store.as_deref()
}
fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
self.clone_box()
}
fn is_enabled(&self) -> bool {
self.args.enable
}
fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
self.delivery_counters
.snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
}
fn record_final_failure(&self) {
self.delivery_counters.record_final_failure();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn base_args() -> KafkaArgs {
KafkaArgs {
enable: true,
brokers: vec!["localhost:9092".to_string()],
topic: "rustfs-events".to_string(),
acks: 1,
tls_enable: false,
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
queue_dir: String::new(),
queue_limit: 0,
target_type: TargetType::NotifyEvent,
}
}
#[test]
fn test_validate_empty_brokers() {
let args = KafkaArgs {
brokers: vec![],
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn test_validate_empty_topic() {
let args = KafkaArgs {
topic: String::new(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn test_validate_relative_queue_dir() {
let args = KafkaArgs {
queue_dir: "relative/path".to_string(),
..base_args()
};
assert!(args.validate().is_err());
}
#[test]
fn test_validate_valid_args() {
assert!(base_args().validate().is_ok());
}
#[test]
fn test_validate_disabled_target_skips_validation() {
let args = KafkaArgs {
enable: false,
brokers: vec![],
topic: String::new(),
..base_args()
};
assert!(args.validate().is_ok());
}
#[test]
fn test_validate_tls_client_cert_and_key_must_be_paired() {
let args = KafkaArgs {
tls_client_cert: "/tmp/client.crt".to_string(),
tls_client_key: String::new(),
..base_args()
};
assert!(args.validate().is_err());
}
}
crates/targets/src/target/mod.rs
+1
View File
@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::warn;
pub mod kafka;
pub mod mqtt;
pub mod nats;
pub mod pulsar;