refactor(targets): unify endpoint source/merge logic and bump rustfs-kafka-async to v1.2.0 (#2654)

Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>

crates/targets/src/check.rs

@@ -128,3 +128,50 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA
         Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())),
     }
 }
+
+pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> {
+    use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
+    use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, RequiredAcks, SecurityConfig};
+    use std::time::Duration;
+
+    let map_kafka_error = |err: KafkaError, context: &str| match err {
+        KafkaError::Connection(ConnectionError::NoHostReachable) => crate::TargetError::NotConnected,
+        KafkaError::Connection(ConnectionError::Timeout(_)) => crate::TargetError::Timeout(format!("{context}: {err}")),
+        KafkaError::Connection(_) => crate::TargetError::Network(format!("{context}: {err}")),
+        KafkaError::Config(_) => crate::TargetError::Configuration(format!("{context}: {err}")),
+        _ => crate::TargetError::Request(format!("{context}: {err}")),
+    };
+
+    let acks = match args.acks {
+        0 => RequiredAcks::None,
+        1 => RequiredAcks::One,
+        _ => RequiredAcks::All,
+    };
+
+    let mut config = AsyncProducerConfig::new()
+        .with_ack_timeout(Duration::from_secs(5))
+        .with_required_acks(acks);
+
+    if args.tls_enable {
+        let mut security = SecurityConfig::new();
+        if !args.tls_ca.is_empty() {
+            security = security.with_ca_cert(args.tls_ca.clone());
+        }
+        if !args.tls_client_cert.is_empty() && !args.tls_client_key.is_empty() {
+            security = security.with_client_cert(args.tls_client_cert.clone(), args.tls_client_key.clone());
+        }
+        config = config.with_security(security);
+    }
+
+    match tokio::time::timeout(Duration::from_secs(5), async {
+        let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config)
+            .await
+            .map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?;
+        Ok(())
+    })
+    .await
+    {
+        Ok(result) => result,
+        Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())),
+    }
+}

crates/targets/src/config/mod.rs

@@ -21,6 +21,6 @@ pub use loader::{
     collect_target_configs_from_env,
 };
 pub use target_args::{
-    build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-    validate_pulsar_config, validate_webhook_config,
+    build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+    validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
 };

crates/targets/src/config/target_args.rs

@@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v
 use crate::error::TargetError;
 use crate::target::{
     TargetType,
+    kafka::KafkaArgs,
     mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
     nats::{NATSArgs, validate_nats_address},
     pulsar::{PulsarArgs, validate_pulsar_broker},
@@ -23,19 +24,38 @@ use crate::target::{
 };
 use rumqttc::QoS;
 use rustfs_config::{
-    DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
-    MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA,
-    MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
-    PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME,
-    RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
+    KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
+    MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
+    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
+    PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use std::path::Path;
 use std::time::Duration;
 
+fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
+    let Some(value) = value else {
+        return Ok(1);
+    };
+
+    let normalized = value.trim();
+    if normalized.is_empty() {
+        return Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string()));
+    }
+
+    match normalized.to_ascii_lowercase().as_str() {
+        "0" => Ok(0),
+        "1" => Ok(1),
+        "-1" | "all" => Ok(-1),
+        _ => Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string())),
+    }
+}
+
 pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<WebhookArgs, TargetError> {
     let endpoint = config
         .lookup(WEBHOOK_ENDPOINT)
@@ -265,3 +285,114 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
         .ok_or_else(|| TargetError::Configuration("Missing Pulsar broker".to_string()))?;
     validate_pulsar_broker_config(&broker, config, default_queue_dir)
 }
+
+pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').all(|s| s.trim().is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+    let brokers: Vec<String> = brokers_raw
+        .split(',')
+        .map(|s| s.trim().to_string())
+        .filter(|s| !s.is_empty())
+        .collect();
+
+    let topic = config
+        .lookup(KAFKA_TOPIC)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka topic".to_string()))?;
+
+    Ok(KafkaArgs {
+        enable: true,
+        brokers,
+        topic,
+        acks: parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?,
+        tls_enable: parse_target_bool(config.lookup(KAFKA_TLS_ENABLE).as_deref()).unwrap_or(false),
+        tls_ca: config.lookup(KAFKA_TLS_CA).unwrap_or_default(),
+        tls_client_cert: config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default(),
+        tls_client_key: config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default(),
+        queue_dir: config
+            .lookup(KAFKA_QUEUE_DIR)
+            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_limit: config
+            .lookup(KAFKA_QUEUE_LIMIT)
+            .and_then(|v| v.parse::<u64>().ok())
+            .unwrap_or(DEFAULT_LIMIT),
+        target_type,
+    })
+}
+
+pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').map(|s| s.trim()).all(|s| s.is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+
+    if config.lookup(KAFKA_TOPIC).is_none() {
+        return Err(TargetError::Configuration("Missing Kafka topic".to_string()));
+    }
+
+    parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?;
+
+    let tls_client_cert = config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default();
+    let tls_client_key = config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default();
+    if tls_client_cert.is_empty() != tls_client_key.is_empty() {
+        return Err(TargetError::Configuration(
+            "Kafka tls_client_cert and tls_client_key must be specified together".to_string(),
+        ));
+    }
+
+    let queue_dir = config
+        .lookup(KAFKA_QUEUE_DIR)
+        .unwrap_or_else(|| default_queue_dir.to_string());
+    if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
+        return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_kafka_args, validate_kafka_config};
+    use crate::target::TargetType;
+    use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
+    use rustfs_ecstore::config::KVS;
+
+    fn kafka_base_config() -> KVS {
+        let mut config = KVS::new();
+        config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string());
+        config.insert(KAFKA_TOPIC.to_string(), "events".to_string());
+        config
+    }
+
+    #[test]
+    fn build_kafka_args_accepts_all_ack_alias() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "all".to_string());
+
+        let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka args");
+        assert_eq!(args.acks, -1);
+    }
+
+    #[test]
+    fn build_kafka_args_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "leader".to_string());
+
+        let err = build_kafka_args(&config, "", TargetType::NotifyEvent).expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+
+    #[test]
+    fn validate_kafka_config_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "2".to_string());
+
+        let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+}

crates/targets/src/lib.rs

@@ -21,7 +21,8 @@ pub mod sys;
 pub mod target;
 
 pub use check::{
-    check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available, check_pulsar_broker_available,
+    check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
+    check_pulsar_broker_available,
 };
 pub use error::{StoreError, TargetError};
 pub use rustfs_s3_common::EventName;

crates/targets/src/target/kafka.rs

@@ -0,0 +1,437 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+    StoreError, Target, TargetLog,
+    arn::TargetID,
+    error::TargetError,
+    store::{Key, QueueStore, Store},
+    target::{
+        ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
+        TargetType,
+    },
+};
+use async_trait::async_trait;
+use rustfs_config::audit::AUDIT_STORE_EXTENSION;
+use rustfs_config::notify::NOTIFY_STORE_EXTENSION;
+use rustfs_kafka_async::error::{ConnectionError, Error as KafkaError};
+use rustfs_kafka_async::{AsyncProducer, AsyncProducerConfig, Record, RequiredAcks, SecurityConfig};
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use std::{marker::PhantomData, path::PathBuf, sync::Arc, time::Duration};
+use tokio::sync::Mutex;
+use tracing::{debug, error, info, instrument, warn};
+
+/// Arguments for configuring a Kafka target
+#[derive(Debug, Clone)]
+pub struct KafkaArgs {
+    /// Whether the target is enabled
+    pub enable: bool,
+    /// Comma-separated list of broker addresses (e.g. "localhost:9092,broker2:9092")
+    pub brokers: Vec<String>,
+    /// The topic to publish events to
+    pub topic: String,
+    /// Required acks: 0 = none, 1 = leader, -1 = all
+    pub acks: i16,
+    /// Whether to enable TLS for Kafka transport
+    pub tls_enable: bool,
+    /// Optional path to CA cert used for broker verification
+    pub tls_ca: String,
+    /// Optional path to client certificate for mTLS
+    pub tls_client_cert: String,
+    /// Optional path to client private key for mTLS
+    pub tls_client_key: String,
+    /// The directory to store events in case of failure
+    pub queue_dir: String,
+    /// The maximum number of events to store
+    pub queue_limit: u64,
+    /// The target type (audit or notify)
+    pub target_type: TargetType,
+}
+
+impl KafkaArgs {
+    /// Validates the KafkaArgs configuration
+    pub fn validate(&self) -> Result<(), TargetError> {
+        if !self.enable {
+            return Ok(());
+        }
+
+        if self.brokers.is_empty() {
+            return Err(TargetError::Configuration("kafka brokers cannot be empty".to_string()));
+        }
+
+        if self.topic.is_empty() {
+            return Err(TargetError::Configuration("kafka topic cannot be empty".to_string()));
+        }
+
+        if !matches!(self.acks, -1..=1) {
+            return Err(TargetError::Configuration("kafka acks must be one of: 0, 1, -1".to_string()));
+        }
+
+        if self.tls_client_cert.is_empty() != self.tls_client_key.is_empty() {
+            return Err(TargetError::Configuration(
+                "kafka tls_client_cert and tls_client_key must be specified together".to_string(),
+            ));
+        }
+
+        if !self.queue_dir.is_empty() {
+            let path = std::path::Path::new(&self.queue_dir);
+            if !path.is_absolute() {
+                return Err(TargetError::Configuration("kafka queueDir path should be absolute".to_string()));
+            }
+        }
+
+        Ok(())
+    }
+}
+
+/// A target that sends events to an Apache Kafka topic
+pub struct KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    id: TargetID,
+    args: KafkaArgs,
+    store: Option<Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>>,
+    producer: Arc<Mutex<Option<Arc<AsyncProducer>>>>,
+    delivery_counters: Arc<TargetDeliveryCounters>,
+    _phantom: PhantomData<E>,
+}
+
+impl<E> KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn map_kafka_error(err: KafkaError, context: &str) -> TargetError {
+        match err {
+            KafkaError::Connection(ConnectionError::NoHostReachable) => TargetError::NotConnected,
+            KafkaError::Connection(ConnectionError::Timeout(_)) => TargetError::Timeout(format!("{context}: {err}")),
+            KafkaError::Connection(_) => TargetError::Network(format!("{context}: {err}")),
+            KafkaError::Config(_) => TargetError::Configuration(format!("{context}: {err}")),
+            _ => TargetError::Request(format!("{context}: {err}")),
+        }
+    }
+
+    fn is_connection_error(err: &TargetError) -> bool {
+        matches!(err, TargetError::NotConnected | TargetError::Timeout(_) | TargetError::Network(_))
+    }
+
+    /// Creates a new KafkaTarget
+    #[instrument(skip(args), fields(target_id = %id))]
+    pub fn new(id: String, args: KafkaArgs) -> Result<Self, TargetError> {
+        args.validate()?;
+
+        let target_id = TargetID::new(id, ChannelTargetType::Kafka.as_str().to_string());
+
+        let queue_store = if !args.queue_dir.is_empty() {
+            let queue_dir =
+                PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id));
+
+            let extension = match args.target_type {
+                TargetType::AuditLog => AUDIT_STORE_EXTENSION,
+                TargetType::NotifyEvent => NOTIFY_STORE_EXTENSION,
+            };
+
+            let store = QueueStore::<QueuedPayload>::new(queue_dir, args.queue_limit, extension);
+            if let Err(e) = store.open() {
+                error!("Failed to open store for Kafka target {}: {}", target_id.id, e);
+                return Err(TargetError::Storage(format!("{e}")));
+            }
+
+            Some(Box::new(store) as Box<dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync>)
+        } else {
+            None
+        };
+
+        info!(target_id = %target_id.id, "Kafka target created");
+        Ok(KafkaTarget {
+            id: target_id,
+            args,
+            store: queue_store,
+            producer: Arc::new(Mutex::new(None)),
+            delivery_counters: Arc::new(TargetDeliveryCounters::default()),
+            _phantom: PhantomData,
+        })
+    }
+
+    /// Builds a Kafka producer from the current args
+    async fn build_producer(&self) -> Result<AsyncProducer, TargetError> {
+        let acks = match self.args.acks {
+            0 => RequiredAcks::None,
+            1 => RequiredAcks::One,
+            _ => RequiredAcks::All,
+        };
+
+        let mut config = AsyncProducerConfig::new()
+            .with_ack_timeout(Duration::from_secs(30))
+            .with_required_acks(acks);
+
+        if self.args.tls_enable {
+            let mut security = SecurityConfig::new();
+            if !self.args.tls_ca.is_empty() {
+                security = security.with_ca_cert(self.args.tls_ca.clone());
+            }
+            if !self.args.tls_client_cert.is_empty() && !self.args.tls_client_key.is_empty() {
+                security = security.with_client_cert(self.args.tls_client_cert.clone(), self.args.tls_client_key.clone());
+            }
+            config = config.with_security(security);
+        }
+
+        AsyncProducer::from_hosts_with_config(self.args.brokers.clone(), config)
+            .await
+            .map_err(|e| Self::map_kafka_error(e, "Failed to create Kafka producer"))
+    }
+
+    async fn get_or_build_producer(&self) -> Result<Arc<AsyncProducer>, TargetError> {
+        let mut cached = self.producer.lock().await;
+        if let Some(producer) = cached.as_ref() {
+            return Ok(Arc::clone(producer));
+        }
+
+        let producer = Arc::new(self.build_producer().await?);
+        *cached = Some(Arc::clone(&producer));
+        Ok(producer)
+    }
+
+    async fn invalidate_cached_producer(&self) {
+        let mut cached = self.producer.lock().await;
+        *cached = None;
+    }
+
+    /// Serializes the event and builds a QueuedPayload
+    fn build_queued_payload(&self, event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError> {
+        let object_name = crate::target::decode_object_name(&event.object_name)?;
+        let key = format!("{}/{}", event.bucket_name, object_name);
+
+        let log = TargetLog {
+            event_name: event.event_name,
+            key,
+            records: vec![event.data.clone()],
+        };
+
+        let body = serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {e}")))?;
+
+        let meta = QueuedPayloadMeta::new(
+            event.event_name,
+            event.bucket_name.clone(),
+            event.object_name.clone(),
+            "application/json",
+            body.len(),
+        );
+
+        Ok(QueuedPayload::new(meta, body))
+    }
+
+    /// Sends the raw body to Kafka
+    #[instrument(skip(self, body, meta), fields(target_id = %self.id))]
+    async fn send_body(&self, body: Vec<u8>, meta: &QueuedPayloadMeta) -> Result<(), TargetError> {
+        debug!(
+            target = %self.id,
+            bucket = %meta.bucket_name,
+            object = %meta.object_name,
+            event = %meta.event_name,
+            payload_len = body.len(),
+            "Sending Kafka payload"
+        );
+
+        let producer = self.get_or_build_producer().await?;
+
+        if let Err(err) = producer.send(&Record::from_value(&self.args.topic, body.as_slice())).await {
+            let mapped = Self::map_kafka_error(err, "Failed to send message to Kafka");
+            if Self::is_connection_error(&mapped) {
+                self.invalidate_cached_producer().await;
+            }
+            return Err(mapped);
+        }
+
+        debug!(target_id = %self.id, topic = %self.args.topic, "Event published to Kafka topic");
+        self.delivery_counters.record_success();
+        Ok(())
+    }
+
+    /// Clones this target into a boxed trait object
+    pub fn clone_box(&self) -> Box<dyn Target<E> + Send + Sync> {
+        Box::new(KafkaTarget::<E> {
+            id: self.id.clone(),
+            args: self.args.clone(),
+            store: self.store.as_ref().map(|s| s.boxed_clone()),
+            producer: Arc::clone(&self.producer),
+            delivery_counters: Arc::clone(&self.delivery_counters),
+            _phantom: PhantomData,
+        })
+    }
+}
+
+#[async_trait]
+impl<E> Target<E> for KafkaTarget<E>
+where
+    E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
+{
+    fn id(&self) -> TargetID {
+        self.id.clone()
+    }
+
+    async fn is_active(&self) -> Result<bool, TargetError> {
+        let _ = self.get_or_build_producer().await?;
+        Ok(true)
+    }
+
+    async fn save(&self, event: Arc<EntityTarget<E>>) -> Result<(), TargetError> {
+        let queued = match self.build_queued_payload(&event) {
+            Ok(queued) => queued,
+            Err(err) => {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+        };
+
+        if let Some(store) = &self.store {
+            let encoded = match queued.encode() {
+                Ok(encoded) => encoded,
+                Err(err) => {
+                    self.delivery_counters.record_final_failure();
+                    return Err(TargetError::Storage(format!("Failed to encode queued payload: {err}")));
+                }
+            };
+            if let Err(e) = store.put_raw(&encoded) {
+                self.delivery_counters.record_final_failure();
+                return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
+            }
+            debug!("Event saved to store for Kafka target: {}", self.id);
+            Ok(())
+        } else {
+            if let Err(err) = self.send_body(queued.body, &queued.meta).await {
+                self.delivery_counters.record_final_failure();
+                return Err(err);
+            }
+            Ok(())
+        }
+    }
+
+    async fn send_raw_from_store(&self, key: Key, body: Vec<u8>, meta: QueuedPayloadMeta) -> Result<(), TargetError> {
+        debug!("Sending queued payload from store for Kafka target: {}, key: {}", self.id, key);
+
+        if let Err(e) = self.send_body(body, &meta).await {
+            if matches!(e, TargetError::NotConnected) {
+                warn!(target_id = %self.id, "Kafka not reachable, event remains in store.");
+                return Err(TargetError::NotConnected);
+            }
+            error!(target_id = %self.id, error = %e, "Failed to send event from store.");
+            return Err(e);
+        }
+
+        debug!("Event sent from store for Kafka target: {}", self.id);
+        Ok(())
+    }
+
+    async fn close(&self) -> Result<(), TargetError> {
+        info!("Kafka target closed: {}", self.id);
+        Ok(())
+    }
+
+    fn store(&self) -> Option<&(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync)> {
+        self.store.as_deref()
+    }
+
+    fn clone_dyn(&self) -> Box<dyn Target<E> + Send + Sync> {
+        self.clone_box()
+    }
+
+    fn is_enabled(&self) -> bool {
+        self.args.enable
+    }
+
+    fn delivery_snapshot(&self) -> TargetDeliverySnapshot {
+        self.delivery_counters
+            .snapshot(self.store.as_deref().map_or(0, |store| store.len() as u64))
+    }
+
+    fn record_final_failure(&self) {
+        self.delivery_counters.record_final_failure();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn base_args() -> KafkaArgs {
+        KafkaArgs {
+            enable: true,
+            brokers: vec!["localhost:9092".to_string()],
+            topic: "rustfs-events".to_string(),
+            acks: 1,
+            tls_enable: false,
+            tls_ca: String::new(),
+            tls_client_cert: String::new(),
+            tls_client_key: String::new(),
+            queue_dir: String::new(),
+            queue_limit: 0,
+            target_type: TargetType::NotifyEvent,
+        }
+    }
+
+    #[test]
+    fn test_validate_empty_brokers() {
+        let args = KafkaArgs {
+            brokers: vec![],
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_empty_topic() {
+        let args = KafkaArgs {
+            topic: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_relative_queue_dir() {
+        let args = KafkaArgs {
+            queue_dir: "relative/path".to_string(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+
+    #[test]
+    fn test_validate_valid_args() {
+        assert!(base_args().validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_disabled_target_skips_validation() {
+        let args = KafkaArgs {
+            enable: false,
+            brokers: vec![],
+            topic: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_ok());
+    }
+
+    #[test]
+    fn test_validate_tls_client_cert_and_key_must_be_paired() {
+        let args = KafkaArgs {
+            tls_client_cert: "/tmp/client.crt".to_string(),
+            tls_client_key: String::new(),
+            ..base_args()
+        };
+        assert!(args.validate().is_err());
+    }
+}

crates/targets/src/target/mod.rs

@@ -25,6 +25,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::{SystemTime, UNIX_EPOCH};
 use tracing::warn;
 
+pub mod kafka;
 pub mod mqtt;
 pub mod nats;
 pub mod pulsar;