refactor(targets): unify endpoint source/merge logic and bump rustfs-kafka-async to v1.2.0 (#2654)

Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt> Co-authored-by: cxymds <Cxymds@qq.com> Co-authored-by: weisd <im@weisd.in> Co-authored-by: loverustfs <hello@rustfs.com>
2026-08-21 03:46:37 +00:00 · 2026-04-23 17:14:36 +08:00
parent bc37cc4001
commit 368ef0f16c
26 changed files with 1589 additions and 403 deletions
@@ -21,6 +21,6 @@ pub use loader::{
    collect_target_configs_from_env,
 };
 pub use target_args::{
-    build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_mqtt_config, validate_nats_config,
-    validate_pulsar_config, validate_webhook_config,
+    build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
+    validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
 };
@@ -16,6 +16,7 @@ use super::common::{parse_target_bool, parse_url, validate_nats_server_config, v
 use crate::error::TargetError;
 use crate::target::{
    TargetType,
+    kafka::KafkaArgs,
    mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
    nats::{NATSArgs, validate_nats_address},
    pulsar::{PulsarArgs, validate_pulsar_broker},
@@ -23,19 +24,38 @@ use crate::target::{
 };
 use rumqttc::QoS;
 use rustfs_config::{
-    DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
-    MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA,
-    MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
-    NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
-    NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
-    PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME,
-    RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
-    WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
+    DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
+    KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
+    MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
+    MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
+    NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
+    NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
+    PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
+    PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
+    WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
 };
 use rustfs_ecstore::config::KVS;
 use std::path::Path;
 use std::time::Duration;

+fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
+    let Some(value) = value else {
+        return Ok(1);
+    };
+
+    let normalized = value.trim();
+    if normalized.is_empty() {
+        return Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string()));
+    }
+
+    match normalized.to_ascii_lowercase().as_str() {
+        "0" => Ok(0),
+        "1" => Ok(1),
+        "-1" | "all" => Ok(-1),
+        _ => Err(TargetError::Configuration("Kafka acks must be one of: 0, 1, -1, all".to_string())),
+    }
+}
+
 pub fn build_webhook_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<WebhookArgs, TargetError> {
    let endpoint = config
        .lookup(WEBHOOK_ENDPOINT)
@@ -265,3 +285,114 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
        .ok_or_else(|| TargetError::Configuration("Missing Pulsar broker".to_string()))?;
    validate_pulsar_broker_config(&broker, config, default_queue_dir)
 }
+
+pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').all(|s| s.trim().is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+    let brokers: Vec<String> = brokers_raw
+        .split(',')
+        .map(|s| s.trim().to_string())
+        .filter(|s| !s.is_empty())
+        .collect();
+
+    let topic = config
+        .lookup(KAFKA_TOPIC)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka topic".to_string()))?;
+
+    Ok(KafkaArgs {
+        enable: true,
+        brokers,
+        topic,
+        acks: parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?,
+        tls_enable: parse_target_bool(config.lookup(KAFKA_TLS_ENABLE).as_deref()).unwrap_or(false),
+        tls_ca: config.lookup(KAFKA_TLS_CA).unwrap_or_default(),
+        tls_client_cert: config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default(),
+        tls_client_key: config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default(),
+        queue_dir: config
+            .lookup(KAFKA_QUEUE_DIR)
+            .unwrap_or_else(|| default_queue_dir.to_string()),
+        queue_limit: config
+            .lookup(KAFKA_QUEUE_LIMIT)
+            .and_then(|v| v.parse::<u64>().ok())
+            .unwrap_or(DEFAULT_LIMIT),
+        target_type,
+    })
+}
+
+pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
+    let brokers_raw = config
+        .lookup(KAFKA_BROKERS)
+        .ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
+    if brokers_raw.split(',').map(|s| s.trim()).all(|s| s.is_empty()) {
+        return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
+    }
+
+    if config.lookup(KAFKA_TOPIC).is_none() {
+        return Err(TargetError::Configuration("Missing Kafka topic".to_string()));
+    }
+
+    parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?;
+
+    let tls_client_cert = config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default();
+    let tls_client_key = config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default();
+    if tls_client_cert.is_empty() != tls_client_key.is_empty() {
+        return Err(TargetError::Configuration(
+            "Kafka tls_client_cert and tls_client_key must be specified together".to_string(),
+        ));
+    }
+
+    let queue_dir = config
+        .lookup(KAFKA_QUEUE_DIR)
+        .unwrap_or_else(|| default_queue_dir.to_string());
+    if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
+        return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{build_kafka_args, validate_kafka_config};
+    use crate::target::TargetType;
+    use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
+    use rustfs_ecstore::config::KVS;
+
+    fn kafka_base_config() -> KVS {
+        let mut config = KVS::new();
+        config.insert(KAFKA_BROKERS.to_string(), "127.0.0.1:9092".to_string());
+        config.insert(KAFKA_TOPIC.to_string(), "events".to_string());
+        config
+    }
+
+    #[test]
+    fn build_kafka_args_accepts_all_ack_alias() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "all".to_string());
+
+        let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka args");
+        assert_eq!(args.acks, -1);
+    }
+
+    #[test]
+    fn build_kafka_args_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "leader".to_string());
+
+        let err = build_kafka_args(&config, "", TargetType::NotifyEvent).expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+
+    #[test]
+    fn validate_kafka_config_rejects_invalid_acks() {
+        let mut config = kafka_base_config();
+        config.insert(KAFKA_ACKS.to_string(), "2".to_string());
+
+        let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
+        assert!(err.to_string().contains("Kafka acks must be one of"));
+    }
+}