gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(targets): support Kafka SASL auth (#3128)

Browse Source 
* feat(targets): support Kafka SASL auth

* fix(targets): infer Kafka SASL when enable is blank

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-05-30 19:01:17 +08:00
committed by GitHub
parent 97cd19becc
commit 5bdbd66d09
7 changed files with 387 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/targets/src/config/target_args.rs
+129 -52
View File
@@ -17,7 +17,7 @@ use crate::error::TargetError;
use crate::target::{
TargetType,
amqp::AMQPArgs,
kafka::KafkaArgs,
kafka::{KAFKA_SASL_PLAIN, KafkaArgs},
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
mysql::MySqlArgs,
nats::{NATSArgs, validate_nats_address},
@@ -30,21 +30,22 @@ use rumqttc::QoS;
use rustfs_config::{
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS,
KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC,
MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL,
MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR,
MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL,
REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS,
REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY,
REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE, KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME,
KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use rustfs_ecstore::config::KVS;
use std::path::Path;
@@ -68,6 +69,19 @@ fn parse_kafka_acks_value(value: Option<&str>) -> Result<i16, TargetError> {
}
}
fn parse_kafka_sasl_enable(config: &KVS, has_sasl_fields: bool) -> Result<bool, TargetError> {
match config.lookup(KAFKA_SASL_ENABLE) {
Some(value) => {
if value.trim().is_empty() {
return Ok(has_sasl_fields);
}
parse_target_bool(Some(value.as_str()))
.ok_or_else(|| TargetError::Configuration(format!("Invalid Kafka {KAFKA_SASL_ENABLE} boolean value: {value}")))
}
None => Ok(has_sasl_fields),
}
}
fn parse_amqp_bool_value(field: &str, config: &KVS, default: bool) -> Result<bool, TargetError> {
match config.lookup(field) {
Some(value) => parse_target_bool(Some(value.as_str()))
@@ -482,7 +496,13 @@ pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: Targ
.lookup(KAFKA_TOPIC)
.ok_or_else(|| TargetError::Configuration("Missing Kafka topic".to_string()))?;
Ok(KafkaArgs {
let sasl_mechanism = config.lookup(KAFKA_SASL_MECHANISM).unwrap_or_default();
let sasl_username = config.lookup(KAFKA_SASL_USERNAME).unwrap_or_default();
let sasl_password = config.lookup(KAFKA_SASL_PASSWORD).unwrap_or_default();
let has_sasl_fields = !sasl_mechanism.trim().is_empty() || !sasl_username.is_empty() || !sasl_password.is_empty();
let sasl_enable = parse_kafka_sasl_enable(config, has_sasl_fields)?;
let args = KafkaArgs {
enable: true,
brokers,
topic,
@@ -491,6 +511,14 @@ pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: Targ
tls_ca: config.lookup(KAFKA_TLS_CA).unwrap_or_default(),
tls_client_cert: config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default(),
tls_client_key: config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default(),
sasl_enable,
sasl_mechanism: if sasl_enable && sasl_mechanism.trim().is_empty() {
KAFKA_SASL_PLAIN.to_string()
} else {
sasl_mechanism.trim().to_string()
},
sasl_username,
sasl_password,
queue_dir: config
.lookup(KAFKA_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string()),
@@ -499,38 +527,13 @@ pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: Targ
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
target_type,
})
};
args.validate()?;
Ok(args)
}
pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
let brokers_raw = config
.lookup(KAFKA_BROKERS)
.ok_or_else(|| TargetError::Configuration("Missing Kafka brokers".to_string()))?;
if brokers_raw.split(',').map(|s| s.trim()).all(|s| s.is_empty()) {
return Err(TargetError::Configuration("Kafka brokers cannot be empty".to_string()));
}
if config.lookup(KAFKA_TOPIC).is_none() {
return Err(TargetError::Configuration("Missing Kafka topic".to_string()));
}
parse_kafka_acks_value(config.lookup(KAFKA_ACKS).as_deref())?;
let tls_client_cert = config.lookup(KAFKA_TLS_CLIENT_CERT).unwrap_or_default();
let tls_client_key = config.lookup(KAFKA_TLS_CLIENT_KEY).unwrap_or_default();
if tls_client_cert.is_empty() != tls_client_key.is_empty() {
return Err(TargetError::Configuration(
"Kafka tls_client_cert and tls_client_key must be specified together".to_string(),
));
}
let queue_dir = config
.lookup(KAFKA_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string());
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
}
let _ = build_kafka_args(config, default_queue_dir, TargetType::NotifyEvent)?;
Ok(())
}
@@ -594,14 +597,19 @@ mod tests {
build_amqp_args, build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_amqp_config,
validate_kafka_config, validate_mysql_config, validate_postgres_config, validate_redis_config,
};
use crate::target::{TargetType, postgres::PostgresFormat};
use crate::target::{
TargetType,
kafka::{KAFKA_SASL_PLAIN, KAFKA_SASL_SCRAM_SHA_512},
postgres::PostgresFormat,
};
use rustfs_config::{
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_ROUTING_KEY, AMQP_TLS_CLIENT_CERT,
AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING,
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY,
POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_SASL_ENABLE, KAFKA_SASL_MECHANISM,
KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS,
MYSQL_QUEUE_DIR, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING,
POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE,
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
};
use rustfs_ecstore::config::KVS;
@@ -778,6 +786,75 @@ mod tests {
assert!(err.to_string().contains("Kafka acks must be one of"));
}
#[test]
fn build_kafka_args_infers_sasl_enable_from_credentials() {
let mut config = kafka_base_config();
config.insert(KAFKA_TLS_ENABLE.to_string(), "on".to_string());
config.insert(KAFKA_SASL_USERNAME.to_string(), "user".to_string());
config.insert(KAFKA_SASL_PASSWORD.to_string(), "secret".to_string());
let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka SASL args");
assert!(args.sasl_enable);
assert_eq!(args.sasl_mechanism, KAFKA_SASL_PLAIN);
assert_eq!(args.sasl_username, "user");
assert_eq!(args.sasl_password, "secret");
}
#[test]
fn build_kafka_args_infers_sasl_enable_when_enable_is_blank() {
let mut config = kafka_base_config();
config.insert(KAFKA_TLS_ENABLE.to_string(), "on".to_string());
config.insert(KAFKA_SASL_ENABLE.to_string(), "".to_string());
config.insert(KAFKA_SASL_USERNAME.to_string(), "user".to_string());
config.insert(KAFKA_SASL_PASSWORD.to_string(), "secret".to_string());
let args =
build_kafka_args(&config, "", TargetType::NotifyEvent).expect("blank sasl_enable should infer from SASL fields");
assert!(args.sasl_enable);
assert_eq!(args.sasl_mechanism, KAFKA_SASL_PLAIN);
}
#[test]
fn build_kafka_args_accepts_scram_sha_512() {
let mut config = kafka_base_config();
config.insert(KAFKA_TLS_ENABLE.to_string(), "true".to_string());
config.insert(KAFKA_SASL_ENABLE.to_string(), "true".to_string());
config.insert(KAFKA_SASL_MECHANISM.to_string(), "SCRAM-SHA-512".to_string());
config.insert(KAFKA_SASL_USERNAME.to_string(), "user".to_string());
config.insert(KAFKA_SASL_PASSWORD.to_string(), "secret".to_string());
let args = build_kafka_args(&config, "", TargetType::NotifyEvent).expect("valid kafka SCRAM args");
assert!(args.sasl_enable);
assert_eq!(args.sasl_mechanism, KAFKA_SASL_SCRAM_SHA_512);
}
#[test]
fn validate_kafka_config_rejects_sasl_without_tls() {
let mut config = kafka_base_config();
config.insert(KAFKA_SASL_ENABLE.to_string(), "on".to_string());
config.insert(KAFKA_SASL_USERNAME.to_string(), "user".to_string());
config.insert(KAFKA_SASL_PASSWORD.to_string(), "secret".to_string());
let err = validate_kafka_config(&config, "").expect_err("SASL without TLS should fail");
assert!(err.to_string().contains("requires tls_enable"));
}
#[test]
fn validate_kafka_config_rejects_sasl_fields_when_disabled() {
let mut config = kafka_base_config();
config.insert(KAFKA_SASL_ENABLE.to_string(), "off".to_string());
config.insert(KAFKA_SASL_USERNAME.to_string(), "user".to_string());
config.insert(KAFKA_SASL_PASSWORD.to_string(), "secret".to_string());
let err = validate_kafka_config(&config, "").expect_err("SASL fields with disabled SASL should fail");
assert!(err.to_string().contains("sasl_enable must be true"));
}
#[test]
fn build_mysql_args_accepts_minimal_config() {
let args = build_mysql_args(&mysql_base_config(), "", TargetType::NotifyEvent).expect("valid mysql args");