mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
feat(targets): complete redis mysql postgres target wiring (#2842)
Signed-off-by: jaehanbyun <awbrg789@naver.com> Signed-off-by: houseme <housemecn@gmail.com> Signed-off-by: Gunther Xing <jiengup@gmail.com> Signed-off-by: JaySon-Huang <tshent@qq.com> Co-authored-by: jaehanbyun <awbrg789@naver.com> Co-authored-by: Gunther Xing <jiengup@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: cxymds <Cxymds@qq.com> Co-authored-by: JaySon <tshent@qq.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
+61
-16
@@ -78,7 +78,7 @@ pub async fn check_mqtt_broker_available_with_tls(
|
||||
std::time::Duration::from_secs(5),
|
||||
None,
|
||||
)?;
|
||||
let (client, mut eventloop) = AsyncClient::new(mqtt_options, 1);
|
||||
let (client, mut eventloop) = AsyncClient::builder(mqtt_options).capacity(1).build();
|
||||
|
||||
// Try to connect and subscribe
|
||||
client
|
||||
@@ -94,7 +94,7 @@ pub async fn check_mqtt_broker_available_with_tls(
|
||||
}
|
||||
|
||||
pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -> Result<(), crate::TargetError> {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let client = crate::target::nats::connect_nats(args).await?;
|
||||
client
|
||||
.flush()
|
||||
@@ -107,14 +107,11 @@ pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(crate::TargetError::Timeout("NATS connection timed out".to_string())),
|
||||
}
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("NATS connection timed out".to_string())))
|
||||
}
|
||||
|
||||
pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarArgs) -> Result<(), crate::TargetError> {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let client = crate::target::pulsar::connect_pulsar(args).await?;
|
||||
client
|
||||
.lookup_partitioned_topic(args.topic.clone())
|
||||
@@ -123,10 +120,52 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())),
|
||||
}
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())))
|
||||
}
|
||||
|
||||
/// Probes a PostgreSQL server for connectivity and verifies the configured
|
||||
/// table is readable.
|
||||
///
|
||||
/// Used by both the admin validation flow (pre-flight before persisting a
|
||||
/// target) and `PostgresTarget::init()` (runtime startup check). The probe is
|
||||
/// strictly read-only:
|
||||
///
|
||||
/// 1. Build a deadpool pool from `args` (cheap, no actual connection yet).
|
||||
/// 2. Check out a single connection.
|
||||
/// 3. Run `SELECT 1` to confirm the credentials work.
|
||||
/// 4. Run `SELECT 1 FROM <schema>.<table> LIMIT 0` to confirm the relation
|
||||
/// exists and the user has read permission. `LIMIT 0` ensures no rows are
|
||||
/// actually returned and no DML side effects occur.
|
||||
///
|
||||
/// The whole flow is wrapped in an 8s `tokio::time::timeout` so a stuck DNS
|
||||
/// resolver or TLS handshake cannot exhaust the admin layer's outer 10s
|
||||
/// timeout.
|
||||
pub async fn check_postgres_server_available(args: &crate::target::postgres::PostgresArgs) -> Result<(), crate::TargetError> {
|
||||
use crate::target::postgres::{build_pool, map_pg_error, map_pool_error, table_probe_sql};
|
||||
|
||||
args.validate()?;
|
||||
|
||||
let timeout = std::time::Duration::from_secs(8);
|
||||
tokio::time::timeout(timeout, async {
|
||||
let pool = build_pool(args)?;
|
||||
let client = pool
|
||||
.get()
|
||||
.await
|
||||
.map_err(|e| map_pool_error(e, "PostgreSQL connectivity probe failed to acquire connection"))?;
|
||||
client
|
||||
.execute("SELECT 1", &[])
|
||||
.await
|
||||
.map_err(|e| map_pg_error(&e, "PostgreSQL liveness probe failed"))?;
|
||||
let probe_sql = table_probe_sql(&args.schema, &args.table);
|
||||
client
|
||||
.execute(probe_sql.as_str(), &[])
|
||||
.await
|
||||
.map_err(|e| map_pg_error(&e, "PostgreSQL table probe failed"))?;
|
||||
pool.close();
|
||||
Ok::<(), crate::TargetError>(())
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("PostgreSQL connectivity probe timed out".to_string())))
|
||||
}
|
||||
|
||||
pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> {
|
||||
@@ -163,15 +202,21 @@ pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs
|
||||
config = config.with_security(security);
|
||||
}
|
||||
|
||||
match tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config)
|
||||
.await
|
||||
.map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())),
|
||||
}
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())))
|
||||
}
|
||||
|
||||
pub async fn check_redis_server_available(args: &crate::target::redis::RedisArgs) -> Result<(), crate::TargetError> {
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let client = crate::target::redis::build_redis_client(args)?;
|
||||
crate::target::redis::ping_redis_server(&client, args).await
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Redis connection timed out".to_string())))
|
||||
}
|
||||
|
||||
@@ -38,10 +38,24 @@ fn is_sensitive_target_field(field_name: &str) -> bool {
|
||||
|| field_name.contains("client_key")
|
||||
|| field_name.contains("access_key")
|
||||
|| field_name.contains("auth")
|
||||
|| field_name.contains(rustfs_config::BASE_DSN_STRING)
|
||||
}
|
||||
|
||||
fn redact_target_field_value(field_name: &str, value: &str) -> String {
|
||||
if is_sensitive_target_field(field_name) && !value.is_empty() {
|
||||
if value.is_empty() {
|
||||
return value.to_string();
|
||||
}
|
||||
// MySQL DSN fields need partial redaction instead of full masking so the
|
||||
// remaining connection details (host, port, database) remain visible in
|
||||
// debug logs while the password is hidden.
|
||||
if field_name == rustfs_config::BASE_DSN_STRING {
|
||||
let trimmed = value.trim_start();
|
||||
if trimmed.starts_with("postgres://") || trimmed.starts_with("postgresql://") {
|
||||
return crate::target::postgres::redact_postgres_dsn(value);
|
||||
}
|
||||
return crate::target::mysql::redact_mysql_dsn(value);
|
||||
}
|
||||
if is_sensitive_target_field(field_name) {
|
||||
return "***redacted***".to_string();
|
||||
}
|
||||
value.to_string()
|
||||
@@ -283,6 +297,23 @@ mod tests {
|
||||
assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_dsn_string_partial_redaction() {
|
||||
let dsn = "rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events";
|
||||
let redacted = redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, dsn);
|
||||
assert_eq!(redacted, "rustfs:***@tcp(mysql.example.com:3306)/rustfs_events");
|
||||
// empty dsn_string value
|
||||
assert_eq!(redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, ""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redact_postgres_dsn_string_partial_redaction() {
|
||||
let dsn = "postgres://rustfs:secret123@pg.example.com:5432/rustfs_events?search_path=public";
|
||||
let redacted = redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, dsn);
|
||||
assert_eq!(redacted, "postgres://rustfs:***@pg.example.com:5432/rustfs_events?search_path=public");
|
||||
assert_eq!(redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, ""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacted_target_config_masks_sensitive_values_without_mutating_shape() {
|
||||
let mut config = KVS::new();
|
||||
|
||||
@@ -21,6 +21,7 @@ pub use loader::{
|
||||
collect_target_configs_from_env,
|
||||
};
|
||||
pub use target_args::{
|
||||
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
|
||||
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
|
||||
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
|
||||
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
|
||||
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
|
||||
};
|
||||
|
||||
@@ -18,8 +18,11 @@ use crate::target::{
|
||||
TargetType,
|
||||
kafka::KafkaArgs,
|
||||
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
|
||||
mysql::MySqlArgs,
|
||||
nats::{NATSArgs, validate_nats_address},
|
||||
postgres::{PostgresArgs, PostgresDsn, parse_postgres_format},
|
||||
pulsar::{PulsarArgs, validate_pulsar_broker},
|
||||
redis::{RedisArgs, RedisTlsConfig, validate_redis_url},
|
||||
webhook::WebhookArgs,
|
||||
};
|
||||
use rumqttc::QoS;
|
||||
@@ -27,12 +30,19 @@ use rustfs_config::{
|
||||
DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
|
||||
KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
|
||||
MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
|
||||
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
|
||||
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
|
||||
PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
|
||||
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING,
|
||||
MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
|
||||
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
|
||||
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
|
||||
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
|
||||
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
|
||||
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
|
||||
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
|
||||
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
|
||||
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
|
||||
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
|
||||
REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
|
||||
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use rustfs_ecstore::config::KVS;
|
||||
use std::path::Path;
|
||||
@@ -286,6 +296,124 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
|
||||
validate_pulsar_broker_config(&broker, config, default_queue_dir)
|
||||
}
|
||||
|
||||
pub fn build_redis_args(
|
||||
config: &KVS,
|
||||
default_queue_dir: &str,
|
||||
default_channel: &str,
|
||||
target_type: TargetType,
|
||||
) -> Result<RedisArgs, TargetError> {
|
||||
let url = config
|
||||
.lookup(REDIS_URL)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?;
|
||||
let url = parse_url(&url, "Redis URL")?;
|
||||
|
||||
let channel = config
|
||||
.lookup(REDIS_CHANNEL)
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| default_channel.to_string());
|
||||
|
||||
Ok(RedisArgs {
|
||||
enable: true,
|
||||
url,
|
||||
channel,
|
||||
username: config.lookup(REDIS_USERNAME).filter(|value| !value.trim().is_empty()),
|
||||
password: config.lookup(REDIS_PASSWORD).filter(|value| !value.trim().is_empty()),
|
||||
tls: RedisTlsConfig::from_values(
|
||||
config.lookup(REDIS_TLS_POLICY).as_deref(),
|
||||
config.lookup(REDIS_TLS_CA).as_deref(),
|
||||
config.lookup(REDIS_TLS_CLIENT_CERT).as_deref(),
|
||||
config.lookup(REDIS_TLS_CLIENT_KEY).as_deref(),
|
||||
config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(),
|
||||
)?,
|
||||
keep_alive: config
|
||||
.lookup(REDIS_KEEP_ALIVE_INTERVAL)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs)
|
||||
.unwrap_or_else(|| Duration::from_secs(15)),
|
||||
queue_dir: config
|
||||
.lookup(REDIS_QUEUE_DIR)
|
||||
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||
queue_limit: config
|
||||
.lookup(REDIS_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
max_retry_attempts: config
|
||||
.lookup(REDIS_MAX_RETRY_ATTEMPTS)
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(3),
|
||||
reconnect_retry_attempts: config
|
||||
.lookup(REDIS_RECONNECT_RETRY_ATTEMPTS)
|
||||
.and_then(|v| v.parse::<usize>().ok()),
|
||||
min_retry_delay: config
|
||||
.lookup(REDIS_MIN_RETRY_DELAY)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_millis),
|
||||
max_retry_delay: config
|
||||
.lookup(REDIS_MAX_RETRY_DELAY)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_millis),
|
||||
connection_timeout: config
|
||||
.lookup(REDIS_CONNECTION_TIMEOUT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs),
|
||||
response_timeout: config
|
||||
.lookup(REDIS_RESPONSE_TIMEOUT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs),
|
||||
pipeline_buffer_size: config
|
||||
.lookup(REDIS_PIPELINE_BUFFER_SIZE)
|
||||
.and_then(|v| v.parse::<usize>().ok()),
|
||||
target_type,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn build_postgres_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<PostgresArgs, TargetError> {
|
||||
let dsn_string = config
|
||||
.lookup(POSTGRES_DSN_STRING)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing PostgreSQL dsn_string".to_string()))?;
|
||||
let table = config
|
||||
.lookup(POSTGRES_TABLE)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing PostgreSQL table".to_string()))?;
|
||||
|
||||
let schema = PostgresDsn::parse(&dsn_string)?.schema;
|
||||
let format = parse_postgres_format(config.lookup(POSTGRES_FORMAT).as_deref())?;
|
||||
|
||||
Ok(PostgresArgs {
|
||||
enable: true,
|
||||
dsn_string,
|
||||
schema,
|
||||
table,
|
||||
format,
|
||||
tls_required: parse_target_bool(config.lookup(POSTGRES_TLS_REQUIRED).as_deref()).unwrap_or(false),
|
||||
tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(),
|
||||
tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||
tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||
queue_dir: config
|
||||
.lookup(POSTGRES_QUEUE_DIR)
|
||||
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||
queue_limit: config
|
||||
.lookup(POSTGRES_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
target_type,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn validate_redis_config(config: &KVS, default_queue_dir: &str, default_channel: &str) -> Result<(), TargetError> {
|
||||
let url = config
|
||||
.lookup(REDIS_URL)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?;
|
||||
let url = parse_url(&url, "Redis URL")?;
|
||||
validate_redis_url(&url)?;
|
||||
|
||||
let args = build_redis_args(config, default_queue_dir, default_channel, TargetType::NotifyEvent)?;
|
||||
args.validate()
|
||||
}
|
||||
pub fn validate_postgres_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
|
||||
let args = build_postgres_args(config, default_queue_dir, TargetType::NotifyEvent)?;
|
||||
args.validate()
|
||||
}
|
||||
|
||||
pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
|
||||
let brokers_raw = config
|
||||
.lookup(KAFKA_BROKERS)
|
||||
@@ -348,18 +476,81 @@ pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<()
|
||||
let queue_dir = config
|
||||
.lookup(KAFKA_QUEUE_DIR)
|
||||
.unwrap_or_else(|| default_queue_dir.to_string());
|
||||
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
|
||||
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
|
||||
return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Builds `MySqlArgs` from a KVS configuration.
|
||||
///
|
||||
/// Parses all MySQL target configuration keys, applies defaults for
|
||||
/// missing optional values, and validates that all required fields
|
||||
/// are present and well-formed.
|
||||
pub fn build_mysql_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<MySqlArgs, TargetError> {
|
||||
let dsn_string = config
|
||||
.lookup(MYSQL_DSN_STRING)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing MySQL dsn_string".to_string()))?;
|
||||
|
||||
let table = config
|
||||
.lookup(MYSQL_TABLE)
|
||||
.ok_or_else(|| TargetError::Configuration("Missing MySQL table".to_string()))?;
|
||||
|
||||
let args = MySqlArgs {
|
||||
enable: true,
|
||||
dsn_string,
|
||||
table,
|
||||
format: config.lookup(MYSQL_FORMAT).unwrap_or_else(|| "access".to_string()),
|
||||
tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(),
|
||||
tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(),
|
||||
tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(),
|
||||
queue_dir: config
|
||||
.lookup(MYSQL_QUEUE_DIR)
|
||||
.unwrap_or_else(|| default_queue_dir.to_string()),
|
||||
queue_limit: config
|
||||
.lookup(MYSQL_QUEUE_LIMIT)
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(DEFAULT_LIMIT),
|
||||
max_open_connections: config
|
||||
.lookup(MYSQL_MAX_OPEN_CONNECTIONS)
|
||||
.map(|value| {
|
||||
value.trim().parse::<usize>().map_err(|_| {
|
||||
TargetError::Configuration(format!("MySQL max_open_connections value '{}' is not a valid number", value))
|
||||
})
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(2),
|
||||
target_type,
|
||||
};
|
||||
|
||||
args.validate()?;
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
/// Validates MySQL target configuration from a KVS without building args.
|
||||
///
|
||||
/// Performs the same checks as `build_mysql_args` but discards the result,
|
||||
/// used for pre-validation before target creation.
|
||||
pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
|
||||
let _ = build_mysql_args(config, default_queue_dir, TargetType::NotifyEvent)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_kafka_args, validate_kafka_config};
|
||||
use crate::target::TargetType;
|
||||
use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
|
||||
use super::{
|
||||
build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_kafka_config, validate_mysql_config,
|
||||
validate_postgres_config, validate_redis_config,
|
||||
};
|
||||
use crate::target::{TargetType, postgres::PostgresFormat};
|
||||
use rustfs_config::{
|
||||
KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
|
||||
MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
|
||||
POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL,
|
||||
REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE,
|
||||
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
|
||||
};
|
||||
use rustfs_ecstore::config::KVS;
|
||||
|
||||
fn kafka_base_config() -> KVS {
|
||||
@@ -369,6 +560,16 @@ mod tests {
|
||||
config
|
||||
}
|
||||
|
||||
fn mysql_base_config() -> KVS {
|
||||
let mut config = KVS::new();
|
||||
config.insert(
|
||||
MYSQL_DSN_STRING.to_string(),
|
||||
"rustfs:password@tcp(127.0.0.1:3306)/rustfs_events".to_string(),
|
||||
);
|
||||
config.insert(MYSQL_TABLE.to_string(), "rustfs_events".to_string());
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_kafka_args_accepts_all_ack_alias() {
|
||||
let mut config = kafka_base_config();
|
||||
@@ -395,4 +596,273 @@ mod tests {
|
||||
let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
|
||||
assert!(err.to_string().contains("Kafka acks must be one of"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_mysql_args_accepts_minimal_config() {
|
||||
let args = build_mysql_args(&mysql_base_config(), "", TargetType::NotifyEvent).expect("valid mysql args");
|
||||
assert!(args.enable);
|
||||
assert_eq!(args.dsn_string, "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events");
|
||||
assert_eq!(args.table, "rustfs_events");
|
||||
assert_eq!(args.format, "access");
|
||||
assert_eq!(args.max_open_connections, 2);
|
||||
assert_eq!(args.queue_limit, rustfs_config::DEFAULT_LIMIT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_mysql_args_applies_defaults() {
|
||||
let args = build_mysql_args(&mysql_base_config(), "/custom/queue", TargetType::NotifyEvent).expect("valid mysql args");
|
||||
assert_eq!(args.queue_dir, "/custom/queue");
|
||||
assert_eq!(args.queue_limit, 100000);
|
||||
assert_eq!(args.max_open_connections, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_mysql_args_rejects_missing_dsn() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(MYSQL_TABLE.to_string(), "events".to_string());
|
||||
|
||||
let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("missing dsn should fail");
|
||||
assert!(err.to_string().contains("dsn_string"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_mysql_args_rejects_relative_queue_dir() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_QUEUE_DIR.to_string(), "relative/path".to_string());
|
||||
|
||||
let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("relative path should fail");
|
||||
assert!(err.to_string().contains("absolute"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mysql_config_rejects_invalid_max_open_connections() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_MAX_OPEN_CONNECTIONS.to_string(), "not-a-number".to_string());
|
||||
|
||||
let err = validate_mysql_config(&config, "").expect_err("invalid max_open_connections should be rejected");
|
||||
assert!(err.to_string().contains("max_open_connections"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mysql_config_rejects_empty_dsn() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_DSN_STRING.to_string(), "".to_string());
|
||||
|
||||
let err = validate_mysql_config(&config, "").expect_err("empty dsn should fail");
|
||||
assert!(err.to_string().contains("empty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mysql_config_rejects_unpaired_tls_client_fields() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string());
|
||||
|
||||
let err = validate_mysql_config(&config, "").expect_err("unpaired mysql TLS client cert should fail");
|
||||
assert!(err.to_string().contains("must be specified together"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_mysql_config_rejects_relative_tls_paths() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_TLS_CA.to_string(), "ca.pem".to_string());
|
||||
|
||||
let err = validate_mysql_config(&config, "").expect_err("relative tls_ca should fail");
|
||||
assert!(err.to_string().contains("tls_ca must be an absolute path"));
|
||||
|
||||
config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string());
|
||||
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "client.pem".to_string());
|
||||
config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "client.key".to_string());
|
||||
|
||||
let err = validate_mysql_config(&config, "").expect_err("relative tls client paths should fail");
|
||||
assert!(err.to_string().contains("absolute path"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_mysql_args_accepts_absolute_tls_paths() {
|
||||
let mut config = mysql_base_config();
|
||||
config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string());
|
||||
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string());
|
||||
config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "/etc/ssl/mysql/client.key".to_string());
|
||||
|
||||
let args = build_mysql_args(&config, "", TargetType::NotifyEvent).expect("absolute mysql TLS paths should pass");
|
||||
assert_eq!(args.tls_ca, "/etc/ssl/mysql/ca.pem");
|
||||
assert_eq!(args.tls_client_cert, "/etc/ssl/mysql/client.pem");
|
||||
assert_eq!(args.tls_client_key, "/etc/ssl/mysql/client.key");
|
||||
}
|
||||
|
||||
fn redis_base_config() -> KVS {
|
||||
let mut config = KVS::new();
|
||||
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
|
||||
config.insert(REDIS_CHANNEL.to_string(), "events".to_string());
|
||||
config
|
||||
}
|
||||
fn postgres_base_config() -> KVS {
|
||||
let mut config = KVS::new();
|
||||
config.insert(
|
||||
POSTGRES_DSN_STRING.to_string(),
|
||||
"postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public".to_string(),
|
||||
);
|
||||
config.insert(POSTGRES_TABLE.to_string(), "rustfs_events_namespace".to_string());
|
||||
config
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_redis_args_keeps_manager_tuning_fields_none_when_unset() {
|
||||
let config = redis_base_config();
|
||||
|
||||
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
|
||||
|
||||
assert_eq!(args.channel, "events");
|
||||
assert_eq!(args.reconnect_retry_attempts, None);
|
||||
assert_eq!(args.min_retry_delay, None);
|
||||
assert_eq!(args.max_retry_delay, None);
|
||||
assert_eq!(args.connection_timeout, None);
|
||||
assert_eq!(args.response_timeout, None);
|
||||
assert_eq!(args.pipeline_buffer_size, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_redis_args_uses_default_channel_when_missing() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
|
||||
|
||||
let args =
|
||||
build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args");
|
||||
|
||||
assert_eq!(args.channel, "fallback-channel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_redis_args_uses_default_channel_when_empty() {
|
||||
let mut config = KVS::new();
|
||||
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
|
||||
config.insert(REDIS_CHANNEL.to_string(), " ".to_string());
|
||||
|
||||
let args =
|
||||
build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args");
|
||||
|
||||
assert_eq!(args.channel, "fallback-channel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_redis_args_parses_optional_tuning_values_when_present() {
|
||||
let mut config = redis_base_config();
|
||||
config.insert(REDIS_RECONNECT_RETRY_ATTEMPTS.to_string(), "9".to_string());
|
||||
config.insert(REDIS_MIN_RETRY_DELAY.to_string(), "250".to_string());
|
||||
config.insert(REDIS_MAX_RETRY_DELAY.to_string(), "5000".to_string());
|
||||
config.insert(REDIS_CONNECTION_TIMEOUT.to_string(), "7".to_string());
|
||||
config.insert(REDIS_RESPONSE_TIMEOUT.to_string(), "11".to_string());
|
||||
config.insert(REDIS_PIPELINE_BUFFER_SIZE.to_string(), "64".to_string());
|
||||
|
||||
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
|
||||
|
||||
assert_eq!(args.reconnect_retry_attempts, Some(9));
|
||||
assert_eq!(args.min_retry_delay, Some(std::time::Duration::from_millis(250)));
|
||||
assert_eq!(args.max_retry_delay, Some(std::time::Duration::from_millis(5000)));
|
||||
assert_eq!(args.connection_timeout, Some(std::time::Duration::from_secs(7)));
|
||||
assert_eq!(args.response_timeout, Some(std::time::Duration::from_secs(11)));
|
||||
assert_eq!(args.pipeline_buffer_size, Some(64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_redis_args_parses_tls_allow_insecure_when_present() {
|
||||
let mut config = redis_base_config();
|
||||
config.insert(REDIS_URL.to_string(), "rediss://127.0.0.1:6379/0".to_string());
|
||||
config.insert(REDIS_TLS_ALLOW_INSECURE.to_string(), "on".to_string());
|
||||
|
||||
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
|
||||
|
||||
assert!(args.tls.allow_insecure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_redis_config_rejects_missing_url() {
|
||||
let config = KVS::new();
|
||||
|
||||
let err = validate_redis_config(&config, "/tmp/queue", "default-channel").expect_err("missing redis url should fail");
|
||||
assert!(err.to_string().contains("Missing Redis URL"));
|
||||
}
|
||||
#[test]
|
||||
fn build_postgres_args_accepts_minimal_config() {
|
||||
let config = postgres_base_config();
|
||||
let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args");
|
||||
assert_eq!(
|
||||
args.dsn_string,
|
||||
"postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public"
|
||||
);
|
||||
assert_eq!(args.format, PostgresFormat::Namespace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_postgres_args_parses_access_format() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_FORMAT.to_string(), "access".to_string());
|
||||
let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args");
|
||||
assert_eq!(args.format, PostgresFormat::Access);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_missing_dsn_string() {
|
||||
let mut config = postgres_base_config();
|
||||
config.0.retain(|kv| kv.key != POSTGRES_DSN_STRING);
|
||||
let err = validate_postgres_config(&config, "").expect_err("missing dsn_string should fail");
|
||||
assert!(err.to_string().contains("Missing PostgreSQL dsn_string"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_empty_dsn_string() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_DSN_STRING.to_string(), "".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("empty dsn_string should fail");
|
||||
assert!(err.to_string().contains("dsn_string cannot be empty"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_missing_table() {
|
||||
let mut config = postgres_base_config();
|
||||
config.0.retain(|kv| kv.key != POSTGRES_TABLE);
|
||||
let err = validate_postgres_config(&config, "").expect_err("missing table should fail");
|
||||
assert!(err.to_string().contains("Missing PostgreSQL table"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_invalid_dsn() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_DSN_STRING.to_string(), "postgres://".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("invalid dsn should fail");
|
||||
assert!(err.to_string().contains("invalid PostgreSQL dsn_string"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_relative_queue_dir() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_QUEUE_DIR.to_string(), "relative/path".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("relative queue_dir should fail");
|
||||
assert!(err.to_string().contains("absolute path"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_mtls_without_key() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "/etc/ssl/cert.pem".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("missing key should fail");
|
||||
assert!(err.to_string().contains("must be specified together"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_relative_tls_ca() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_TLS_CA.to_string(), "relative/ca.pem".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("relative tls_ca should fail");
|
||||
assert!(err.to_string().contains("must be an absolute path"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_postgres_config_rejects_relative_tls_client_cert() {
|
||||
let mut config = postgres_base_config();
|
||||
config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "relative/client.pem".to_string());
|
||||
config.insert(POSTGRES_TLS_CLIENT_KEY.to_string(), "/etc/ssl/client.key".to_string());
|
||||
let err = validate_postgres_config(&config, "").expect_err("relative tls_client_cert should fail");
|
||||
assert!(err.to_string().contains("must be an absolute path"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ pub mod target;
|
||||
|
||||
pub use check::{
|
||||
check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
|
||||
check_pulsar_broker_available,
|
||||
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
|
||||
};
|
||||
pub use error::{StoreError, TargetError};
|
||||
pub use rustfs_s3_common::EventName;
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::{
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType,
|
||||
TargetType, queue_store_subdir_name,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -136,7 +136,7 @@ where
|
||||
|
||||
let queue_store = if !args.queue_dir.is_empty() {
|
||||
let queue_dir =
|
||||
PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id));
|
||||
PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Kafka.as_str(), &target_id.id));
|
||||
|
||||
let extension = match args.target_type {
|
||||
TargetType::AuditLog => AUDIT_STORE_EXTENSION,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::arn::TargetID;
|
||||
use crate::store::{Key, Store};
|
||||
use crate::{StoreError, TargetError};
|
||||
use crate::{StoreError, TargetError, TargetLog};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_s3_common::EventName;
|
||||
use serde::de::DeserializeOwned;
|
||||
@@ -27,8 +27,11 @@ use tracing::warn;
|
||||
|
||||
pub mod kafka;
|
||||
pub mod mqtt;
|
||||
pub mod mysql;
|
||||
pub mod nats;
|
||||
pub mod postgres;
|
||||
pub mod pulsar;
|
||||
pub mod redis;
|
||||
pub mod webhook;
|
||||
|
||||
/// A read-only snapshot of delivery counters for a target.
|
||||
@@ -289,8 +292,11 @@ pub enum ChannelTargetType {
|
||||
Webhook,
|
||||
Kafka,
|
||||
Mqtt,
|
||||
MySql,
|
||||
Nats,
|
||||
Postgres,
|
||||
Pulsar,
|
||||
Redis,
|
||||
}
|
||||
|
||||
impl ChannelTargetType {
|
||||
@@ -299,8 +305,11 @@ impl ChannelTargetType {
|
||||
ChannelTargetType::Webhook => "webhook",
|
||||
ChannelTargetType::Kafka => "kafka",
|
||||
ChannelTargetType::Mqtt => "mqtt",
|
||||
ChannelTargetType::MySql => "mysql",
|
||||
ChannelTargetType::Nats => "nats",
|
||||
ChannelTargetType::Postgres => "postgres",
|
||||
ChannelTargetType::Pulsar => "pulsar",
|
||||
ChannelTargetType::Redis => "redis",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -311,20 +320,15 @@ impl std::fmt::Display for ChannelTargetType {
|
||||
ChannelTargetType::Webhook => write!(f, "webhook"),
|
||||
ChannelTargetType::Kafka => write!(f, "kafka"),
|
||||
ChannelTargetType::Mqtt => write!(f, "mqtt"),
|
||||
ChannelTargetType::MySql => write!(f, "mysql"),
|
||||
ChannelTargetType::Nats => write!(f, "nats"),
|
||||
ChannelTargetType::Postgres => write!(f, "postgres"),
|
||||
ChannelTargetType::Pulsar => write!(f, "pulsar"),
|
||||
ChannelTargetType::Redis => write!(f, "redis"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_bool(value: &str) -> Result<bool, TargetError> {
|
||||
match value.to_lowercase().as_str() {
|
||||
"true" | "on" | "yes" | "1" => Ok(true),
|
||||
"false" | "off" | "no" | "0" => Ok(false),
|
||||
_ => Err(TargetError::ParseError(format!("Unable to parse boolean: {value}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// `TargetType` enum represents the type of target in the notification system.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum TargetType {
|
||||
@@ -350,6 +354,23 @@ impl std::fmt::Display for TargetType {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sanitize_queue_dir_component(component: &str) -> String {
|
||||
let mut sanitized = String::with_capacity(component.len());
|
||||
for ch in component.chars() {
|
||||
if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') {
|
||||
sanitized.push(ch);
|
||||
} else {
|
||||
sanitized.push('_');
|
||||
}
|
||||
}
|
||||
|
||||
if sanitized.is_empty() { "_".to_string() } else { sanitized }
|
||||
}
|
||||
|
||||
pub(crate) fn queue_store_subdir_name(target_type: &str, target_id: &str) -> String {
|
||||
format!("rustfs-{target_type}-{}", sanitize_queue_dir_component(target_id))
|
||||
}
|
||||
|
||||
/// Decodes a form-urlencoded object name to its original form.
|
||||
///
|
||||
/// This function properly handles form-urlencoded strings where spaces are
|
||||
@@ -377,6 +398,31 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
|
||||
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}")))
|
||||
}
|
||||
|
||||
pub(crate) fn build_queued_payload<E>(event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError>
|
||||
where
|
||||
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
|
||||
{
|
||||
let object_name = decode_object_name(&event.object_name)?;
|
||||
let key = format!("{}/{}", event.bucket_name, object_name);
|
||||
|
||||
let log = TargetLog {
|
||||
event_name: event.event_name,
|
||||
key,
|
||||
records: vec![event.data.clone()],
|
||||
};
|
||||
|
||||
let body = serde_json::to_vec(&log).map_err(|err| TargetError::Serialization(format!("Failed to serialize event: {err}")))?;
|
||||
let meta = QueuedPayloadMeta::new(
|
||||
event.event_name,
|
||||
event.bucket_name.clone(),
|
||||
event.object_name.clone(),
|
||||
"application/json",
|
||||
body.len(),
|
||||
);
|
||||
|
||||
Ok(QueuedPayload::new(meta, body))
|
||||
}
|
||||
|
||||
pub(crate) fn delete_stored_payload(
|
||||
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
|
||||
key: &Key,
|
||||
@@ -412,9 +458,37 @@ mod tests {
|
||||
assert_eq!(decoded.body, br#"{"ok":true}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_queued_payload_uses_event_data_shape() {
|
||||
let event = EntityTarget {
|
||||
object_name: "greeting+file+%282%29.csv".to_string(),
|
||||
bucket_name: "bucket-a".to_string(),
|
||||
event_name: EventName::ObjectCreatedPut,
|
||||
data: "payload-data".to_string(),
|
||||
};
|
||||
|
||||
let payload = build_queued_payload(&event).unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(&payload.body).unwrap();
|
||||
|
||||
assert_eq!(value["Key"], "bucket-a/greeting file (2).csv");
|
||||
assert_eq!(value["Records"][0], "payload-data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_payload_decode_rejects_invalid_magic() {
|
||||
let err = QueuedPayload::decode(b"bad-payload").unwrap_err();
|
||||
assert!(err.to_string().contains("magic") || err.to_string().contains("short"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitize_queue_dir_component_replaces_non_path_safe_characters() {
|
||||
let sanitized = sanitize_queue_dir_component("tenant:alpha/beta\\gamma?*");
|
||||
assert_eq!(sanitized, "tenant_alpha_beta_gamma__");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_store_subdir_name_sanitizes_target_id() {
|
||||
let dir = queue_store_subdir_name("redis", "tenant:alpha");
|
||||
assert_eq!(dir, "rustfs-redis-tenant_alpha");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::{
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType,
|
||||
TargetType, queue_store_subdir_name,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -465,7 +465,7 @@ impl MQTTArgs {
|
||||
}
|
||||
|
||||
if !self.queue_dir.is_empty() {
|
||||
let path = std::path::Path::new(&self.queue_dir);
|
||||
let path = Path::new(&self.queue_dir);
|
||||
if !path.is_absolute() {
|
||||
return Err(TargetError::Configuration("mqtt queue_dir path should be absolute".to_string()));
|
||||
}
|
||||
@@ -512,7 +512,7 @@ where
|
||||
let target_id = TargetID::new(id, ChannelTargetType::Mqtt.as_str().to_string());
|
||||
let queue_store = if !args.queue_dir.is_empty() {
|
||||
let base_path = PathBuf::from(&args.queue_dir);
|
||||
let unique_dir_name = format!("rustfs-{}-{}", ChannelTargetType::Mqtt.as_str(), target_id.id).replace(":", "_");
|
||||
let unique_dir_name = queue_store_subdir_name(ChannelTargetType::Mqtt.as_str(), &target_id.id);
|
||||
// Ensure the directory name is valid for filesystem
|
||||
let specific_queue_path = base_path.join(unique_dir_name);
|
||||
debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target");
|
||||
@@ -582,7 +582,7 @@ where
|
||||
Some(MAX_MQTT_PACKET_SIZE_BYTES),
|
||||
)?;
|
||||
|
||||
let (new_client, eventloop) = AsyncClient::new(mqtt_options, 10);
|
||||
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
|
||||
|
||||
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
|
||||
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
|
||||
@@ -828,7 +828,7 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool {
|
||||
match state_err {
|
||||
// If StateError is caused by deserialization issues, check the underlying MqttBytesError
|
||||
rumqttc::StateError::Deserialization(mqtt_bytes_err) => { // The type of mqtt_bytes_err is &rumqttc::mqttbytes::Error
|
||||
matches!(
|
||||
matches!(
|
||||
mqtt_bytes_err,
|
||||
MqttBytesError::InvalidProtocol // Invalid agreement
|
||||
| MqttBytesError::InvalidProtocolLevel(_) // Invalid protocol level
|
||||
@@ -922,7 +922,7 @@ where
|
||||
Err(e) => {
|
||||
error!(target_id = %self.id, error = %e, "Failed to save event to store");
|
||||
self.delivery_counters.record_final_failure();
|
||||
return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
|
||||
Err(TargetError::Storage(format!("Failed to save event to store: {e}")))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ use crate::{
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType,
|
||||
TargetType, queue_store_subdir_name,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -197,7 +197,7 @@ where
|
||||
let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string());
|
||||
let queue_store = if !args.queue_dir.is_empty() {
|
||||
let base_path = PathBuf::from(&args.queue_dir);
|
||||
let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Nats.as_str(), target_id.id));
|
||||
let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Nats.as_str(), &target_id.id));
|
||||
let extension = match args.target_type {
|
||||
TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
|
||||
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ use crate::{
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType,
|
||||
TargetType, queue_store_subdir_name,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -188,7 +188,7 @@ where
|
||||
let target_id = TargetID::new(id, ChannelTargetType::Pulsar.as_str().to_string());
|
||||
let queue_store = if !args.queue_dir.is_empty() {
|
||||
let base_path = PathBuf::from(&args.queue_dir);
|
||||
let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Pulsar.as_str(), target_id.id));
|
||||
let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Pulsar.as_str(), &target_id.id));
|
||||
let extension = match args.target_type {
|
||||
TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
|
||||
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,7 @@ use crate::{
|
||||
store::{Key, QueueStore, Store},
|
||||
target::{
|
||||
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
|
||||
TargetType,
|
||||
TargetType, queue_store_subdir_name,
|
||||
},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
@@ -147,7 +147,7 @@ where
|
||||
// Build storage
|
||||
let queue_store = if !args.queue_dir.is_empty() {
|
||||
let queue_dir =
|
||||
PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Webhook.as_str(), target_id.id));
|
||||
PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Webhook.as_str(), &target_id.id));
|
||||
|
||||
let extension = match args.target_type {
|
||||
TargetType::AuditLog => AUDIT_STORE_EXTENSION,
|
||||
|
||||
Reference in New Issue
Block a user