gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-27 08:38:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(targets): complete redis mysql postgres target wiring (#2842)

Browse Source 
Signed-off-by: jaehanbyun <awbrg789@naver.com>
Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: Gunther Xing <jiengup@gmail.com>
Signed-off-by: JaySon-Huang <tshent@qq.com>
Co-authored-by: jaehanbyun <awbrg789@naver.com>
Co-authored-by: Gunther Xing <jiengup@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: JaySon <tshent@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-05-07 18:00:59 +08:00
committed by GitHub
parent b159d656cc
commit 5431b9273d
41 changed files with 6787 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+649 -98
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+18 -10
View File
@@ -115,10 +115,11 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.2" }
# Async Runtime and Networking
async-channel = "2.5.0"
mysql_async = { version = "0.36.1", default-features = false, features = ["default-rustls", "tracing"], git = "https://github.com/blackbeam/mysql_async", rev = "98d3d8067efdf97d3e93cdca7b9231753c904aca" }
async-compression = { version = "0.4.42" }
async-recursion = "1.1.1"
async-trait = "0.1.89"
async-nats = "0.47.0"
async-nats = "0.48.0"
axum = "0.8.9"
futures = "0.3.32"
futures-core = "0.3.32"
@@ -139,11 +140,11 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = ["logg
tokio-stream = { version = "0.1.18" }
tokio-test = "0.4.5"
tokio-util = { version = "0.7.18", features = ["io", "compat"] }
tonic = { version = "0.14.5", features = ["gzip"] }
tonic-prost = { version = "0.14.5" }
tonic-prost-build = { version = "0.14.5" }
tonic = { version = "0.14.6", features = ["gzip"] }
tonic-prost = { version = "0.14.6" }
tonic-prost-build = { version = "0.14.6" }
tower = { version = "0.5.3", features = ["timeout"] }
tower-http = { version = "0.6.8", features = ["cors"] }
tower-http = { version = "0.6.10", features = ["cors"] }
# Serialization and Data Formats
bytes = { version = "1.11.1", features = ["serde"] }
@@ -171,6 +172,7 @@ openidconnect = { version = "4.0", default-features = false }
pbkdf2 = "0.13.0"
rsa = { version = "0.10.0-rc.18" }
rustls = { version = "0.23.40", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
rustls-native-certs = "0.8"
rustls-pki-types = "1.14.1"
sha1 = "0.11.0"
sha2 = "0.11.0"
@@ -183,6 +185,11 @@ humantime = "2.3.0"
jiff = { version = "0.2.24", features = ["serde"] }
time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
# Database
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
tokio-postgres = { version = "0.7", default-features = false, features = ["runtime", "with-serde_json-1"] }
tokio-postgres-rustls = "0.13"
# Utilities and Tools
anyhow = "1.0.102"
arc-swap = "1.9.1"
@@ -191,7 +198,7 @@ atoi = "2.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.16" }
aws-credential-types = { version = "1.2.14" }
aws-sdk-s3 = { version = "1.131.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-sdk-s3 = { version = "1.132.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
aws-smithy-types = { version = "1.4.7" }
backtrace = "0.3.76"
@@ -208,12 +215,12 @@ crossbeam-deque = "0.8.6"
crossbeam-utils = "0.8.21"
datafusion = "53.1.0"
derive_builder = "0.20.2"
enumset = "1.1.10"
enumset = "1.1.12"
faster-hex = "0.10.0"
flate2 = "1.1.9"
glob = "0.3.3"
google-cloud-storage = "1.11.0"
google-cloud-auth = "1.9.0"
google-cloud-storage = "1.12.0"
google-cloud-auth = "1.10.0"
hashbrown = { version = "0.17.0", features = ["serde", "rayon"] }
hex = "0.4.3"
hex-simd = "0.8.0"
@@ -246,7 +253,8 @@ rayon = "1.12.0"
reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
reed-solomon-simd = "3.1.0"
regex = { version = "1.12.3" }
rumqttc = { package = "rumqttc-next", version = "0.30.1", features = ["websocket"] }
rumqttc = { package = "rumqttc-next", version = "0.33.0", features = ["websocket"] }
redis = { version = "1.2.1", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
rustix = { version = "1.1.4", features = ["fs"] }
rust-embed = { version = "8.11.0" }
rustc-hash = { version = "2.1.2" }
crates/audit/src/factory.rs
+64 -3
View File
@@ -15,13 +15,17 @@
use crate::AuditEntry;
use async_trait::async_trait;
use rustfs_config::AUDIT_DEFAULT_DIR;
use rustfs_config::audit::{AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_NATS_KEYS, AUDIT_PULSAR_KEYS, AUDIT_WEBHOOK_KEYS};
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_MQTT_KEYS, AUDIT_MYSQL_KEYS, AUDIT_NATS_KEYS, AUDIT_POSTGRES_KEYS, AUDIT_PULSAR_KEYS,
AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_WEBHOOK_KEYS,
};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
@@ -138,3 +142,60 @@ impl TargetFactory for KafkaTargetFactory {
AUDIT_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct RedisTargetFactory;
#[async_trait]
impl TargetFactory for RedisTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_redis_args(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL, TargetType::AuditLog)?;
let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_redis_config(config, AUDIT_DEFAULT_DIR, AUDIT_REDIS_DEFAULT_CHANNEL)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_REDIS_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct MySqlTargetFactory;
#[async_trait]
impl TargetFactory for MySqlTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_mysql_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mysql_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_MYSQL_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct PostgresTargetFactory;
#[async_trait]
impl TargetFactory for PostgresTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<AuditEntry> + Send + Sync>, TargetError> {
let args = build_postgres_args(config, AUDIT_DEFAULT_DIR, TargetType::AuditLog)?;
let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_postgres_config(config, AUDIT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
AUDIT_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect()
}
}
crates/audit/src/registry.rs
+5 -1
View File
@@ -15,7 +15,8 @@
use crate::{
AuditEntry, AuditError, AuditResult,
factory::{
KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory,
RedisTargetFactory, TargetFactory, WebhookTargetFactory,
},
};
use futures::StreamExt;
@@ -56,6 +57,9 @@ impl AuditRegistry {
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory));
registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory));
registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory));
registry
}
crates/config/src/audit/mod.rs
+13
View File
@@ -18,14 +18,20 @@
mod kafka;
mod mqtt;
mod mysql;
mod nats;
mod postgres;
mod pulsar;
mod redis;
mod webhook;
pub use kafka::*;
pub use mqtt::*;
pub use mysql::*;
pub use nats::*;
pub use postgres::*;
pub use pulsar::*;
pub use redis::*;
pub use webhook::*;
use crate::DEFAULT_DELIMITER;
@@ -37,15 +43,22 @@ pub const AUDIT_ROUTE_PREFIX: &str = const_str::concat!(AUDIT_PREFIX, DEFAULT_DE
pub const AUDIT_WEBHOOK_SUB_SYS: &str = "audit_webhook";
pub const AUDIT_KAFKA_SUB_SYS: &str = "audit_kafka";
pub const AUDIT_MQTT_SUB_SYS: &str = "audit_mqtt";
pub const AUDIT_MYSQL_SUB_SYS: &str = "audit_mysql";
pub const AUDIT_NATS_SUB_SYS: &str = "audit_nats";
pub const AUDIT_POSTGRES_SUB_SYS: &str = "audit_postgres";
pub const AUDIT_PULSAR_SUB_SYS: &str = "audit_pulsar";
pub const AUDIT_REDIS_SUB_SYS: &str = "audit_redis";
pub const AUDIT_REDIS_DEFAULT_CHANNEL: &str = "rustfs_audit_channel";
pub const AUDIT_STORE_EXTENSION: &str = ".audit";
#[allow(dead_code)]
pub const AUDIT_SUB_SYSTEMS: &[&str] = &[
AUDIT_KAFKA_SUB_SYS,
AUDIT_MQTT_SUB_SYS,
AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_SUB_SYS,
AUDIT_POSTGRES_SUB_SYS,
AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_SUB_SYS,
AUDIT_WEBHOOK_SUB_SYS,
];
crates/config/src/audit/mysql.rs
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// A list of all valid configuration keys for an audit MySQL target.
pub const AUDIT_MYSQL_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::MYSQL_DSN_STRING,
crate::MYSQL_TABLE,
crate::MYSQL_FORMAT,
crate::MYSQL_TLS_CA,
crate::MYSQL_TLS_CLIENT_CERT,
crate::MYSQL_TLS_CLIENT_KEY,
crate::MYSQL_QUEUE_DIR,
crate::MYSQL_QUEUE_LIMIT,
crate::MYSQL_MAX_OPEN_CONNECTIONS,
crate::COMMENT_KEY,
];
// MySQL environment variables for audit target.
pub const ENV_AUDIT_MYSQL_ENABLE: &str = "RUSTFS_AUDIT_MYSQL_ENABLE";
pub const ENV_AUDIT_MYSQL_DSN_STRING: &str = "RUSTFS_AUDIT_MYSQL_DSN_STRING";
pub const ENV_AUDIT_MYSQL_TABLE: &str = "RUSTFS_AUDIT_MYSQL_TABLE";
pub const ENV_AUDIT_MYSQL_FORMAT: &str = "RUSTFS_AUDIT_MYSQL_FORMAT";
pub const ENV_AUDIT_MYSQL_TLS_CA: &str = "RUSTFS_AUDIT_MYSQL_TLS_CA";
pub const ENV_AUDIT_MYSQL_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_MYSQL_TLS_CLIENT_CERT";
pub const ENV_AUDIT_MYSQL_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_MYSQL_TLS_CLIENT_KEY";
pub const ENV_AUDIT_MYSQL_QUEUE_DIR: &str = "RUSTFS_AUDIT_MYSQL_QUEUE_DIR";
pub const ENV_AUDIT_MYSQL_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_MYSQL_QUEUE_LIMIT";
pub const ENV_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS: &str = "RUSTFS_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS";
pub const ENV_AUDIT_MYSQL_KEYS: &[&str; 10] = &[
ENV_AUDIT_MYSQL_ENABLE,
ENV_AUDIT_MYSQL_DSN_STRING,
ENV_AUDIT_MYSQL_TABLE,
ENV_AUDIT_MYSQL_FORMAT,
ENV_AUDIT_MYSQL_TLS_CA,
ENV_AUDIT_MYSQL_TLS_CLIENT_CERT,
ENV_AUDIT_MYSQL_TLS_CLIENT_KEY,
ENV_AUDIT_MYSQL_QUEUE_DIR,
ENV_AUDIT_MYSQL_QUEUE_LIMIT,
ENV_AUDIT_MYSQL_MAX_OPEN_CONNECTIONS,
];
crates/config/src/audit/postgres.rs
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub const AUDIT_POSTGRES_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::POSTGRES_DSN_STRING,
crate::POSTGRES_TABLE,
crate::POSTGRES_FORMAT,
crate::POSTGRES_TLS_REQUIRED,
crate::POSTGRES_TLS_CA,
crate::POSTGRES_TLS_CLIENT_CERT,
crate::POSTGRES_TLS_CLIENT_KEY,
crate::POSTGRES_QUEUE_DIR,
crate::POSTGRES_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
pub const ENV_AUDIT_POSTGRES_ENABLE: &str = "RUSTFS_AUDIT_POSTGRES_ENABLE";
pub const ENV_AUDIT_POSTGRES_DSN_STRING: &str = "RUSTFS_AUDIT_POSTGRES_DSN_STRING";
pub const ENV_AUDIT_POSTGRES_TABLE: &str = "RUSTFS_AUDIT_POSTGRES_TABLE";
pub const ENV_AUDIT_POSTGRES_FORMAT: &str = "RUSTFS_AUDIT_POSTGRES_FORMAT";
pub const ENV_AUDIT_POSTGRES_TLS_REQUIRED: &str = "RUSTFS_AUDIT_POSTGRES_TLS_REQUIRED";
pub const ENV_AUDIT_POSTGRES_TLS_CA: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CA";
pub const ENV_AUDIT_POSTGRES_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CLIENT_CERT";
pub const ENV_AUDIT_POSTGRES_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_POSTGRES_TLS_CLIENT_KEY";
pub const ENV_AUDIT_POSTGRES_QUEUE_DIR: &str = "RUSTFS_AUDIT_POSTGRES_QUEUE_DIR";
pub const ENV_AUDIT_POSTGRES_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_POSTGRES_QUEUE_LIMIT";
pub const ENV_AUDIT_POSTGRES_KEYS: &[&str; 10] = &[
ENV_AUDIT_POSTGRES_ENABLE,
ENV_AUDIT_POSTGRES_DSN_STRING,
ENV_AUDIT_POSTGRES_TABLE,
ENV_AUDIT_POSTGRES_FORMAT,
ENV_AUDIT_POSTGRES_TLS_REQUIRED,
ENV_AUDIT_POSTGRES_TLS_CA,
ENV_AUDIT_POSTGRES_TLS_CLIENT_CERT,
ENV_AUDIT_POSTGRES_TLS_CLIENT_KEY,
ENV_AUDIT_POSTGRES_QUEUE_DIR,
ENV_AUDIT_POSTGRES_QUEUE_LIMIT,
];
crates/config/src/audit/redis.rs
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub const ENV_AUDIT_REDIS_ENABLE: &str = "RUSTFS_AUDIT_REDIS_ENABLE";
pub const ENV_AUDIT_REDIS_URL: &str = "RUSTFS_AUDIT_REDIS_URL";
pub const ENV_AUDIT_REDIS_CHANNEL: &str = "RUSTFS_AUDIT_REDIS_CHANNEL";
pub const ENV_AUDIT_REDIS_USERNAME: &str = "RUSTFS_AUDIT_REDIS_USERNAME";
pub const ENV_AUDIT_REDIS_PASSWORD: &str = "RUSTFS_AUDIT_REDIS_PASSWORD";
pub const ENV_AUDIT_REDIS_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_AUDIT_REDIS_KEEP_ALIVE_INTERVAL";
pub const ENV_AUDIT_REDIS_QUEUE_DIR: &str = "RUSTFS_AUDIT_REDIS_QUEUE_DIR";
pub const ENV_AUDIT_REDIS_QUEUE_LIMIT: &str = "RUSTFS_AUDIT_REDIS_QUEUE_LIMIT";
pub const ENV_AUDIT_REDIS_MAX_RETRY_ATTEMPTS: &str = "RUSTFS_AUDIT_REDIS_MAX_RETRY_ATTEMPTS";
pub const ENV_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "RUSTFS_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS";
pub const ENV_AUDIT_REDIS_MIN_RETRY_DELAY: &str = "RUSTFS_AUDIT_REDIS_MIN_RETRY_DELAY";
pub const ENV_AUDIT_REDIS_MAX_RETRY_DELAY: &str = "RUSTFS_AUDIT_REDIS_MAX_RETRY_DELAY";
pub const ENV_AUDIT_REDIS_CONNECTION_TIMEOUT: &str = "RUSTFS_AUDIT_REDIS_CONNECTION_TIMEOUT";
pub const ENV_AUDIT_REDIS_RESPONSE_TIMEOUT: &str = "RUSTFS_AUDIT_REDIS_RESPONSE_TIMEOUT";
pub const ENV_AUDIT_REDIS_PIPELINE_BUFFER_SIZE: &str = "RUSTFS_AUDIT_REDIS_PIPELINE_BUFFER_SIZE";
pub const ENV_AUDIT_REDIS_TLS_POLICY: &str = "RUSTFS_AUDIT_REDIS_TLS_POLICY";
pub const ENV_AUDIT_REDIS_TLS_CA: &str = "RUSTFS_AUDIT_REDIS_TLS_CA";
pub const ENV_AUDIT_REDIS_TLS_CLIENT_CERT: &str = "RUSTFS_AUDIT_REDIS_TLS_CLIENT_CERT";
pub const ENV_AUDIT_REDIS_TLS_CLIENT_KEY: &str = "RUSTFS_AUDIT_REDIS_TLS_CLIENT_KEY";
pub const ENV_AUDIT_REDIS_TLS_ALLOW_INSECURE: &str = "RUSTFS_AUDIT_REDIS_TLS_ALLOW_INSECURE";
pub const ENV_AUDIT_REDIS_KEYS: &[&str; 20] = &[
ENV_AUDIT_REDIS_ENABLE,
ENV_AUDIT_REDIS_URL,
ENV_AUDIT_REDIS_CHANNEL,
ENV_AUDIT_REDIS_USERNAME,
ENV_AUDIT_REDIS_PASSWORD,
ENV_AUDIT_REDIS_KEEP_ALIVE_INTERVAL,
ENV_AUDIT_REDIS_QUEUE_DIR,
ENV_AUDIT_REDIS_QUEUE_LIMIT,
ENV_AUDIT_REDIS_MAX_RETRY_ATTEMPTS,
ENV_AUDIT_REDIS_RECONNECT_RETRY_ATTEMPTS,
ENV_AUDIT_REDIS_MIN_RETRY_DELAY,
ENV_AUDIT_REDIS_MAX_RETRY_DELAY,
ENV_AUDIT_REDIS_CONNECTION_TIMEOUT,
ENV_AUDIT_REDIS_RESPONSE_TIMEOUT,
ENV_AUDIT_REDIS_PIPELINE_BUFFER_SIZE,
ENV_AUDIT_REDIS_TLS_POLICY,
ENV_AUDIT_REDIS_TLS_CA,
ENV_AUDIT_REDIS_TLS_CLIENT_CERT,
ENV_AUDIT_REDIS_TLS_CLIENT_KEY,
ENV_AUDIT_REDIS_TLS_ALLOW_INSECURE,
];
pub const AUDIT_REDIS_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::REDIS_URL,
crate::REDIS_CHANNEL,
crate::REDIS_USERNAME,
crate::REDIS_PASSWORD,
crate::REDIS_KEEP_ALIVE_INTERVAL,
crate::REDIS_QUEUE_DIR,
crate::REDIS_QUEUE_LIMIT,
crate::REDIS_MAX_RETRY_ATTEMPTS,
crate::REDIS_RECONNECT_RETRY_ATTEMPTS,
crate::REDIS_MIN_RETRY_DELAY,
crate::REDIS_MAX_RETRY_DELAY,
crate::REDIS_CONNECTION_TIMEOUT,
crate::REDIS_RESPONSE_TIMEOUT,
crate::REDIS_PIPELINE_BUFFER_SIZE,
crate::REDIS_TLS_POLICY,
crate::REDIS_TLS_CA,
crate::REDIS_TLS_CLIENT_CERT,
crate::REDIS_TLS_CLIENT_KEY,
crate::REDIS_TLS_ALLOW_INSECURE,
crate::COMMENT_KEY,
];
crates/config/src/constants/targets.rs
+42
View File
@@ -74,6 +74,48 @@ pub const PULSAR_TLS_HOSTNAME_VERIFICATION: &str = "tls_hostname_verification";
pub const PULSAR_QUEUE_DIR: &str = "queue_dir";
pub const PULSAR_QUEUE_LIMIT: &str = "queue_limit";
pub const BASE_DSN_STRING: &str = "dsn_string";
pub const MYSQL_DSN_STRING: &str = BASE_DSN_STRING;
pub const MYSQL_TABLE: &str = "table";
pub const MYSQL_FORMAT: &str = "format";
pub const MYSQL_TLS_CA: &str = "tls_ca";
pub const MYSQL_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const MYSQL_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const MYSQL_QUEUE_DIR: &str = "queue_dir";
pub const MYSQL_QUEUE_LIMIT: &str = "queue_limit";
pub const MYSQL_MAX_OPEN_CONNECTIONS: &str = "max_open_connections";
pub const REDIS_URL: &str = "url";
pub const REDIS_CHANNEL: &str = "channel";
pub const REDIS_USERNAME: &str = "username";
pub const REDIS_PASSWORD: &str = "password";
pub const REDIS_KEEP_ALIVE_INTERVAL: &str = "keep_alive_interval";
pub const REDIS_QUEUE_DIR: &str = "queue_dir";
pub const REDIS_QUEUE_LIMIT: &str = "queue_limit";
pub const REDIS_MAX_RETRY_ATTEMPTS: &str = "max_retry_attempts";
pub const REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "reconnect_retry_attempts";
pub const REDIS_MIN_RETRY_DELAY: &str = "min_retry_delay";
pub const REDIS_MAX_RETRY_DELAY: &str = "max_retry_delay";
pub const REDIS_CONNECTION_TIMEOUT: &str = "connection_timeout";
pub const REDIS_RESPONSE_TIMEOUT: &str = "response_timeout";
pub const REDIS_PIPELINE_BUFFER_SIZE: &str = "pipeline_buffer_size";
pub const REDIS_TLS_POLICY: &str = "tls_policy";
pub const REDIS_TLS_CA: &str = "tls_ca";
pub const REDIS_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const REDIS_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const REDIS_TLS_ALLOW_INSECURE: &str = "tls_allow_insecure";
pub const POSTGRES_DSN_STRING: &str = BASE_DSN_STRING;
pub const POSTGRES_TABLE: &str = "table";
pub const POSTGRES_FORMAT: &str = "format";
pub const POSTGRES_TLS_REQUIRED: &str = "tls_required";
pub const POSTGRES_TLS_CA: &str = "tls_ca";
pub const POSTGRES_TLS_CLIENT_CERT: &str = "tls_client_cert";
pub const POSTGRES_TLS_CLIENT_KEY: &str = "tls_client_key";
pub const POSTGRES_QUEUE_DIR: &str = "queue_dir";
pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit";
/// Environment variable controlling whether target queue files are Snappy-compressed.
/// Applies to both notify and audit target queue stores.
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
crates/config/src/notify/mod.rs
+11 -4
View File
@@ -15,16 +15,22 @@
mod arn;
mod kafka;
mod mqtt;
mod mysql;
mod nats;
mod postgres;
mod pulsar;
mod redis;
mod store;
mod webhook;
pub use arn::*;
pub use kafka::*;
pub use mqtt::*;
pub use mysql::*;
pub use nats::*;
pub use postgres::*;
pub use pulsar::*;
pub use redis::*;
pub use store::*;
pub use webhook::*;
@@ -69,19 +75,20 @@ pub const ENV_NOTIFY_SEND_CONCURRENCY: &str = "RUSTFS_NOTIFY_SEND_CONCURRENCY";
/// Adjust this value based on your system's capabilities and expected load.
pub const DEFAULT_NOTIFY_SEND_CONCURRENCY: usize = 64;
#[allow(dead_code)]
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
NOTIFY_KAFKA_SUB_SYS,
NOTIFY_MQTT_SUB_SYS,
NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_SUB_SYS,
NOTIFY_POSTGRES_SUB_SYS,
NOTIFY_PULSAR_SUB_SYS,
NOTIFY_REDIS_SUB_SYS,
NOTIFY_WEBHOOK_SUB_SYS,
];
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
#[allow(dead_code)]
pub const NOTIFY_MY_SQL_SUB_SYS: &str = "notify_mysql";
pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql";
#[allow(dead_code)]
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
#[allow(dead_code)]
@@ -90,9 +97,9 @@ pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
#[allow(dead_code)]
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
#[allow(dead_code)]
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
#[allow(dead_code)]
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel";
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
crates/config/src/notify/mysql.rs
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// A list of all valid configuration keys for a MySQL target.
pub const NOTIFY_MYSQL_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::MYSQL_DSN_STRING,
crate::MYSQL_TABLE,
crate::MYSQL_FORMAT,
crate::MYSQL_TLS_CA,
crate::MYSQL_TLS_CLIENT_CERT,
crate::MYSQL_TLS_CLIENT_KEY,
crate::MYSQL_QUEUE_DIR,
crate::MYSQL_QUEUE_LIMIT,
crate::MYSQL_MAX_OPEN_CONNECTIONS,
crate::COMMENT_KEY,
];
// MySQL Environment Variables
pub const ENV_NOTIFY_MYSQL_ENABLE: &str = "RUSTFS_NOTIFY_MYSQL_ENABLE";
pub const ENV_NOTIFY_MYSQL_DSN_STRING: &str = "RUSTFS_NOTIFY_MYSQL_DSN_STRING";
pub const ENV_NOTIFY_MYSQL_TABLE: &str = "RUSTFS_NOTIFY_MYSQL_TABLE";
pub const ENV_NOTIFY_MYSQL_FORMAT: &str = "RUSTFS_NOTIFY_MYSQL_FORMAT";
pub const ENV_NOTIFY_MYSQL_TLS_CA: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CA";
pub const ENV_NOTIFY_MYSQL_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_CERT";
pub const ENV_NOTIFY_MYSQL_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_MYSQL_TLS_CLIENT_KEY";
pub const ENV_NOTIFY_MYSQL_QUEUE_DIR: &str = "RUSTFS_NOTIFY_MYSQL_QUEUE_DIR";
pub const ENV_NOTIFY_MYSQL_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_MYSQL_QUEUE_LIMIT";
pub const ENV_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS: &str = "RUSTFS_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS";
pub const ENV_NOTIFY_MYSQL_KEYS: &[&str; 10] = &[
ENV_NOTIFY_MYSQL_ENABLE,
ENV_NOTIFY_MYSQL_DSN_STRING,
ENV_NOTIFY_MYSQL_TABLE,
ENV_NOTIFY_MYSQL_FORMAT,
ENV_NOTIFY_MYSQL_TLS_CA,
ENV_NOTIFY_MYSQL_TLS_CLIENT_CERT,
ENV_NOTIFY_MYSQL_TLS_CLIENT_KEY,
ENV_NOTIFY_MYSQL_QUEUE_DIR,
ENV_NOTIFY_MYSQL_QUEUE_LIMIT,
ENV_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS,
];
crates/config/src/notify/postgres.rs
+51
View File
@@ -0,0 +1,51 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub const NOTIFY_POSTGRES_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::POSTGRES_DSN_STRING,
crate::POSTGRES_TABLE,
crate::POSTGRES_FORMAT,
crate::POSTGRES_TLS_REQUIRED,
crate::POSTGRES_TLS_CA,
crate::POSTGRES_TLS_CLIENT_CERT,
crate::POSTGRES_TLS_CLIENT_KEY,
crate::POSTGRES_QUEUE_DIR,
crate::POSTGRES_QUEUE_LIMIT,
crate::COMMENT_KEY,
];
pub const ENV_NOTIFY_POSTGRES_ENABLE: &str = "RUSTFS_NOTIFY_POSTGRES_ENABLE";
pub const ENV_NOTIFY_POSTGRES_DSN_STRING: &str = "RUSTFS_NOTIFY_POSTGRES_DSN_STRING";
pub const ENV_NOTIFY_POSTGRES_TABLE: &str = "RUSTFS_NOTIFY_POSTGRES_TABLE";
pub const ENV_NOTIFY_POSTGRES_FORMAT: &str = "RUSTFS_NOTIFY_POSTGRES_FORMAT";
pub const ENV_NOTIFY_POSTGRES_TLS_REQUIRED: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_REQUIRED";
pub const ENV_NOTIFY_POSTGRES_TLS_CA: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CA";
pub const ENV_NOTIFY_POSTGRES_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CLIENT_CERT";
pub const ENV_NOTIFY_POSTGRES_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_POSTGRES_TLS_CLIENT_KEY";
pub const ENV_NOTIFY_POSTGRES_QUEUE_DIR: &str = "RUSTFS_NOTIFY_POSTGRES_QUEUE_DIR";
pub const ENV_NOTIFY_POSTGRES_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_POSTGRES_QUEUE_LIMIT";
pub const ENV_NOTIFY_POSTGRES_KEYS: &[&str; 10] = &[
ENV_NOTIFY_POSTGRES_ENABLE,
ENV_NOTIFY_POSTGRES_DSN_STRING,
ENV_NOTIFY_POSTGRES_TABLE,
ENV_NOTIFY_POSTGRES_FORMAT,
ENV_NOTIFY_POSTGRES_TLS_REQUIRED,
ENV_NOTIFY_POSTGRES_TLS_CA,
ENV_NOTIFY_POSTGRES_TLS_CLIENT_CERT,
ENV_NOTIFY_POSTGRES_TLS_CLIENT_KEY,
ENV_NOTIFY_POSTGRES_QUEUE_DIR,
ENV_NOTIFY_POSTGRES_QUEUE_LIMIT,
];
crates/config/src/notify/redis.rs
+82
View File
@@ -0,0 +1,82 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/// A list of all valid configuration keys for a Redis target.
pub const NOTIFY_REDIS_KEYS: &[&str] = &[
crate::ENABLE_KEY,
crate::REDIS_URL,
crate::REDIS_CHANNEL,
crate::REDIS_USERNAME,
crate::REDIS_PASSWORD,
crate::REDIS_KEEP_ALIVE_INTERVAL,
crate::REDIS_QUEUE_DIR,
crate::REDIS_QUEUE_LIMIT,
crate::REDIS_MAX_RETRY_ATTEMPTS,
crate::REDIS_RECONNECT_RETRY_ATTEMPTS,
crate::REDIS_MIN_RETRY_DELAY,
crate::REDIS_MAX_RETRY_DELAY,
crate::REDIS_CONNECTION_TIMEOUT,
crate::REDIS_RESPONSE_TIMEOUT,
crate::REDIS_PIPELINE_BUFFER_SIZE,
crate::REDIS_TLS_POLICY,
crate::REDIS_TLS_CA,
crate::REDIS_TLS_CLIENT_CERT,
crate::REDIS_TLS_CLIENT_KEY,
crate::REDIS_TLS_ALLOW_INSECURE,
crate::COMMENT_KEY,
];
pub const ENV_NOTIFY_REDIS_ENABLE: &str = "RUSTFS_NOTIFY_REDIS_ENABLE";
pub const ENV_NOTIFY_REDIS_URL: &str = "RUSTFS_NOTIFY_REDIS_URL";
pub const ENV_NOTIFY_REDIS_CHANNEL: &str = "RUSTFS_NOTIFY_REDIS_CHANNEL";
pub const ENV_NOTIFY_REDIS_USERNAME: &str = "RUSTFS_NOTIFY_REDIS_USERNAME";
pub const ENV_NOTIFY_REDIS_PASSWORD: &str = "RUSTFS_NOTIFY_REDIS_PASSWORD";
pub const ENV_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL: &str = "RUSTFS_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL";
pub const ENV_NOTIFY_REDIS_QUEUE_DIR: &str = "RUSTFS_NOTIFY_REDIS_QUEUE_DIR";
pub const ENV_NOTIFY_REDIS_QUEUE_LIMIT: &str = "RUSTFS_NOTIFY_REDIS_QUEUE_LIMIT";
pub const ENV_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS: &str = "RUSTFS_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS";
pub const ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS: &str = "RUSTFS_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS";
pub const ENV_NOTIFY_REDIS_MIN_RETRY_DELAY: &str = "RUSTFS_NOTIFY_REDIS_MIN_RETRY_DELAY";
pub const ENV_NOTIFY_REDIS_MAX_RETRY_DELAY: &str = "RUSTFS_NOTIFY_REDIS_MAX_RETRY_DELAY";
pub const ENV_NOTIFY_REDIS_CONNECTION_TIMEOUT: &str = "RUSTFS_NOTIFY_REDIS_CONNECTION_TIMEOUT";
pub const ENV_NOTIFY_REDIS_RESPONSE_TIMEOUT: &str = "RUSTFS_NOTIFY_REDIS_RESPONSE_TIMEOUT";
pub const ENV_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE: &str = "RUSTFS_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE";
pub const ENV_NOTIFY_REDIS_TLS_POLICY: &str = "RUSTFS_NOTIFY_REDIS_TLS_POLICY";
pub const ENV_NOTIFY_REDIS_TLS_CA: &str = "RUSTFS_NOTIFY_REDIS_TLS_CA";
pub const ENV_NOTIFY_REDIS_TLS_CLIENT_CERT: &str = "RUSTFS_NOTIFY_REDIS_TLS_CLIENT_CERT";
pub const ENV_NOTIFY_REDIS_TLS_CLIENT_KEY: &str = "RUSTFS_NOTIFY_REDIS_TLS_CLIENT_KEY";
pub const ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE: &str = "RUSTFS_NOTIFY_REDIS_TLS_ALLOW_INSECURE";
pub const ENV_NOTIFY_REDIS_KEYS: &[&str; 20] = &[
ENV_NOTIFY_REDIS_ENABLE,
ENV_NOTIFY_REDIS_URL,
ENV_NOTIFY_REDIS_CHANNEL,
ENV_NOTIFY_REDIS_USERNAME,
ENV_NOTIFY_REDIS_PASSWORD,
ENV_NOTIFY_REDIS_KEEP_ALIVE_INTERVAL,
ENV_NOTIFY_REDIS_QUEUE_DIR,
ENV_NOTIFY_REDIS_QUEUE_LIMIT,
ENV_NOTIFY_REDIS_MAX_RETRY_ATTEMPTS,
ENV_NOTIFY_REDIS_RECONNECT_RETRY_ATTEMPTS,
ENV_NOTIFY_REDIS_MIN_RETRY_DELAY,
ENV_NOTIFY_REDIS_MAX_RETRY_DELAY,
ENV_NOTIFY_REDIS_CONNECTION_TIMEOUT,
ENV_NOTIFY_REDIS_RESPONSE_TIMEOUT,
ENV_NOTIFY_REDIS_PIPELINE_BUFFER_SIZE,
ENV_NOTIFY_REDIS_TLS_POLICY,
ENV_NOTIFY_REDIS_TLS_CA,
ENV_NOTIFY_REDIS_TLS_CLIENT_CERT,
ENV_NOTIFY_REDIS_TLS_CLIENT_KEY,
ENV_NOTIFY_REDIS_TLS_ALLOW_INSECURE,
];
crates/ecstore/src/config/audit.rs
+244 -6
View File
@@ -13,17 +13,25 @@
// limitations under the License.
use crate::config::{KV, KVS};
use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -339,6 +347,176 @@ pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
])
});
pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: REDIS_URL.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_CHANNEL.to_owned(),
value: AUDIT_REDIS_DEFAULT_CHANNEL.to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
value: "15".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
value: "3".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_MIN_RETRY_DELAY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_MAX_RETRY_DELAY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_CONNECTION_TIMEOUT.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_RESPONSE_TIMEOUT.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_TLS_POLICY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_DSN_STRING.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TABLE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_FORMAT.to_owned(),
value: "namespace".to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_TLS_REQUIRED.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
@@ -398,3 +576,63 @@ pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
},
])
});
pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: MYSQL_DSN_STRING.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TABLE.to_owned(),
value: "rustfs_audit_logs".to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_FORMAT.to_owned(),
value: "access".to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
value: "2".to_owned(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
crates/ecstore/src/config/com.rs
+66 -8
View File
@@ -19,12 +19,14 @@ use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
@@ -58,7 +60,7 @@ struct TargetConfigDescriptor {
valid_keys: &'static [&'static str],
}
fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
fn notify_target_descriptors() -> [TargetConfigDescriptor; 8] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -78,12 +80,30 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
default_kvs: &notify::DEFAULT_NOTIFY_MQTT_KVS,
valid_keys: NOTIFY_MQTT_KEYS,
},
TargetConfigDescriptor {
external_key: "mysql",
subsystem_key: NOTIFY_MYSQL_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_MYSQL_KVS,
valid_keys: NOTIFY_MYSQL_KEYS,
},
TargetConfigDescriptor {
external_key: "nats",
subsystem_key: NOTIFY_NATS_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_NATS_KVS,
valid_keys: NOTIFY_NATS_KEYS,
},
TargetConfigDescriptor {
external_key: "postgres",
subsystem_key: NOTIFY_POSTGRES_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_POSTGRES_KVS,
valid_keys: NOTIFY_POSTGRES_KEYS,
},
TargetConfigDescriptor {
external_key: "redis",
subsystem_key: NOTIFY_REDIS_SUB_SYS,
default_kvs: &notify::DEFAULT_NOTIFY_REDIS_KVS,
valid_keys: NOTIFY_REDIS_KEYS,
},
TargetConfigDescriptor {
external_key: "pulsar",
subsystem_key: NOTIFY_PULSAR_SUB_SYS,
@@ -93,7 +113,7 @@ fn notify_target_descriptors() -> [TargetConfigDescriptor; 5] {
]
}
fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
fn audit_target_descriptors() -> [TargetConfigDescriptor; 8] {
[
TargetConfigDescriptor {
external_key: "webhook",
@@ -113,18 +133,36 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 5] {
default_kvs: &audit::DEFAULT_AUDIT_MQTT_KVS,
valid_keys: AUDIT_MQTT_KEYS,
},
TargetConfigDescriptor {
external_key: "mysql",
subsystem_key: AUDIT_MYSQL_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_MYSQL_KVS,
valid_keys: AUDIT_MYSQL_KEYS,
},
TargetConfigDescriptor {
external_key: "nats",
subsystem_key: AUDIT_NATS_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_NATS_KVS,
valid_keys: AUDIT_NATS_KEYS,
},
TargetConfigDescriptor {
external_key: "postgres",
subsystem_key: AUDIT_POSTGRES_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_POSTGRES_KVS,
valid_keys: AUDIT_POSTGRES_KEYS,
},
TargetConfigDescriptor {
external_key: "pulsar",
subsystem_key: AUDIT_PULSAR_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_PULSAR_KVS,
valid_keys: AUDIT_PULSAR_KEYS,
},
TargetConfigDescriptor {
external_key: "redis",
subsystem_key: AUDIT_REDIS_SUB_SYS,
default_kvs: &audit::DEFAULT_AUDIT_REDIS_KVS,
valid_keys: AUDIT_REDIS_KEYS,
},
]
}
@@ -1132,9 +1170,11 @@ mod tests {
};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState};
use rustfs_config::{
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
};
use rustfs_filemeta::FileInfo;
use rustfs_lock::client::LockClient;
use rustfs_lock::client::local::LocalClient;
@@ -1775,6 +1815,15 @@ mod tests {
"acks":"all",
"tls_enable":true
}
},
"mysql":{
"primary":{
"enable":true,
"dsn_string":"rustfs:password@tcp(127.0.0.1:3306)/rustfs_events",
"table":"rustfs_events",
"queue_dir":"/tmp/mysql-queue",
"max_open_connections":"2"
}
}
}
}"#;
@@ -1811,6 +1860,15 @@ mod tests {
assert_eq!(kafka.get(rustfs_config::KAFKA_TOPIC), "events-kafka");
assert_eq!(kafka.get(rustfs_config::KAFKA_ACKS), "all");
assert_eq!(kafka.get(rustfs_config::KAFKA_TLS_ENABLE), "true");
let mysql = cfg
.get_value(NOTIFY_MYSQL_SUB_SYS, "primary")
.expect("mysql target should be decoded");
assert_eq!(mysql.get(ENABLE_KEY), EnableState::On.to_string());
assert_eq!(mysql.get(MYSQL_DSN_STRING), "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events");
assert_eq!(mysql.get(MYSQL_TABLE), "rustfs_events");
assert_eq!(mysql.get(MYSQL_QUEUE_DIR), "/tmp/mysql-queue");
assert_eq!(mysql.get(MYSQL_MAX_OPEN_CONNECTIONS), "2");
}
#[test]
crates/ecstore/src/config/mod.rs
+10 -2
View File
@@ -26,10 +26,12 @@ use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use rustfs_config::COMMENT_KEY;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::audit::{
AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS,
AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::{
NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_SUB_SYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use serde::{Deserialize, Serialize};
@@ -243,10 +245,16 @@ pub fn init() {
kvs.insert(AUDIT_MQTT_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MQTT_KVS.clone());
kvs.insert(NOTIFY_NATS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_NATS_KVS.clone());
kvs.insert(AUDIT_NATS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_NATS_KVS.clone());
kvs.insert(NOTIFY_REDIS_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_REDIS_KVS.clone());
kvs.insert(AUDIT_REDIS_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_REDIS_KVS.clone());
kvs.insert(NOTIFY_POSTGRES_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_POSTGRES_KVS.clone());
kvs.insert(AUDIT_POSTGRES_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_POSTGRES_KVS.clone());
kvs.insert(NOTIFY_PULSAR_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_PULSAR_KVS.clone());
kvs.insert(AUDIT_PULSAR_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_PULSAR_KVS.clone());
kvs.insert(NOTIFY_KAFKA_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_KAFKA_KVS.clone());
kvs.insert(AUDIT_KAFKA_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_KAFKA_KVS.clone());
kvs.insert(NOTIFY_MYSQL_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_MYSQL_KVS.clone());
kvs.insert(AUDIT_MYSQL_SUB_SYS.to_owned(), audit::DEFAULT_AUDIT_MYSQL_KVS.clone());
kvs.insert(IDENTITY_OPENID_SUB_SYS.to_owned(), oidc::DEFAULT_IDENTITY_OPENID_KVS.clone());
// Register all default configurations
crates/ecstore/src/config/notify.rs
+244 -5
View File
@@ -13,16 +13,24 @@
// limitations under the License.
use crate::config::{KV, KVS};
use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR,
KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER,
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN,
PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT,
MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD,
NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED,
NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -316,6 +324,176 @@ pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
])
});
pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: REDIS_URL.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_CHANNEL.to_owned(),
value: NOTIFY_REDIS_DEFAULT_CHANNEL.to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
value: "15".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
value: "3".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_MIN_RETRY_DELAY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_MAX_RETRY_DELAY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_CONNECTION_TIMEOUT.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_RESPONSE_TIMEOUT.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: REDIS_TLS_POLICY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_DSN_STRING.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TABLE.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_FORMAT.to_owned(),
value: "namespace".to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_TLS_REQUIRED.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: POSTGRES_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: POSTGRES_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
@@ -375,3 +553,64 @@ pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
},
])
});
/// MySQL notification target default configuration
pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![
KV {
key: ENABLE_KEY.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: MYSQL_DSN_STRING.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TABLE.to_owned(),
value: "rustfs_events".to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_FORMAT.to_owned(),
value: "access".to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_TLS_CA.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TLS_CLIENT_CERT.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_TLS_CLIENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: MYSQL_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
hidden_if_empty: false,
},
KV {
key: MYSQL_QUEUE_LIMIT.to_owned(),
value: DEFAULT_LIMIT.to_string(),
hidden_if_empty: false,
},
KV {
key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
value: "2".to_owned(),
hidden_if_empty: false,
},
KV {
key: COMMENT_KEY.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
])
});
crates/notify/src/factory.rs
+64 -3
View File
@@ -15,13 +15,17 @@
use crate::Event;
use async_trait::async_trait;
use rustfs_config::EVENT_DEFAULT_DIR;
use rustfs_config::notify::{NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_NATS_KEYS, NOTIFY_PULSAR_KEYS, NOTIFY_WEBHOOK_KEYS};
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_MQTT_KEYS, NOTIFY_MYSQL_KEYS, NOTIFY_NATS_KEYS, NOTIFY_POSTGRES_KEYS, NOTIFY_PULSAR_KEYS,
NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_WEBHOOK_KEYS,
};
use rustfs_ecstore::config::KVS;
use rustfs_targets::{
Target,
config::{
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
},
error::TargetError,
target::TargetType,
@@ -120,6 +124,25 @@ impl TargetFactory for PulsarTargetFactory {
}
}
pub struct PostgresTargetFactory;
#[async_trait]
impl TargetFactory for PostgresTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_postgres_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::postgres::PostgresTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_postgres_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_POSTGRES_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct KafkaTargetFactory;
#[async_trait]
@@ -138,3 +161,41 @@ impl TargetFactory for KafkaTargetFactory {
NOTIFY_KAFKA_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct MySqlTargetFactory;
#[async_trait]
impl TargetFactory for MySqlTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_mysql_args(config, EVENT_DEFAULT_DIR, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::mysql::MySqlTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_mysql_config(config, EVENT_DEFAULT_DIR)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_MYSQL_KEYS.iter().map(|s| s.to_string()).collect()
}
}
pub struct RedisTargetFactory;
#[async_trait]
impl TargetFactory for RedisTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target<Event> + Send + Sync>, TargetError> {
let args = build_redis_args(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL, TargetType::NotifyEvent)?;
let target = rustfs_targets::target::redis::RedisTarget::new(id, args)?;
Ok(Box::new(target))
}
fn validate_config(&self, _id: &str, config: &KVS) -> Result<(), TargetError> {
validate_redis_config(config, EVENT_DEFAULT_DIR, NOTIFY_REDIS_DEFAULT_CHANNEL)
}
fn get_valid_fields(&self) -> HashSet<String> {
NOTIFY_REDIS_KEYS.iter().map(|s| s.to_string()).collect()
}
}
crates/notify/src/integration.rs
+18 -2
View File
@@ -25,8 +25,8 @@ use crate::{
use hashbrown::HashMap;
use rustfs_config::notify::{
DEFAULT_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_TARGET_STREAM_CONCURRENCY, ENV_NOTIFY_WEBHOOK_ENABLE,
ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_SUB_SYS,
NOTIFY_WEBHOOK_SUB_SYS,
ENV_NOTIFY_WEBHOOK_ENDPOINT, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_SUB_SYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::{ENV_NOTIFY_ENABLE, EVENT_DEFAULT_DIR};
use rustfs_ecstore::config::{Config, KVS};
@@ -57,8 +57,11 @@ fn subsystem_target_type(target_type: &str) -> &str {
NOTIFY_WEBHOOK_SUB_SYS => "webhook",
NOTIFY_KAFKA_SUB_SYS => "kafka",
NOTIFY_MQTT_SUB_SYS => "mqtt",
NOTIFY_MYSQL_SUB_SYS => "mysql",
NOTIFY_NATS_SUB_SYS => "nats",
NOTIFY_POSTGRES_SUB_SYS => "postgres",
NOTIFY_PULSAR_SUB_SYS => "pulsar",
NOTIFY_REDIS_SUB_SYS => "redis",
_ => target_type,
}
}
@@ -802,4 +805,17 @@ mod tests {
assert_eq!(target_id.id, "ledger");
assert_eq!(target_id.name, "pulsar");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_redis_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_REDIS_SUB_SYS, "Primary");
assert_eq!(target_id.id, "primary");
assert_eq!(target_id.name, "redis");
}
#[test]
fn runtime_target_id_for_subsystem_maps_notify_postgres_to_runtime_type() {
let target_id = runtime_target_id_for_subsystem(NOTIFY_POSTGRES_SUB_SYS, "AuditTrail");
assert_eq!(target_id.id, "audittrail");
assert_eq!(target_id.name, "postgres");
}
}
crates/notify/src/registry.rs
+5 -1
View File
@@ -14,7 +14,8 @@
use crate::Event;
use crate::factory::{
KafkaTargetFactory, MQTTTargetFactory, NATSTargetFactory, PulsarTargetFactory, TargetFactory, WebhookTargetFactory,
KafkaTargetFactory, MQTTTargetFactory, MySqlTargetFactory, NATSTargetFactory, PostgresTargetFactory, PulsarTargetFactory,
RedisTargetFactory, TargetFactory, WebhookTargetFactory,
};
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashMap;
@@ -46,8 +47,11 @@ impl TargetRegistry {
registry.register(ChannelTargetType::Webhook.as_str(), Box::new(WebhookTargetFactory));
registry.register(ChannelTargetType::Mqtt.as_str(), Box::new(MQTTTargetFactory));
registry.register(ChannelTargetType::Nats.as_str(), Box::new(NATSTargetFactory));
registry.register(ChannelTargetType::Postgres.as_str(), Box::new(PostgresTargetFactory));
registry.register(ChannelTargetType::Pulsar.as_str(), Box::new(PulsarTargetFactory));
registry.register(ChannelTargetType::Kafka.as_str(), Box::new(KafkaTargetFactory));
registry.register(ChannelTargetType::MySql.as_str(), Box::new(MySqlTargetFactory));
registry.register(ChannelTargetType::Redis.as_str(), Box::new(RedisTargetFactory));
registry
}
crates/targets/AGENTS.md
+62
View File
@@ -0,0 +1,62 @@
# Targets Crate Instructions
Applies to `crates/targets/`.
`rustfs-targets` provides the notification target abstraction layer:
the `Target<E>` trait, built-in implementations (Webhook, Kafka, MQTT,
NATS, Pulsar, MySQL), persistent queue store, DSN/configuration
builders, and the `ChannelTargetType` registry that maps target types
to their runtime factories.
## Library Design
- Treat crate code as reusable library code by default.
- Prefer `thiserror` for library-facing error types.
- Do not use `unwrap()`, `expect()`, or panic-driven control flow outside tests.
## Testing
- Keep unit tests close to the module they test.
- Keep integration tests under `crates/targets/tests/` directory.
- Add regression tests for bug fixes and behavior changes.
## Async and Performance
- Keep async paths non-blocking.
- Move CPU-heavy operations out of async hot paths with `tokio::task::spawn_blocking` when appropriate.
## Integration Tests
### MySQL Integration Tests
Integration tests in `tests/mysql_integration.rs` require a running MySQL 8.0+
or TiDB 8.5+ instance. They are `#[ignore]` by default so CI never runs them.
Start a test MySQL instance with Podman:
```bash
podman run -d --name rustfs-mysql-test \
-e MYSQL_ROOT_PASSWORD=testpass \
-e MYSQL_DATABASE=testdb \
-p 3306:3306 \
docker.io/library/mysql:8.0.36
```
Wait for MySQL to be ready (look for `ready for connections` in logs),
then run the integration tests:
```bash
RUSTFS_MYSQL_TEST_DSN="root:testpass@tcp(127.0.0.1:3306)/testdb" \
cargo test -p rustfs-targets -- --ignored
```
Clean up:
```bash
podman rm -f rustfs-mysql-test
```
## Suggested Validation
- `cargo test -p rustfs-targets`
- Full gate before commit: `make pre-commit`
crates/targets/Cargo.toml
+9
View File
@@ -18,25 +18,34 @@ rustfs-utils = { workspace = true, features = ["notify", "tls"] }
rustfs-s3-common = { workspace = true }
async-trait = { workspace = true }
async-nats = { workspace = true }
deadpool-postgres = { workspace = true }
hyper-rustls = { workspace = true }
pulsar = { workspace = true }
reqwest = { workspace = true }
rumqttc = { workspace = true }
redis = { workspace = true }
rustls = { workspace = true }
rustls-native-certs = { workspace = true }
rustls-pki-types = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
snap = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"] }
tokio-postgres = { workspace = true }
tokio-postgres-rustls = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde"] }
sysinfo = { workspace = true, features = ["multithread"] }
rustfs-kafka-async = { workspace = true }
mysql_async = { workspace = true }
chrono = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
tempfile = { workspace = true }
[[bench]]
name = "queue_store_benchmark"
crates/targets/src/check.rs
+61 -16
View File
@@ -78,7 +78,7 @@ pub async fn check_mqtt_broker_available_with_tls(
std::time::Duration::from_secs(5),
None,
)?;
let (client, mut eventloop) = AsyncClient::new(mqtt_options, 1);
let (client, mut eventloop) = AsyncClient::builder(mqtt_options).capacity(1).build();
// Try to connect and subscribe
client
@@ -94,7 +94,7 @@ pub async fn check_mqtt_broker_available_with_tls(
}
pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -> Result<(), crate::TargetError> {
match tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let client = crate::target::nats::connect_nats(args).await?;
client
.flush()
@@ -107,14 +107,11 @@ pub async fn check_nats_server_available(args: &crate::target::nats::NATSArgs) -
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err(crate::TargetError::Timeout("NATS connection timed out".to_string())),
}
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("NATS connection timed out".to_string())))
}
pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarArgs) -> Result<(), crate::TargetError> {
match tokio::time::timeout(std::time::Duration::from_secs(5), async {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let client = crate::target::pulsar::connect_pulsar(args).await?;
client
.lookup_partitioned_topic(args.topic.clone())
@@ -123,10 +120,52 @@ pub async fn check_pulsar_broker_available(args: &crate::target::pulsar::PulsarA
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())),
}
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Pulsar connection timed out".to_string())))
}
/// Probes a PostgreSQL server for connectivity and verifies the configured
/// table is readable.
///
/// Used by both the admin validation flow (pre-flight before persisting a
/// target) and `PostgresTarget::init()` (runtime startup check). The probe is
/// strictly read-only:
///
/// 1. Build a deadpool pool from `args` (cheap, no actual connection yet).
/// 2. Check out a single connection.
/// 3. Run `SELECT 1` to confirm the credentials work.
/// 4. Run `SELECT 1 FROM <schema>.<table> LIMIT 0` to confirm the relation
/// exists and the user has read permission. `LIMIT 0` ensures no rows are
/// actually returned and no DML side effects occur.
///
/// The whole flow is wrapped in an 8s `tokio::time::timeout` so a stuck DNS
/// resolver or TLS handshake cannot exhaust the admin layer's outer 10s
/// timeout.
pub async fn check_postgres_server_available(args: &crate::target::postgres::PostgresArgs) -> Result<(), crate::TargetError> {
use crate::target::postgres::{build_pool, map_pg_error, map_pool_error, table_probe_sql};
args.validate()?;
let timeout = std::time::Duration::from_secs(8);
tokio::time::timeout(timeout, async {
let pool = build_pool(args)?;
let client = pool
.get()
.await
.map_err(|e| map_pool_error(e, "PostgreSQL connectivity probe failed to acquire connection"))?;
client
.execute("SELECT 1", &[])
.await
.map_err(|e| map_pg_error(&e, "PostgreSQL liveness probe failed"))?;
let probe_sql = table_probe_sql(&args.schema, &args.table);
client
.execute(probe_sql.as_str(), &[])
.await
.map_err(|e| map_pg_error(&e, "PostgreSQL table probe failed"))?;
pool.close();
Ok::<(), crate::TargetError>(())
})
.await
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("PostgreSQL connectivity probe timed out".to_string())))
}
pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs) -> Result<(), crate::TargetError> {
@@ -163,15 +202,21 @@ pub async fn check_kafka_broker_available(args: &crate::target::kafka::KafkaArgs
config = config.with_security(security);
}
match tokio::time::timeout(Duration::from_secs(5), async {
tokio::time::timeout(Duration::from_secs(5), async {
let _ = AsyncProducer::from_hosts_with_config(args.brokers.clone(), config)
.await
.map_err(|err| map_kafka_error(err, "Kafka broker check failed to create producer"))?;
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())),
}
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Kafka connection timed out".to_string())))
}
pub async fn check_redis_server_available(args: &crate::target::redis::RedisArgs) -> Result<(), crate::TargetError> {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let client = crate::target::redis::build_redis_client(args)?;
crate::target::redis::ping_redis_server(&client, args).await
})
.await
.unwrap_or_else(|_| Err(crate::TargetError::Timeout("Redis connection timed out".to_string())))
}
crates/targets/src/config/loader.rs
+32 -1
View File
@@ -38,10 +38,24 @@ fn is_sensitive_target_field(field_name: &str) -> bool {
|| field_name.contains("client_key")
|| field_name.contains("access_key")
|| field_name.contains("auth")
|| field_name.contains(rustfs_config::BASE_DSN_STRING)
}
fn redact_target_field_value(field_name: &str, value: &str) -> String {
if is_sensitive_target_field(field_name) && !value.is_empty() {
if value.is_empty() {
return value.to_string();
}
// MySQL DSN fields need partial redaction instead of full masking so the
// remaining connection details (host, port, database) remain visible in
// debug logs while the password is hidden.
if field_name == rustfs_config::BASE_DSN_STRING {
let trimmed = value.trim_start();
if trimmed.starts_with("postgres://") || trimmed.starts_with("postgresql://") {
return crate::target::postgres::redact_postgres_dsn(value);
}
return crate::target::mysql::redact_mysql_dsn(value);
}
if is_sensitive_target_field(field_name) {
return "***redacted***".to_string();
}
value.to_string()
@@ -283,6 +297,23 @@ mod tests {
assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000");
}
#[test]
fn redact_dsn_string_partial_redaction() {
let dsn = "rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events";
let redacted = redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, dsn);
assert_eq!(redacted, "rustfs:***@tcp(mysql.example.com:3306)/rustfs_events");
// empty dsn_string value
assert_eq!(redact_target_field_value(rustfs_config::MYSQL_DSN_STRING, ""), "");
}
#[test]
fn redact_postgres_dsn_string_partial_redaction() {
let dsn = "postgres://rustfs:secret123@pg.example.com:5432/rustfs_events?search_path=public";
let redacted = redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, dsn);
assert_eq!(redacted, "postgres://rustfs:***@pg.example.com:5432/rustfs_events?search_path=public");
assert_eq!(redact_target_field_value(rustfs_config::POSTGRES_DSN_STRING, ""), "");
}
#[test]
fn redacted_target_config_masks_sensitive_values_without_mutating_shape() {
let mut config = KVS::new();
crates/targets/src/config/mod.rs
+3 -2
View File
@@ -21,6 +21,7 @@ pub use loader::{
collect_target_configs_from_env,
};
pub use target_args::{
build_kafka_args, build_mqtt_args, build_nats_args, build_pulsar_args, build_webhook_args, validate_kafka_config,
validate_mqtt_config, validate_nats_config, validate_pulsar_config, validate_webhook_config,
build_kafka_args, build_mqtt_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, build_webhook_args, validate_kafka_config, validate_mqtt_config, validate_mysql_config,
validate_nats_config, validate_postgres_config, validate_pulsar_config, validate_redis_config, validate_webhook_config,
};
crates/targets/src/config/target_args.rs
+480 -10
View File
@@ -18,8 +18,11 @@ use crate::target::{
TargetType,
kafka::KafkaArgs,
mqtt::{MQTTArgs, MQTTTlsConfig, validate_mqtt_broker_url},
mysql::MySqlArgs,
nats::{NATSArgs, validate_nats_address},
postgres::{PostgresArgs, PostgresDsn, parse_postgres_format},
pulsar::{PulsarArgs, validate_pulsar_broker},
redis::{RedisArgs, RedisTlsConfig, validate_redis_url},
webhook::WebhookArgs,
};
use rumqttc::QoS;
@@ -27,12 +30,19 @@ use rustfs_config::{
DEFAULT_LIMIT, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT,
KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS,
MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, NATS_ADDRESS,
NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD,
PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION,
PULSAR_TOPIC, PULSAR_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING,
MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, RUSTFS_WEBHOOK_SKIP_TLS_VERIFY_DEFAULT, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT,
WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use rustfs_ecstore::config::KVS;
use std::path::Path;
@@ -286,6 +296,124 @@ pub fn validate_pulsar_config(config: &KVS, default_queue_dir: &str) -> Result<(
validate_pulsar_broker_config(&broker, config, default_queue_dir)
}
pub fn build_redis_args(
config: &KVS,
default_queue_dir: &str,
default_channel: &str,
target_type: TargetType,
) -> Result<RedisArgs, TargetError> {
let url = config
.lookup(REDIS_URL)
.ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?;
let url = parse_url(&url, "Redis URL")?;
let channel = config
.lookup(REDIS_CHANNEL)
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| default_channel.to_string());
Ok(RedisArgs {
enable: true,
url,
channel,
username: config.lookup(REDIS_USERNAME).filter(|value| !value.trim().is_empty()),
password: config.lookup(REDIS_PASSWORD).filter(|value| !value.trim().is_empty()),
tls: RedisTlsConfig::from_values(
config.lookup(REDIS_TLS_POLICY).as_deref(),
config.lookup(REDIS_TLS_CA).as_deref(),
config.lookup(REDIS_TLS_CLIENT_CERT).as_deref(),
config.lookup(REDIS_TLS_CLIENT_KEY).as_deref(),
config.lookup(REDIS_TLS_ALLOW_INSECURE).as_deref(),
)?,
keep_alive: config
.lookup(REDIS_KEEP_ALIVE_INTERVAL)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(15)),
queue_dir: config
.lookup(REDIS_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string()),
queue_limit: config
.lookup(REDIS_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
max_retry_attempts: config
.lookup(REDIS_MAX_RETRY_ATTEMPTS)
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or(3),
reconnect_retry_attempts: config
.lookup(REDIS_RECONNECT_RETRY_ATTEMPTS)
.and_then(|v| v.parse::<usize>().ok()),
min_retry_delay: config
.lookup(REDIS_MIN_RETRY_DELAY)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_millis),
max_retry_delay: config
.lookup(REDIS_MAX_RETRY_DELAY)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_millis),
connection_timeout: config
.lookup(REDIS_CONNECTION_TIMEOUT)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs),
response_timeout: config
.lookup(REDIS_RESPONSE_TIMEOUT)
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs),
pipeline_buffer_size: config
.lookup(REDIS_PIPELINE_BUFFER_SIZE)
.and_then(|v| v.parse::<usize>().ok()),
target_type,
})
}
pub fn build_postgres_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<PostgresArgs, TargetError> {
let dsn_string = config
.lookup(POSTGRES_DSN_STRING)
.ok_or_else(|| TargetError::Configuration("Missing PostgreSQL dsn_string".to_string()))?;
let table = config
.lookup(POSTGRES_TABLE)
.ok_or_else(|| TargetError::Configuration("Missing PostgreSQL table".to_string()))?;
let schema = PostgresDsn::parse(&dsn_string)?.schema;
let format = parse_postgres_format(config.lookup(POSTGRES_FORMAT).as_deref())?;
Ok(PostgresArgs {
enable: true,
dsn_string,
schema,
table,
format,
tls_required: parse_target_bool(config.lookup(POSTGRES_TLS_REQUIRED).as_deref()).unwrap_or(false),
tls_ca: config.lookup(POSTGRES_TLS_CA).unwrap_or_default(),
tls_client_cert: config.lookup(POSTGRES_TLS_CLIENT_CERT).unwrap_or_default(),
tls_client_key: config.lookup(POSTGRES_TLS_CLIENT_KEY).unwrap_or_default(),
queue_dir: config
.lookup(POSTGRES_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string()),
queue_limit: config
.lookup(POSTGRES_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
target_type,
})
}
pub fn validate_redis_config(config: &KVS, default_queue_dir: &str, default_channel: &str) -> Result<(), TargetError> {
let url = config
.lookup(REDIS_URL)
.ok_or_else(|| TargetError::Configuration("Missing Redis URL".to_string()))?;
let url = parse_url(&url, "Redis URL")?;
validate_redis_url(&url)?;
let args = build_redis_args(config, default_queue_dir, default_channel, TargetType::NotifyEvent)?;
args.validate()
}
pub fn validate_postgres_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
let args = build_postgres_args(config, default_queue_dir, TargetType::NotifyEvent)?;
args.validate()
}
pub fn build_kafka_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<KafkaArgs, TargetError> {
let brokers_raw = config
.lookup(KAFKA_BROKERS)
@@ -348,18 +476,81 @@ pub fn validate_kafka_config(config: &KVS, default_queue_dir: &str) -> Result<()
let queue_dir = config
.lookup(KAFKA_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string());
if !queue_dir.is_empty() && !std::path::Path::new(&queue_dir).is_absolute() {
if !queue_dir.is_empty() && !Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("Kafka queue directory must be an absolute path".to_string()));
}
Ok(())
}
/// Builds `MySqlArgs` from a KVS configuration.
///
/// Parses all MySQL target configuration keys, applies defaults for
/// missing optional values, and validates that all required fields
/// are present and well-formed.
pub fn build_mysql_args(config: &KVS, default_queue_dir: &str, target_type: TargetType) -> Result<MySqlArgs, TargetError> {
let dsn_string = config
.lookup(MYSQL_DSN_STRING)
.ok_or_else(|| TargetError::Configuration("Missing MySQL dsn_string".to_string()))?;
let table = config
.lookup(MYSQL_TABLE)
.ok_or_else(|| TargetError::Configuration("Missing MySQL table".to_string()))?;
let args = MySqlArgs {
enable: true,
dsn_string,
table,
format: config.lookup(MYSQL_FORMAT).unwrap_or_else(|| "access".to_string()),
tls_ca: config.lookup(MYSQL_TLS_CA).unwrap_or_default(),
tls_client_cert: config.lookup(MYSQL_TLS_CLIENT_CERT).unwrap_or_default(),
tls_client_key: config.lookup(MYSQL_TLS_CLIENT_KEY).unwrap_or_default(),
queue_dir: config
.lookup(MYSQL_QUEUE_DIR)
.unwrap_or_else(|| default_queue_dir.to_string()),
queue_limit: config
.lookup(MYSQL_QUEUE_LIMIT)
.and_then(|v| v.parse::<u64>().ok())
.unwrap_or(DEFAULT_LIMIT),
max_open_connections: config
.lookup(MYSQL_MAX_OPEN_CONNECTIONS)
.map(|value| {
value.trim().parse::<usize>().map_err(|_| {
TargetError::Configuration(format!("MySQL max_open_connections value '{}' is not a valid number", value))
})
})
.transpose()?
.unwrap_or(2),
target_type,
};
args.validate()?;
Ok(args)
}
/// Validates MySQL target configuration from a KVS without building args.
///
/// Performs the same checks as `build_mysql_args` but discards the result,
/// used for pre-validation before target creation.
pub fn validate_mysql_config(config: &KVS, default_queue_dir: &str) -> Result<(), TargetError> {
let _ = build_mysql_args(config, default_queue_dir, TargetType::NotifyEvent)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::{build_kafka_args, validate_kafka_config};
use crate::target::TargetType;
use rustfs_config::{KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC};
use super::{
build_kafka_args, build_mysql_args, build_postgres_args, build_redis_args, validate_kafka_config, validate_mysql_config,
validate_postgres_config, validate_redis_config,
};
use crate::target::{TargetType, postgres::PostgresFormat};
use rustfs_config::{
KAFKA_ACKS, KAFKA_BROKERS, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE,
MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, REDIS_CHANNEL,
REDIS_CONNECTION_TIMEOUT, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PIPELINE_BUFFER_SIZE,
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_URL,
};
use rustfs_ecstore::config::KVS;
fn kafka_base_config() -> KVS {
@@ -369,6 +560,16 @@ mod tests {
config
}
fn mysql_base_config() -> KVS {
let mut config = KVS::new();
config.insert(
MYSQL_DSN_STRING.to_string(),
"rustfs:password@tcp(127.0.0.1:3306)/rustfs_events".to_string(),
);
config.insert(MYSQL_TABLE.to_string(), "rustfs_events".to_string());
config
}
#[test]
fn build_kafka_args_accepts_all_ack_alias() {
let mut config = kafka_base_config();
@@ -395,4 +596,273 @@ mod tests {
let err = validate_kafka_config(&config, "").expect_err("invalid acks should fail");
assert!(err.to_string().contains("Kafka acks must be one of"));
}
#[test]
fn build_mysql_args_accepts_minimal_config() {
let args = build_mysql_args(&mysql_base_config(), "", TargetType::NotifyEvent).expect("valid mysql args");
assert!(args.enable);
assert_eq!(args.dsn_string, "rustfs:password@tcp(127.0.0.1:3306)/rustfs_events");
assert_eq!(args.table, "rustfs_events");
assert_eq!(args.format, "access");
assert_eq!(args.max_open_connections, 2);
assert_eq!(args.queue_limit, rustfs_config::DEFAULT_LIMIT);
}
#[test]
fn build_mysql_args_applies_defaults() {
let args = build_mysql_args(&mysql_base_config(), "/custom/queue", TargetType::NotifyEvent).expect("valid mysql args");
assert_eq!(args.queue_dir, "/custom/queue");
assert_eq!(args.queue_limit, 100000);
assert_eq!(args.max_open_connections, 2);
}
#[test]
fn build_mysql_args_rejects_missing_dsn() {
let mut config = KVS::new();
config.insert(MYSQL_TABLE.to_string(), "events".to_string());
let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("missing dsn should fail");
assert!(err.to_string().contains("dsn_string"));
}
#[test]
fn build_mysql_args_rejects_relative_queue_dir() {
let mut config = mysql_base_config();
config.insert(MYSQL_QUEUE_DIR.to_string(), "relative/path".to_string());
let err = build_mysql_args(&config, "", TargetType::NotifyEvent).expect_err("relative path should fail");
assert!(err.to_string().contains("absolute"));
}
#[test]
fn validate_mysql_config_rejects_invalid_max_open_connections() {
let mut config = mysql_base_config();
config.insert(MYSQL_MAX_OPEN_CONNECTIONS.to_string(), "not-a-number".to_string());
let err = validate_mysql_config(&config, "").expect_err("invalid max_open_connections should be rejected");
assert!(err.to_string().contains("max_open_connections"));
}
#[test]
fn validate_mysql_config_rejects_empty_dsn() {
let mut config = mysql_base_config();
config.insert(MYSQL_DSN_STRING.to_string(), "".to_string());
let err = validate_mysql_config(&config, "").expect_err("empty dsn should fail");
assert!(err.to_string().contains("empty"));
}
#[test]
fn validate_mysql_config_rejects_unpaired_tls_client_fields() {
let mut config = mysql_base_config();
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string());
let err = validate_mysql_config(&config, "").expect_err("unpaired mysql TLS client cert should fail");
assert!(err.to_string().contains("must be specified together"));
}
#[test]
fn validate_mysql_config_rejects_relative_tls_paths() {
let mut config = mysql_base_config();
config.insert(MYSQL_TLS_CA.to_string(), "ca.pem".to_string());
let err = validate_mysql_config(&config, "").expect_err("relative tls_ca should fail");
assert!(err.to_string().contains("tls_ca must be an absolute path"));
config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string());
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "client.pem".to_string());
config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "client.key".to_string());
let err = validate_mysql_config(&config, "").expect_err("relative tls client paths should fail");
assert!(err.to_string().contains("absolute path"));
}
#[test]
fn build_mysql_args_accepts_absolute_tls_paths() {
let mut config = mysql_base_config();
config.insert(MYSQL_TLS_CA.to_string(), "/etc/ssl/mysql/ca.pem".to_string());
config.insert(MYSQL_TLS_CLIENT_CERT.to_string(), "/etc/ssl/mysql/client.pem".to_string());
config.insert(MYSQL_TLS_CLIENT_KEY.to_string(), "/etc/ssl/mysql/client.key".to_string());
let args = build_mysql_args(&config, "", TargetType::NotifyEvent).expect("absolute mysql TLS paths should pass");
assert_eq!(args.tls_ca, "/etc/ssl/mysql/ca.pem");
assert_eq!(args.tls_client_cert, "/etc/ssl/mysql/client.pem");
assert_eq!(args.tls_client_key, "/etc/ssl/mysql/client.key");
}
fn redis_base_config() -> KVS {
let mut config = KVS::new();
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
config.insert(REDIS_CHANNEL.to_string(), "events".to_string());
config
}
fn postgres_base_config() -> KVS {
let mut config = KVS::new();
config.insert(
POSTGRES_DSN_STRING.to_string(),
"postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public".to_string(),
);
config.insert(POSTGRES_TABLE.to_string(), "rustfs_events_namespace".to_string());
config
}
#[test]
fn build_redis_args_keeps_manager_tuning_fields_none_when_unset() {
let config = redis_base_config();
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
assert_eq!(args.channel, "events");
assert_eq!(args.reconnect_retry_attempts, None);
assert_eq!(args.min_retry_delay, None);
assert_eq!(args.max_retry_delay, None);
assert_eq!(args.connection_timeout, None);
assert_eq!(args.response_timeout, None);
assert_eq!(args.pipeline_buffer_size, None);
}
#[test]
fn build_redis_args_uses_default_channel_when_missing() {
let mut config = KVS::new();
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
let args =
build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args");
assert_eq!(args.channel, "fallback-channel");
}
#[test]
fn build_redis_args_uses_default_channel_when_empty() {
let mut config = KVS::new();
config.insert(REDIS_URL.to_string(), "redis://127.0.0.1:6379/0".to_string());
config.insert(REDIS_CHANNEL.to_string(), " ".to_string());
let args =
build_redis_args(&config, "/tmp/queue", "fallback-channel", TargetType::NotifyEvent).expect("valid redis args");
assert_eq!(args.channel, "fallback-channel");
}
#[test]
fn build_redis_args_parses_optional_tuning_values_when_present() {
let mut config = redis_base_config();
config.insert(REDIS_RECONNECT_RETRY_ATTEMPTS.to_string(), "9".to_string());
config.insert(REDIS_MIN_RETRY_DELAY.to_string(), "250".to_string());
config.insert(REDIS_MAX_RETRY_DELAY.to_string(), "5000".to_string());
config.insert(REDIS_CONNECTION_TIMEOUT.to_string(), "7".to_string());
config.insert(REDIS_RESPONSE_TIMEOUT.to_string(), "11".to_string());
config.insert(REDIS_PIPELINE_BUFFER_SIZE.to_string(), "64".to_string());
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
assert_eq!(args.reconnect_retry_attempts, Some(9));
assert_eq!(args.min_retry_delay, Some(std::time::Duration::from_millis(250)));
assert_eq!(args.max_retry_delay, Some(std::time::Duration::from_millis(5000)));
assert_eq!(args.connection_timeout, Some(std::time::Duration::from_secs(7)));
assert_eq!(args.response_timeout, Some(std::time::Duration::from_secs(11)));
assert_eq!(args.pipeline_buffer_size, Some(64));
}
#[test]
fn build_redis_args_parses_tls_allow_insecure_when_present() {
let mut config = redis_base_config();
config.insert(REDIS_URL.to_string(), "rediss://127.0.0.1:6379/0".to_string());
config.insert(REDIS_TLS_ALLOW_INSECURE.to_string(), "on".to_string());
let args = build_redis_args(&config, "/tmp/queue", "default-channel", TargetType::NotifyEvent).expect("valid redis args");
assert!(args.tls.allow_insecure);
}
#[test]
fn validate_redis_config_rejects_missing_url() {
let config = KVS::new();
let err = validate_redis_config(&config, "/tmp/queue", "default-channel").expect_err("missing redis url should fail");
assert!(err.to_string().contains("Missing Redis URL"));
}
#[test]
fn build_postgres_args_accepts_minimal_config() {
let config = postgres_base_config();
let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args");
assert_eq!(
args.dsn_string,
"postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public"
);
assert_eq!(args.format, PostgresFormat::Namespace);
}
#[test]
fn build_postgres_args_parses_access_format() {
let mut config = postgres_base_config();
config.insert(POSTGRES_FORMAT.to_string(), "access".to_string());
let args = build_postgres_args(&config, "", TargetType::NotifyEvent).expect("valid postgres args");
assert_eq!(args.format, PostgresFormat::Access);
}
#[test]
fn validate_postgres_config_rejects_missing_dsn_string() {
let mut config = postgres_base_config();
config.0.retain(|kv| kv.key != POSTGRES_DSN_STRING);
let err = validate_postgres_config(&config, "").expect_err("missing dsn_string should fail");
assert!(err.to_string().contains("Missing PostgreSQL dsn_string"));
}
#[test]
fn validate_postgres_config_rejects_empty_dsn_string() {
let mut config = postgres_base_config();
config.insert(POSTGRES_DSN_STRING.to_string(), "".to_string());
let err = validate_postgres_config(&config, "").expect_err("empty dsn_string should fail");
assert!(err.to_string().contains("dsn_string cannot be empty"));
}
#[test]
fn validate_postgres_config_rejects_missing_table() {
let mut config = postgres_base_config();
config.0.retain(|kv| kv.key != POSTGRES_TABLE);
let err = validate_postgres_config(&config, "").expect_err("missing table should fail");
assert!(err.to_string().contains("Missing PostgreSQL table"));
}
#[test]
fn validate_postgres_config_rejects_invalid_dsn() {
let mut config = postgres_base_config();
config.insert(POSTGRES_DSN_STRING.to_string(), "postgres://".to_string());
let err = validate_postgres_config(&config, "").expect_err("invalid dsn should fail");
assert!(err.to_string().contains("invalid PostgreSQL dsn_string"));
}
#[test]
fn validate_postgres_config_rejects_relative_queue_dir() {
let mut config = postgres_base_config();
config.insert(POSTGRES_QUEUE_DIR.to_string(), "relative/path".to_string());
let err = validate_postgres_config(&config, "").expect_err("relative queue_dir should fail");
assert!(err.to_string().contains("absolute path"));
}
#[test]
fn validate_postgres_config_rejects_mtls_without_key() {
let mut config = postgres_base_config();
config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "/etc/ssl/cert.pem".to_string());
let err = validate_postgres_config(&config, "").expect_err("missing key should fail");
assert!(err.to_string().contains("must be specified together"));
}
#[test]
fn validate_postgres_config_rejects_relative_tls_ca() {
let mut config = postgres_base_config();
config.insert(POSTGRES_TLS_CA.to_string(), "relative/ca.pem".to_string());
let err = validate_postgres_config(&config, "").expect_err("relative tls_ca should fail");
assert!(err.to_string().contains("must be an absolute path"));
}
#[test]
fn validate_postgres_config_rejects_relative_tls_client_cert() {
let mut config = postgres_base_config();
config.insert(POSTGRES_TLS_CLIENT_CERT.to_string(), "relative/client.pem".to_string());
config.insert(POSTGRES_TLS_CLIENT_KEY.to_string(), "/etc/ssl/client.key".to_string());
let err = validate_postgres_config(&config, "").expect_err("relative tls_client_cert should fail");
assert!(err.to_string().contains("must be an absolute path"));
}
}
crates/targets/src/lib.rs
+1 -1
View File
@@ -22,7 +22,7 @@ pub mod target;
pub use check::{
check_kafka_broker_available, check_mqtt_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_pulsar_broker_available,
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
};
pub use error::{StoreError, TargetError};
pub use rustfs_s3_common::EventName;
crates/targets/src/target/kafka.rs
+2 -2
View File
@@ -19,7 +19,7 @@ use crate::{
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
@@ -136,7 +136,7 @@ where
let queue_store = if !args.queue_dir.is_empty() {
let queue_dir =
PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Kafka.as_str(), target_id.id));
PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Kafka.as_str(), &target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => AUDIT_STORE_EXTENSION,
crates/targets/src/target/mod.rs
+83 -9
View File
@@ -14,7 +14,7 @@
use crate::arn::TargetID;
use crate::store::{Key, Store};
use crate::{StoreError, TargetError};
use crate::{StoreError, TargetError, TargetLog};
use async_trait::async_trait;
use rustfs_s3_common::EventName;
use serde::de::DeserializeOwned;
@@ -27,8 +27,11 @@ use tracing::warn;
pub mod kafka;
pub mod mqtt;
pub mod mysql;
pub mod nats;
pub mod postgres;
pub mod pulsar;
pub mod redis;
pub mod webhook;
/// A read-only snapshot of delivery counters for a target.
@@ -289,8 +292,11 @@ pub enum ChannelTargetType {
Webhook,
Kafka,
Mqtt,
MySql,
Nats,
Postgres,
Pulsar,
Redis,
}
impl ChannelTargetType {
@@ -299,8 +305,11 @@ impl ChannelTargetType {
ChannelTargetType::Webhook => "webhook",
ChannelTargetType::Kafka => "kafka",
ChannelTargetType::Mqtt => "mqtt",
ChannelTargetType::MySql => "mysql",
ChannelTargetType::Nats => "nats",
ChannelTargetType::Postgres => "postgres",
ChannelTargetType::Pulsar => "pulsar",
ChannelTargetType::Redis => "redis",
}
}
}
@@ -311,20 +320,15 @@ impl std::fmt::Display for ChannelTargetType {
ChannelTargetType::Webhook => write!(f, "webhook"),
ChannelTargetType::Kafka => write!(f, "kafka"),
ChannelTargetType::Mqtt => write!(f, "mqtt"),
ChannelTargetType::MySql => write!(f, "mysql"),
ChannelTargetType::Nats => write!(f, "nats"),
ChannelTargetType::Postgres => write!(f, "postgres"),
ChannelTargetType::Pulsar => write!(f, "pulsar"),
ChannelTargetType::Redis => write!(f, "redis"),
}
}
}
pub fn parse_bool(value: &str) -> Result<bool, TargetError> {
match value.to_lowercase().as_str() {
"true" | "on" | "yes" | "1" => Ok(true),
"false" | "off" | "no" | "0" => Ok(false),
_ => Err(TargetError::ParseError(format!("Unable to parse boolean: {value}"))),
}
}
/// `TargetType` enum represents the type of target in the notification system.
#[derive(Debug, Clone)]
pub enum TargetType {
@@ -350,6 +354,23 @@ impl std::fmt::Display for TargetType {
}
}
pub(crate) fn sanitize_queue_dir_component(component: &str) -> String {
let mut sanitized = String::with_capacity(component.len());
for ch in component.chars() {
if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') {
sanitized.push(ch);
} else {
sanitized.push('_');
}
}
if sanitized.is_empty() { "_".to_string() } else { sanitized }
}
pub(crate) fn queue_store_subdir_name(target_type: &str, target_id: &str) -> String {
format!("rustfs-{target_type}-{}", sanitize_queue_dir_component(target_id))
}
/// Decodes a form-urlencoded object name to its original form.
///
/// This function properly handles form-urlencoded strings where spaces are
@@ -377,6 +398,31 @@ pub fn decode_object_name(encoded: &str) -> Result<String, TargetError> {
.map_err(|e| TargetError::Encoding(format!("Failed to decode object key: {e}")))
}
pub(crate) fn build_queued_payload<E>(event: &EntityTarget<E>) -> Result<QueuedPayload, TargetError>
where
E: Send + Sync + 'static + Clone + Serialize + DeserializeOwned,
{
let object_name = decode_object_name(&event.object_name)?;
let key = format!("{}/{}", event.bucket_name, object_name);
let log = TargetLog {
event_name: event.event_name,
key,
records: vec![event.data.clone()],
};
let body = serde_json::to_vec(&log).map_err(|err| TargetError::Serialization(format!("Failed to serialize event: {err}")))?;
let meta = QueuedPayloadMeta::new(
event.event_name,
event.bucket_name.clone(),
event.object_name.clone(),
"application/json",
body.len(),
);
Ok(QueuedPayload::new(meta, body))
}
pub(crate) fn delete_stored_payload(
store: &(dyn Store<QueuedPayload, Error = StoreError, Key = Key> + Send + Sync),
key: &Key,
@@ -412,9 +458,37 @@ mod tests {
assert_eq!(decoded.body, br#"{"ok":true}"#);
}
#[test]
fn build_queued_payload_uses_event_data_shape() {
let event = EntityTarget {
object_name: "greeting+file+%282%29.csv".to_string(),
bucket_name: "bucket-a".to_string(),
event_name: EventName::ObjectCreatedPut,
data: "payload-data".to_string(),
};
let payload = build_queued_payload(&event).unwrap();
let value: serde_json::Value = serde_json::from_slice(&payload.body).unwrap();
assert_eq!(value["Key"], "bucket-a/greeting file (2).csv");
assert_eq!(value["Records"][0], "payload-data");
}
#[test]
fn queued_payload_decode_rejects_invalid_magic() {
let err = QueuedPayload::decode(b"bad-payload").unwrap_err();
assert!(err.to_string().contains("magic") || err.to_string().contains("short"));
}
#[test]
fn sanitize_queue_dir_component_replaces_non_path_safe_characters() {
let sanitized = sanitize_queue_dir_component("tenant:alpha/beta\\gamma?*");
assert_eq!(sanitized, "tenant_alpha_beta_gamma__");
}
#[test]
fn queue_store_subdir_name_sanitizes_target_id() {
let dir = queue_store_subdir_name("redis", "tenant:alpha");
assert_eq!(dir, "rustfs-redis-tenant_alpha");
}
}
crates/targets/src/target/mqtt.rs
+6 -6
View File
@@ -19,7 +19,7 @@ use crate::{
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
@@ -465,7 +465,7 @@ impl MQTTArgs {
}
if !self.queue_dir.is_empty() {
let path = std::path::Path::new(&self.queue_dir);
let path = Path::new(&self.queue_dir);
if !path.is_absolute() {
return Err(TargetError::Configuration("mqtt queue_dir path should be absolute".to_string()));
}
@@ -512,7 +512,7 @@ where
let target_id = TargetID::new(id, ChannelTargetType::Mqtt.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let base_path = PathBuf::from(&args.queue_dir);
let unique_dir_name = format!("rustfs-{}-{}", ChannelTargetType::Mqtt.as_str(), target_id.id).replace(":", "_");
let unique_dir_name = queue_store_subdir_name(ChannelTargetType::Mqtt.as_str(), &target_id.id);
// Ensure the directory name is valid for filesystem
let specific_queue_path = base_path.join(unique_dir_name);
debug!(target_id = %target_id, path = %specific_queue_path.display(), "Initializing queue store for MQTT target");
@@ -582,7 +582,7 @@ where
Some(MAX_MQTT_PACKET_SIZE_BYTES),
)?;
let (new_client, eventloop) = AsyncClient::new(mqtt_options, 10);
let (new_client, eventloop) = AsyncClient::builder(mqtt_options).capacity(10).build();
if let Err(e) = new_client.subscribe(&args_clone.topic, args_clone.qos).await {
error!(target_id = %target_id_clone, error = %e, "Failed to subscribe to MQTT topic during init");
@@ -828,7 +828,7 @@ fn is_fatal_mqtt_error(err: &ConnectionError) -> bool {
match state_err {
// If StateError is caused by deserialization issues, check the underlying MqttBytesError
rumqttc::StateError::Deserialization(mqtt_bytes_err) => { // The type of mqtt_bytes_err is &rumqttc::mqttbytes::Error
matches!(
matches!(
mqtt_bytes_err,
MqttBytesError::InvalidProtocol // Invalid agreement
| MqttBytesError::InvalidProtocolLevel(_) // Invalid protocol level
@@ -922,7 +922,7 @@ where
Err(e) => {
error!(target_id = %self.id, error = %e, "Failed to save event to store");
self.delivery_counters.record_final_failure();
return Err(TargetError::Storage(format!("Failed to save event to store: {e}")));
Err(TargetError::Storage(format!("Failed to save event to store: {e}")))
}
}
} else {
crates/targets/src/target/mysql.rs
+1226
View File
File diff suppressed because it is too large Load Diff
crates/targets/src/target/nats.rs
+2 -2
View File
@@ -19,7 +19,7 @@ use crate::{
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
@@ -197,7 +197,7 @@ where
let target_id = TargetID::new(id, ChannelTargetType::Nats.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let base_path = PathBuf::from(&args.queue_dir);
let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Nats.as_str(), target_id.id));
let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Nats.as_str(), &target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
crates/targets/src/target/postgres.rs
+1095
View File
File diff suppressed because it is too large Load Diff
crates/targets/src/target/pulsar.rs
+2 -2
View File
@@ -19,7 +19,7 @@ use crate::{
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
@@ -188,7 +188,7 @@ where
let target_id = TargetID::new(id, ChannelTargetType::Pulsar.as_str().to_string());
let queue_store = if !args.queue_dir.is_empty() {
let base_path = PathBuf::from(&args.queue_dir);
let specific_queue_path = base_path.join(format!("rustfs-{}-{}", ChannelTargetType::Pulsar.as_str(), target_id.id));
let specific_queue_path = base_path.join(queue_store_subdir_name(ChannelTargetType::Pulsar.as_str(), &target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => rustfs_config::audit::AUDIT_STORE_EXTENSION,
TargetType::NotifyEvent => rustfs_config::notify::NOTIFY_STORE_EXTENSION,
crates/targets/src/target/redis.rs
+1205
View File
File diff suppressed because it is too large Load Diff
crates/targets/src/target/webhook.rs
+2 -2
View File
@@ -19,7 +19,7 @@ use crate::{
store::{Key, QueueStore, Store},
target::{
ChannelTargetType, EntityTarget, QueuedPayload, QueuedPayloadMeta, TargetDeliveryCounters, TargetDeliverySnapshot,
TargetType,
TargetType, queue_store_subdir_name,
},
};
use async_trait::async_trait;
@@ -147,7 +147,7 @@ where
// Build storage
let queue_store = if !args.queue_dir.is_empty() {
let queue_dir =
PathBuf::from(&args.queue_dir).join(format!("rustfs-{}-{}", ChannelTargetType::Webhook.as_str(), target_id.id));
PathBuf::from(&args.queue_dir).join(queue_store_subdir_name(ChannelTargetType::Webhook.as_str(), &target_id.id));
let extension = match args.target_type {
TargetType::AuditLog => AUDIT_STORE_EXTENSION,
crates/targets/tests/mysql_integration.rs
+252
View File
@@ -0,0 +1,252 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! MySQL notification target integration tests.
//!
//! These tests require a running MySQL 8.0+ or TiDB 8.5+ instance.
//! Set `RUSTFS_MYSQL_TEST_DSN` to enable them:
//!
//! ```bash
//! RUSTFS_MYSQL_TEST_DSN="user:pass@tcp(127.0.0.1:3306)/testdb" \
//! cargo test -p rustfs-targets -- --ignored
//! ```
use mysql_async::{Opts, OptsBuilder, Pool, SslOpts, prelude::Queryable};
use rustfs_targets::{Target, TargetError, target::mysql::*, target::*};
use std::env;
use std::sync::Arc;
use tempfile::TempDir;
use uuid::Uuid;
fn test_dsn() -> String {
env::var("RUSTFS_MYSQL_TEST_DSN").expect("RUSTFS_MYSQL_TEST_DSN must be set")
}
fn table_name(prefix: &str) -> String {
let suffix = Uuid::new_v4().simple().to_string();
format!("{prefix}_{}", &suffix[..16])
}
fn make_args(dsn: &str, table: &str, queue_dir: &str) -> MySqlArgs {
MySqlArgs {
enable: true,
dsn_string: dsn.to_string(),
table: table.to_string(),
format: "access".to_string(),
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
queue_dir: queue_dir.to_string(),
queue_limit: 100,
max_open_connections: 2,
target_type: TargetType::NotifyEvent,
}
}
fn make_entity(bucket: &str, object: &str, event_name: rustfs_s3_common::EventName) -> EntityTarget<serde_json::Value> {
EntityTarget {
object_name: object.to_string(),
bucket_name: bucket.to_string(),
event_name,
data: serde_json::json!({"eventTime": "2026-05-03T10:00:00Z"}),
}
}
fn build_test_pool(dsn_string: &str) -> Pool {
let parsed = MySqlDsn::parse(dsn_string).expect("parse test DSN");
let mut builder = OptsBuilder::default()
.user(Some(parsed.user))
.pass(Some(parsed.password))
.ip_or_hostname(parsed.host)
.tcp_port(parsed.port)
.db_name(Some(parsed.database));
if parsed.tls {
if rustls::crypto::CryptoProvider::get_default().is_none() {
rustls::crypto::aws_lc_rs::default_provider().install_default().ok();
}
builder = builder.ssl_opts(Some(SslOpts::default()));
}
Pool::new(Opts::from(builder))
}
async fn drop_table(dsn: &str, table: &str) {
let pool = build_test_pool(dsn);
let mut conn = pool.get_conn().await.expect("get conn for drop table");
let _ = conn
.query_drop(format!("DROP TABLE IF EXISTS `{}`", table.replace('.', "`.`")))
.await;
}
#[ignore]
#[tokio::test]
async fn direct_write_and_read() {
let dsn = test_dsn();
let table = table_name("test_direct");
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("direct".to_string(), make_args(&dsn, &table, "")).expect("create target");
target.init().await.expect("init");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save");
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
let rows: Vec<mysql_async::Row> = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select");
assert_eq!(rows.len(), 1);
let data: String = mysql_async::from_value(rows[0].get(1).unwrap());
assert!(data.contains("mybucket"), "event_data should contain bucket name, got: {data}");
drop_table(&dsn, &table).await;
}
#[ignore]
#[tokio::test]
async fn delete_appends_row_does_not_remove_old() {
let dsn = test_dsn();
let table = table_name("test_delete");
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("delete".to_string(), make_args(&dsn, &table, "")).expect("create target");
target.init().await.expect("init");
let put = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
target.save(Arc::new(put)).await.expect("save put");
let delete = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectRemovedDelete);
target.save(Arc::new(delete)).await.expect("save delete");
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
let rows: Vec<mysql_async::Row> = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select");
assert_eq!(rows.len(), 2, "both PUT and DELETE should produce rows");
drop_table(&dsn, &table).await;
}
#[ignore]
#[tokio::test]
async fn queue_store_saves_entry_and_replays() {
let dsn = test_dsn();
let table = table_name("test_queue");
let tmpdir = TempDir::new().expect("temp dir");
let queue_dir = tmpdir.path().to_str().expect("valid path");
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("queue".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save to queue");
let store = target.store().expect("store should exist");
assert_eq!(store.len(), 1, "one entry should be in queue");
// Init creates the table; no rows should exist yet
target.init().await.expect("init");
{
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
let rows: Vec<mysql_async::Row> = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select");
assert_eq!(rows.len(), 0, "no row should exist before replay");
}
for key in store.list() {
target.send_from_store(key).await.expect("replay should succeed");
}
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
let rows: Vec<mysql_async::Row> = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select");
assert_eq!(rows.len(), 1, "one row should exist after replay");
assert_eq!(store.len(), 0, "queue should be empty after replay");
drop_table(&dsn, &table).await;
}
#[ignore]
#[tokio::test]
async fn duplicate_replay_produces_duplicate_rows() {
let dsn = test_dsn();
let table = table_name("test_dupe");
let tmpdir = TempDir::new().expect("temp dir");
let queue_dir = tmpdir.path().to_str().expect("valid path");
let target: MySqlTarget<serde_json::Value> =
MySqlTarget::new("dupe".to_string(), make_args(&dsn, &table, queue_dir)).expect("create target");
let entity = make_entity("mybucket", "obj.txt", rustfs_s3_common::EventName::ObjectCreatedPut);
target.save(Arc::new(entity)).await.expect("save to queue");
target.init().await.expect("init");
let store = target.store().expect("store should exist");
let keys: Vec<_> = store.list();
for key in &keys {
let raw = store.get_raw(key).expect("get raw");
let queued = QueuedPayload::decode(&raw).expect("decode");
// Replay twice: duplicate rows are expected (at-least-once)
for _ in 0..2 {
target
.send_raw_from_store(key.clone(), queued.body.clone(), queued.meta.clone())
.await
.expect("replay");
}
let _ = store.del(key);
}
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
let rows: Vec<mysql_async::Row> = conn.query(format!("SELECT * FROM `{table}`")).await.expect("select");
assert_eq!(rows.len(), 2, "duplicate replay should produce 2 rows");
drop_table(&dsn, &table).await;
}
#[ignore]
#[tokio::test]
async fn incompatible_schema_init_fails() {
let dsn = test_dsn();
let table = table_name("test_schema");
{
let pool = build_test_pool(&dsn);
let mut conn = pool.get_conn().await.expect("get conn");
conn.query_drop(format!("CREATE TABLE IF NOT EXISTS `{table}` (wrong_col INT NOT NULL)"))
.await
.expect("create incompatible table");
}
let target = MySqlTarget::<serde_json::Value>::new("schema".to_string(), make_args(&dsn, &table, "")).expect("create target");
let result = target.init().await;
match result {
Err(TargetError::Initialization(msg)) => {
assert!(
msg.contains("event_time") || msg.contains("event_data"),
"error should mention missing columns, got: {msg}"
);
}
other => panic!("expected Initialization error, got {:?}", other),
}
drop_table(&dsn, &table).await;
}
crates/targets/tests/postgres_integration.rs
+305
View File
@@ -0,0 +1,305 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Integration tests for the PostgreSQL notification target.
//!
//! All tests in this file are `#[ignore]` because they require a running
//! PostgreSQL server. CI does not run them by default. To run locally:
//!
//! ```bash
//! docker run -d --name rustfs-pg -p 5432:5432 \
//! -e POSTGRES_PASSWORD=rustfs -e POSTGRES_DB=rustfs_events postgres:16
//! export RUSTFS_TEST_PG_DSN="postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public"
//! cargo test -p rustfs-targets --test postgres_integration -- --ignored
//! ```
//!
//! Connection parameters can be overridden via environment variable:
//! `RUSTFS_TEST_PG_DSN`.
use rustfs_s3_common::EventName;
use rustfs_targets::Target;
use rustfs_targets::check_postgres_server_available;
use rustfs_targets::target::EntityTarget;
use rustfs_targets::target::TargetType;
use rustfs_targets::target::postgres::{PostgresArgs, PostgresDsn, PostgresFormat, PostgresTarget};
use serde_json::Value;
use std::sync::Arc;
use tokio_postgres::NoTls;
use url::Url;
use uuid::Uuid;
fn env_or(key: &str, default: &str) -> String {
std::env::var(key).unwrap_or_else(|_| default.to_string())
}
fn test_args(table: &str, format: PostgresFormat) -> PostgresArgs {
let dsn = env_or(
"RUSTFS_TEST_PG_DSN",
"postgres://postgres:rustfs@localhost:5432/rustfs_events?search_path=public",
);
let schema = PostgresDsn::parse(&dsn)
.expect("RUSTFS_TEST_PG_DSN must be a valid PostgreSQL DSN")
.schema;
PostgresArgs {
enable: true,
dsn_string: dsn,
schema,
table: table.to_string(),
format,
tls_required: false,
tls_ca: String::new(),
tls_client_cert: String::new(),
tls_client_key: String::new(),
queue_dir: String::new(),
queue_limit: 100_000,
target_type: TargetType::NotifyEvent,
}
}
fn with_search_path(dsn: &str, schema: &str) -> String {
let mut url = Url::parse(dsn).expect("RUSTFS_TEST_PG_DSN must be a valid PostgreSQL DSN URL");
url.query_pairs_mut().clear().append_pair("search_path", schema);
url.to_string()
}
async fn raw_client(args: &PostgresArgs) -> tokio_postgres::Client {
let (client, connection) = tokio_postgres::connect(&args.dsn_string, NoTls)
.await
.expect("connect to postgres test server");
tokio::spawn(async move {
let _ = connection.await;
});
client
}
fn unique_table(prefix: &str) -> String {
let suffix = Uuid::new_v4().simple().to_string();
format!("{prefix}_{}", &suffix[..16])
}
fn entity_for(bucket: &str, object: &str) -> Arc<EntityTarget<serde_json::Value>> {
Arc::new(EntityTarget {
bucket_name: bucket.to_string(),
object_name: object.to_string(),
event_name: EventName::ObjectCreatedPut,
data: serde_json::json!({"bucket": bucket, "object": object}),
})
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_check_postgres_server_available_with_existing_table() {
let args = test_args("pg_class", PostgresFormat::Namespace);
// Use a real existing table: pg_class always exists.
let mut args = args;
args.dsn_string = with_search_path(&args.dsn_string, "pg_catalog");
args.schema = "pg_catalog".to_string();
args.table = "pg_class".to_string();
check_postgres_server_available(&args)
.await
.expect("connectivity probe should succeed against pg_catalog.pg_class");
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_check_postgres_server_available_missing_table_fails() {
let args = test_args("does_not_exist_table_xyz", PostgresFormat::Namespace);
let result = check_postgres_server_available(&args).await;
assert!(result.is_err(), "missing table should fail the probe");
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_namespace_format_upsert_replaces_value() {
let table = unique_table("rustfs_test_namespace");
let args = test_args(&table, PostgresFormat::Namespace);
// Setup: create the namespace table.
let setup = raw_client(&args).await;
setup
.execute(
&format!("CREATE TABLE \"{}\" (key VARCHAR PRIMARY KEY, value JSONB NOT NULL)", table),
&[],
)
.await
.expect("create namespace table");
// Build target and deliver two events for the same key.
let target: PostgresTarget<serde_json::Value> =
PostgresTarget::new("test_namespace".to_string(), args.clone()).expect("construct target");
target
.save(entity_for("bucket1", "obj-A"))
.await
.expect("first save should succeed");
target
.save(entity_for("bucket1", "obj-A"))
.await
.expect("second save should succeed");
// Verify only one row exists for the key (UPSERT collapsed).
let row = setup
.query_one(&format!("SELECT count(*)::bigint FROM \"{}\" WHERE key = $1", table), &[&"bucket1/obj-A"])
.await
.expect("count rows");
let count: i64 = row.get(0);
assert_eq!(count, 1, "namespace format should keep only one row per key");
// Cleanup.
setup
.execute(&format!("DROP TABLE \"{}\"", table), &[])
.await
.expect("drop namespace table");
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_access_format_appends_distinct_events() {
let table = unique_table("rustfs_test_access");
let args = test_args(&table, PostgresFormat::Access);
let setup = raw_client(&args).await;
setup
.execute(
&format!(
"CREATE TABLE \"{}\" (\
event_id TEXT PRIMARY KEY, \
event_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), \
event_name TEXT NOT NULL, \
key TEXT NOT NULL, \
value JSONB NOT NULL, \
queued_at_ms BIGINT NOT NULL\
)",
table
),
&[],
)
.await
.expect("create access table");
let target: PostgresTarget<serde_json::Value> =
PostgresTarget::new("test_access".to_string(), args.clone()).expect("construct target");
// Two distinct events for different objects produce two rows.
target.save(entity_for("bucket1", "obj-A")).await.expect("save A");
target.save(entity_for("bucket1", "obj-B")).await.expect("save B");
let row = setup
.query_one(&format!("SELECT count(*)::bigint FROM \"{}\"", table), &[])
.await
.expect("count rows");
let count: i64 = row.get(0);
assert_eq!(count, 2, "access format should append two distinct rows");
setup
.execute(&format!("DROP TABLE \"{}\"", table), &[])
.await
.expect("drop access table");
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_access_format_replay_is_idempotent() {
let table = unique_table("rustfs_test_access_replay");
let args = test_args(&table, PostgresFormat::Access);
let setup = raw_client(&args).await;
setup
.execute(
&format!(
"CREATE TABLE \"{}\" (\
event_id TEXT PRIMARY KEY, \
event_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), \
event_name TEXT NOT NULL, \
key TEXT NOT NULL, \
value JSONB NOT NULL, \
queued_at_ms BIGINT NOT NULL\
)",
table
),
&[],
)
.await
.expect("create access table");
// Insert the same row twice with the same event_id via direct SQL — this
// simulates store replay where send_raw_from_store is called twice with
// the same Key.
let event_id = Uuid::new_v4().to_string();
let payload: Value = serde_json::json!({"EventName": "s3:ObjectCreated:Put", "Key": "bucket1/obj-A", "Records": []});
let queued_at_ms: i64 = 1234567890;
let sql = format!(
"INSERT INTO \"{}\" (event_id, event_name, key, value, queued_at_ms) \
VALUES ($1, $2, $3, $4::jsonb, $5) ON CONFLICT (event_id) DO NOTHING",
table
);
setup
.execute(&sql, &[&event_id, &"s3:ObjectCreated:Put", &"bucket1/obj-A", &payload, &queued_at_ms])
.await
.expect("first insert");
setup
.execute(&sql, &[&event_id, &"s3:ObjectCreated:Put", &"bucket1/obj-A", &payload, &queued_at_ms])
.await
.expect("second insert (should be silent skip)");
let row = setup
.query_one(&format!("SELECT count(*)::bigint FROM \"{}\"", table), &[])
.await
.expect("count rows");
let count: i64 = row.get(0);
assert_eq!(count, 1, "ON CONFLICT (event_id) DO NOTHING should make replay idempotent");
setup
.execute(&format!("DROP TABLE \"{}\"", table), &[])
.await
.expect("drop access table");
}
#[tokio::test]
#[ignore = "requires running PostgreSQL server"]
async fn test_init_succeeds_against_existing_table() {
let table = unique_table("rustfs_test_init");
let args = test_args(&table, PostgresFormat::Namespace);
let setup = raw_client(&args).await;
setup
.execute(
&format!("CREATE TABLE \"{}\" (key VARCHAR PRIMARY KEY, value JSONB NOT NULL)", table),
&[],
)
.await
.expect("create table");
let target: PostgresTarget<serde_json::Value> =
PostgresTarget::new("test_init".to_string(), args.clone()).expect("construct target");
target.init().await.expect("init should succeed against existing table");
target.close().await.expect("close should succeed");
setup
.execute(&format!("DROP TABLE \"{}\"", table), &[])
.await
.expect("drop table");
}
#[tokio::test]
async fn test_invalid_identifier_rejected_at_construction() {
// No #[ignore] — pure validation, no DB needed.
let args = test_args("malicious; DROP TABLE users", PostgresFormat::Namespace);
match PostgresTarget::<serde_json::Value>::new("bad_id".to_string(), args) {
Ok(_) => panic!("malicious table identifier must fail at construction"),
Err(e) => assert!(e.to_string().contains("table"), "unexpected error: {e}"),
}
}
rustfs/src/admin/handlers/audit.rs
+29 -18
View File
@@ -16,7 +16,7 @@ use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
collect_validated_key_values as shared_collect_validated_key_values,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
@@ -28,18 +28,20 @@ use crate::server::{
ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store,
};
use futures::stream::{FuturesUnordered, StreamExt};
use http::{HeaderMap, StatusCode};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS,
AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -93,7 +95,7 @@ struct AuditEndpointsResponse {
audit_endpoints: Vec<AuditEndpoint>,
}
fn audit_target_specs() -> [AdminTargetSpec; 5] {
fn audit_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: AUDIT_WEBHOOK_SUB_SYS,
@@ -113,18 +115,36 @@ fn audit_target_specs() -> [AdminTargetSpec; 5] {
valid_keys: AUDIT_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: AUDIT_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: AUDIT_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: AUDIT_NATS_SUB_SYS,
service: "nats",
valid_keys: AUDIT_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: AUDIT_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_PULSAR_SUB_SYS,
service: "pulsar",
valid_keys: AUDIT_PULSAR_KEYS,
validator: AdminTargetValidator::Pulsar(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_REDIS_SUB_SYS,
service: "redis",
valid_keys: AUDIT_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL),
},
]
}
@@ -138,15 +158,6 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
}
fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
if let Some(v) = request_id {
header.insert("x-request-id", v.clone());
}
S3Response::with_headers((status, body), header)
}
fn has_any_audit_targets(config: &Config) -> bool {
for spec in audit_target_specs() {
let Some(targets) = config.0.get(spec.subsystem) else {
@@ -334,7 +345,7 @@ impl Operation for AuditTargetConfig {
})
.await?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
@@ -375,7 +386,7 @@ impl Operation for ListAuditTargets {
let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints })
.map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?;
Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
}
}
@@ -411,7 +422,7 @@ impl Operation for RemoveAuditTarget {
})
.await?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
rustfs/src/admin/handlers/event.rs
+30 -19
View File
@@ -16,7 +16,7 @@ use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
collect_validated_key_values as shared_collect_validated_key_values,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
@@ -29,17 +29,19 @@ use crate::server::{
refresh_persisted_module_switches_from_store,
};
use futures::stream::{FuturesUnordered, StreamExt};
use http::{HeaderMap, StatusCode};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX,
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -99,7 +101,7 @@ struct NotificationEndpointsResponse {
notification_endpoints: Vec<NotificationEndpoint>,
}
fn notification_target_specs() -> [AdminTargetSpec; 5] {
fn notification_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: NOTIFY_WEBHOOK_SUB_SYS,
@@ -119,12 +121,30 @@ fn notification_target_specs() -> [AdminTargetSpec; 5] {
valid_keys: NOTIFY_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: NOTIFY_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: NOTIFY_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: NOTIFY_NATS_SUB_SYS,
service: "nats",
valid_keys: NOTIFY_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: NOTIFY_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_REDIS_SUB_SYS,
service: "redis",
valid_keys: NOTIFY_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL),
},
AdminTargetSpec {
subsystem: NOTIFY_PULSAR_SUB_SYS,
service: "pulsar",
@@ -150,15 +170,6 @@ fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>>
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
}
fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
if let Some(v) = request_id {
header.insert("x-request-id", v.clone());
}
S3Response::with_headers((status, body), header)
}
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
&notification_target_specs(),
@@ -258,7 +269,7 @@ impl Operation for NotificationTarget {
.await
.map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
@@ -297,7 +308,7 @@ impl Operation for ListNotificationTargets {
let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
.map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
}
}
@@ -347,7 +358,7 @@ impl Operation for ListTargetsArns {
let data = serde_json::to_vec(&data_target_arn_list)
.map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;
Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
}
}
@@ -374,7 +385,7 @@ impl Operation for RemoveNotificationTarget {
.await
.map_err(|e| s3_error!(InternalError, "failed to remove target config: {}", e))?;
Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
}
}
rustfs/src/admin/handlers/target_descriptor.rs
+76 -4
View File
@@ -13,19 +13,23 @@
// limitations under the License.
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, HeaderValue, StatusCode};
use rustfs_config::{
ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST,
MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
};
use rustfs_ecstore::config::Config;
use rustfs_targets::{
TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_pulsar_broker_available,
config::{build_kafka_args, build_nats_args, build_pulsar_args, collect_env_target_instance_ids},
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
config::{
build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
collect_env_target_instance_ids, validate_mysql_config, validate_redis_config,
},
target::{TargetType, mqtt::MQTTTlsConfig},
};
use s3s::{S3Result, s3_error};
use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
use serde::Serialize;
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
@@ -71,8 +75,11 @@ pub(crate) enum AdminTargetValidator {
Webhook,
Mqtt,
Kafka(TargetDomain),
MySql,
Nats(TargetDomain),
Postgres(TargetDomain),
Pulsar(TargetDomain),
Redis(TargetDomain, &'static str),
}
#[derive(Clone, Copy)]
@@ -201,6 +208,19 @@ pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, en
})
}
pub(crate) fn build_json_response(
status: StatusCode,
body: Body,
request_id: Option<&HeaderValue>,
) -> S3Response<(StatusCode, Body)> {
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
if let Some(v) = request_id {
header.insert("x-request-id", v.clone());
}
S3Response::with_headers((status, body), header)
}
pub(crate) fn merge_target_endpoints(
specs: &[AdminTargetSpec],
route_prefix: &str,
@@ -336,8 +356,13 @@ pub(crate) async fn validate_target_request(
AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await,
AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await,
AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await,
AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Redis(domain, default_channel) => {
validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await
}
}
}
@@ -487,6 +512,53 @@ async fn validate_pulsar_request(
})
}
async fn validate_mysql_request(kv_map: &HashMap<String, String>, default_queue_dir: &str) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(MYSQL_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
validate_mysql_config(&to_kvs(kv_map), default_queue_dir).map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
Ok(())
}
async fn validate_postgres_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(POSTGRES_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
check_postgres_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e),
})
}
async fn validate_redis_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
default_channel: &str,
) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(REDIS_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel)
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
check_redis_server_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "Redis server check failed: {}", e),
})
}
fn to_kvs(kv_map: &HashMap<String, String>) -> rustfs_ecstore::config::KVS {
let mut kvs = rustfs_ecstore::config::KVS::new();
for (key, value) in kv_map {