feat(targets): complete redis mysql postgres target wiring (#2842)

Signed-off-by: jaehanbyun <awbrg789@naver.com> Signed-off-by: houseme <housemecn@gmail.com> Signed-off-by: Gunther Xing <jiengup@gmail.com> Signed-off-by: JaySon-Huang <tshent@qq.com> Co-authored-by: jaehanbyun <awbrg789@naver.com> Co-authored-by: Gunther Xing <jiengup@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: cxymds <Cxymds@qq.com> Co-authored-by: JaySon <tshent@qq.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
2026-08-09 22:59:59 +00:00 · 2026-05-07 18:00:59 +08:00
parent b159d656cc
commit 5431b9273d
41 changed files with 6787 additions and 237 deletions
@@ -16,7 +16,7 @@ use crate::admin::{
    auth::validate_admin_request,
    handlers::target_descriptor::{
        AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
-        collect_validated_key_values as shared_collect_validated_key_values,
+        build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
        target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
        validate_target_request,
@@ -28,18 +28,20 @@ use crate::server::{
    ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store,
 };
 use futures::stream::{FuturesUnordered, StreamExt};
-use http::{HeaderMap, StatusCode};
+use http::StatusCode;
 use hyper::Method;
 use matchit::Params;
 use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
 use rustfs_config::audit::{
-    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
-    AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
+    AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
+    AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
+    AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS,
+    AUDIT_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
 use rustfs_ecstore::config::Config;
 use rustfs_policy::policy::action::{Action, AdminAction};
-use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
+use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
@@ -93,7 +95,7 @@ struct AuditEndpointsResponse {
    audit_endpoints: Vec<AuditEndpoint>,
 }

-fn audit_target_specs() -> [AdminTargetSpec; 5] {
+fn audit_target_specs() -> [AdminTargetSpec; 8] {
    [
        AdminTargetSpec {
            subsystem: AUDIT_WEBHOOK_SUB_SYS,
@@ -113,18 +115,36 @@ fn audit_target_specs() -> [AdminTargetSpec; 5] {
            valid_keys: AUDIT_MQTT_KEYS,
            validator: AdminTargetValidator::Mqtt,
        },
+        AdminTargetSpec {
+            subsystem: AUDIT_MYSQL_SUB_SYS,
+            service: "mysql",
+            valid_keys: AUDIT_MYSQL_KEYS,
+            validator: AdminTargetValidator::MySql,
+        },
        AdminTargetSpec {
            subsystem: AUDIT_NATS_SUB_SYS,
            service: "nats",
            valid_keys: AUDIT_NATS_KEYS,
            validator: AdminTargetValidator::Nats(TargetDomain::Audit),
        },
+        AdminTargetSpec {
+            subsystem: AUDIT_POSTGRES_SUB_SYS,
+            service: "postgres",
+            valid_keys: AUDIT_POSTGRES_KEYS,
+            validator: AdminTargetValidator::Postgres(TargetDomain::Audit),
+        },
        AdminTargetSpec {
            subsystem: AUDIT_PULSAR_SUB_SYS,
            service: "pulsar",
            valid_keys: AUDIT_PULSAR_KEYS,
            validator: AdminTargetValidator::Pulsar(TargetDomain::Audit),
        },
+        AdminTargetSpec {
+            subsystem: AUDIT_REDIS_SUB_SYS,
+            service: "redis",
+            valid_keys: AUDIT_REDIS_KEYS,
+            validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL),
+        },
    ]
 }

@@ -138,15 +158,6 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
    validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
 }

-fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
-    let mut header = HeaderMap::new();
-    header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
-    if let Some(v) = request_id {
-        header.insert("x-request-id", v.clone());
-    }
-    S3Response::with_headers((status, body), header)
-}
-
 fn has_any_audit_targets(config: &Config) -> bool {
    for spec in audit_target_specs() {
        let Some(targets) = config.0.get(spec.subsystem) else {
@@ -334,7 +345,7 @@ impl Operation for AuditTargetConfig {
        })
        .await?;

-        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
    }
 }

@@ -375,7 +386,7 @@ impl Operation for ListAuditTargets {
        let data = serde_json::to_vec(&AuditEndpointsResponse { audit_endpoints })
            .map_err(|e| s3_error!(InternalError, "failed to serialize audit targets: {}", e))?;

-        Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
    }
 }

@@ -411,7 +422,7 @@ impl Operation for RemoveAuditTarget {
        })
        .await?;

-        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
    }
 }

@@ -16,7 +16,7 @@ use crate::admin::{
    auth::validate_admin_request,
    handlers::target_descriptor::{
        AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
-        collect_validated_key_values as shared_collect_validated_key_values,
+        build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
        merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
        target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
        validate_target_request,
@@ -29,17 +29,19 @@ use crate::server::{
    refresh_persisted_module_switches_from_store,
 };
 use futures::stream::{FuturesUnordered, StreamExt};
-use http::{HeaderMap, StatusCode};
+use http::StatusCode;
 use hyper::Method;
 use matchit::Params;
 use rustfs_config::notify::{
-    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
-    NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
+    NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
+    NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
+    NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX,
+    NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
 };
 use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
 use rustfs_ecstore::config::Config;
 use rustfs_policy::policy::action::{Action, AdminAction};
-use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
+use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
@@ -99,7 +101,7 @@ struct NotificationEndpointsResponse {
    notification_endpoints: Vec<NotificationEndpoint>,
 }

-fn notification_target_specs() -> [AdminTargetSpec; 5] {
+fn notification_target_specs() -> [AdminTargetSpec; 8] {
    [
        AdminTargetSpec {
            subsystem: NOTIFY_WEBHOOK_SUB_SYS,
@@ -119,12 +121,30 @@ fn notification_target_specs() -> [AdminTargetSpec; 5] {
            valid_keys: NOTIFY_MQTT_KEYS,
            validator: AdminTargetValidator::Mqtt,
        },
+        AdminTargetSpec {
+            subsystem: NOTIFY_MYSQL_SUB_SYS,
+            service: "mysql",
+            valid_keys: NOTIFY_MYSQL_KEYS,
+            validator: AdminTargetValidator::MySql,
+        },
        AdminTargetSpec {
            subsystem: NOTIFY_NATS_SUB_SYS,
            service: "nats",
            valid_keys: NOTIFY_NATS_KEYS,
            validator: AdminTargetValidator::Nats(TargetDomain::Notify),
        },
+        AdminTargetSpec {
+            subsystem: NOTIFY_POSTGRES_SUB_SYS,
+            service: "postgres",
+            valid_keys: NOTIFY_POSTGRES_KEYS,
+            validator: AdminTargetValidator::Postgres(TargetDomain::Notify),
+        },
+        AdminTargetSpec {
+            subsystem: NOTIFY_REDIS_SUB_SYS,
+            service: "redis",
+            valid_keys: NOTIFY_REDIS_KEYS,
+            validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL),
+        },
        AdminTargetSpec {
            subsystem: NOTIFY_PULSAR_SUB_SYS,
            service: "pulsar",
@@ -150,15 +170,6 @@ fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>>
    rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
 }

-fn build_response(status: StatusCode, body: Body, request_id: Option<&http::HeaderValue>) -> S3Response<(StatusCode, Body)> {
-    let mut header = HeaderMap::new();
-    header.insert(CONTENT_TYPE, "application/json".parse().unwrap());
-    if let Some(v) = request_id {
-        header.insert("x-request-id", v.clone());
-    }
-    S3Response::with_headers((status, body), header)
-}
-
 fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
    shared_target_mutation_block_reason(
        &notification_target_specs(),
@@ -258,7 +269,7 @@ impl Operation for NotificationTarget {
            .await
            .map_err(|e| s3_error!(InternalError, "failed to set target config: {}", e))?;

-        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
    }
 }

@@ -297,7 +308,7 @@ impl Operation for ListNotificationTargets {
        let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
            .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;

-        Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
    }
 }

@@ -347,7 +358,7 @@ impl Operation for ListTargetsArns {
        let data = serde_json::to_vec(&data_target_arn_list)
            .map_err(|e| s3_error!(InternalError, "failed to serialize targets: {}", e))?;

-        Ok(build_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::from(data), req.headers.get("x-request-id")))
    }
 }

@@ -374,7 +385,7 @@ impl Operation for RemoveNotificationTarget {
            .await
            .map_err(|e| s3_error!(InternalError, "failed to remove target config: {}", e))?;

-        Ok(build_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
+        Ok(build_json_response(StatusCode::OK, Body::empty(), req.headers.get("x-request-id")))
    }
 }

@@ -13,19 +13,23 @@
 // limitations under the License.

 use hashbrown::HashSet as HbHashSet;
+use http::{HeaderMap, HeaderValue, StatusCode};
 use rustfs_config::{
    ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
    MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
-    MQTT_WS_PATH_ALLOWLIST,
+    MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
 };
 use rustfs_ecstore::config::Config;
 use rustfs_targets::{
    TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
-    check_pulsar_broker_available,
-    config::{build_kafka_args, build_nats_args, build_pulsar_args, collect_env_target_instance_ids},
+    check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
+    config::{
+        build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
+        collect_env_target_instance_ids, validate_mysql_config, validate_redis_config,
+    },
    target::{TargetType, mqtt::MQTTTlsConfig},
 };
-use s3s::{S3Result, s3_error};
+use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
 use serde::Serialize;
 use std::collections::{HashMap, HashSet};
 use std::io::{Error, ErrorKind};
@@ -71,8 +75,11 @@ pub(crate) enum AdminTargetValidator {
    Webhook,
    Mqtt,
    Kafka(TargetDomain),
+    MySql,
    Nats(TargetDomain),
+    Postgres(TargetDomain),
    Pulsar(TargetDomain),
+    Redis(TargetDomain, &'static str),
 }

 #[derive(Clone, Copy)]
@@ -201,6 +208,19 @@ pub(crate) fn target_module_disabled_reason(module_name: &str, env_key: &str, en
    })
 }

+pub(crate) fn build_json_response(
+    status: StatusCode,
+    body: Body,
+    request_id: Option<&HeaderValue>,
+) -> S3Response<(StatusCode, Body)> {
+    let mut header = HeaderMap::new();
+    header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+    if let Some(v) = request_id {
+        header.insert("x-request-id", v.clone());
+    }
+    S3Response::with_headers((status, body), header)
+}
+
 pub(crate) fn merge_target_endpoints(
    specs: &[AdminTargetSpec],
    route_prefix: &str,
@@ -336,8 +356,13 @@ pub(crate) async fn validate_target_request(
        AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await,
        AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await,
        AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await,
+        AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await,
        AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await,
        AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await,
+        AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await,
+        AdminTargetValidator::Redis(domain, default_channel) => {
+            validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await
+        }
    }
 }

@@ -487,6 +512,53 @@ async fn validate_pulsar_request(
    })
 }

+async fn validate_mysql_request(kv_map: &HashMap<String, String>, default_queue_dir: &str) -> S3Result<()> {
+    if let Some(queue_dir) = kv_map.get(MYSQL_QUEUE_DIR) {
+        validate_queue_dir(queue_dir.as_str()).await?;
+    }
+
+    validate_mysql_config(&to_kvs(kv_map), default_queue_dir).map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
+
+    Ok(())
+}
+
+async fn validate_postgres_request(
+    kv_map: &HashMap<String, String>,
+    default_queue_dir: &str,
+    domain: TargetDomain,
+) -> S3Result<()> {
+    if let Some(queue_dir) = kv_map.get(POSTGRES_QUEUE_DIR) {
+        validate_queue_dir(queue_dir.as_str()).await?;
+    }
+    let args = build_postgres_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
+        .map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
+    check_postgres_server_available(&args).await.map_err(|e| match e {
+        TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
+        _ => s3_error!(InvalidArgument, "PostgreSQL server check failed: {}", e),
+    })
+}
+
+async fn validate_redis_request(
+    kv_map: &HashMap<String, String>,
+    default_queue_dir: &str,
+    domain: TargetDomain,
+    default_channel: &str,
+) -> S3Result<()> {
+    if let Some(queue_dir) = kv_map.get(REDIS_QUEUE_DIR) {
+        validate_queue_dir(queue_dir.as_str()).await?;
+    }
+
+    validate_redis_config(&to_kvs(kv_map), default_queue_dir, default_channel)
+        .map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
+
+    let args = build_redis_args(&to_kvs(kv_map), default_queue_dir, default_channel, domain.runtime_target_type())
+        .map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
+    check_redis_server_available(&args).await.map_err(|e| match e {
+        TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
+        _ => s3_error!(InvalidArgument, "Redis server check failed: {}", e),
+    })
+}
+
 fn to_kvs(kv_map: &HashMap<String, String>) -> rustfs_ecstore::config::KVS {
    let mut kvs = rustfs_ecstore::config::KVS::new();
    for (key, value) in kv_map {