gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(targets): add AMQP support for notify and audit (#2879)

Browse Source 
Co-authored-by: Hyesook Yun <74169420+suk13574@users.noreply.github.com>
This commit is contained in:
houseme
2026-05-09 09:56:26 +08:00
committed by GitHub
parent 1582f216fe
commit 81ad48dac2
29 changed files with 3280 additions and 830 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/audit.rs
+75 -63
View File
@@ -15,7 +15,7 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
@@ -31,13 +31,9 @@ use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::audit::{
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_KEYS, AUDIT_MYSQL_SUB_SYS,
AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_KEYS, AUDIT_POSTGRES_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_REDIS_KEYS, AUDIT_REDIS_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS,
AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_policy::policy::action::{Action, AdminAction};
@@ -45,6 +41,7 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, warn};
@@ -95,57 +92,15 @@ struct AuditEndpointsResponse {
audit_endpoints: Vec<AuditEndpoint>,
}
fn audit_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: AUDIT_WEBHOOK_SUB_SYS,
service: "webhook",
valid_keys: AUDIT_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: AUDIT_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: AUDIT_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_MQTT_SUB_SYS,
service: "mqtt",
valid_keys: AUDIT_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: AUDIT_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: AUDIT_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: AUDIT_NATS_SUB_SYS,
service: "nats",
valid_keys: AUDIT_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: AUDIT_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_PULSAR_SUB_SYS,
service: "pulsar",
valid_keys: AUDIT_PULSAR_KEYS,
validator: AdminTargetValidator::Pulsar(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_REDIS_SUB_SYS,
service: "redis",
valid_keys: AUDIT_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Audit, AUDIT_REDIS_DEFAULT_CHANNEL),
},
]
static AUDIT_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_audit_target_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
});
fn audit_target_specs() -> &'static [AdminTargetSpec] {
&AUDIT_TARGET_SPECS
}
async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<()> {
@@ -172,7 +127,7 @@ fn has_any_audit_targets(config: &Config) -> bool {
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
&audit_target_specs(),
audit_target_specs(),
AUDIT_ROUTE_PREFIX,
config,
target_type,
@@ -193,7 +148,7 @@ async fn audit_target_operation_block_reason(action: &str) -> Option<String> {
}
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
shared_merge_target_endpoints(&audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
shared_merge_target_endpoints(audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| AuditEndpoint {
account_id: endpoint.account_id,
@@ -208,7 +163,7 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &
let target_type = params
.get("target_type")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
if target_service_name(&audit_target_specs(), target_type).is_none() {
if target_service_name(audit_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
}
let target_name = params
@@ -314,7 +269,7 @@ impl Operation for AuditTargetConfig {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
let specs = audit_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
@@ -323,7 +278,7 @@ impl Operation for AuditTargetConfig {
"audit target",
)?;
let spec = target_spec(&specs, target_type)
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR))
.await
@@ -431,7 +386,9 @@ mod tests {
use super::*;
use matchit::Router;
use rustfs_config::ENV_PREFIX;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_ecstore::config::{KV, KVS};
use serial_test::serial;
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars, with_vars_unset};
@@ -469,6 +426,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_config_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -513,6 +471,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_KAFKA_SUB_SYS.to_string(),
@@ -549,6 +508,44 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_amqp_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_AMQP_SUB_SYS.to_string(),
HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_AUDIT_AMQP_ENABLE_MIXED-AMQP", Some("on")),
("RUSTFS_AUDIT_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
("RUSTFS_AUDIT_AMQP_ENABLE_ENV-AMQP", Some("on")),
("RUSTFS_AUDIT_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
],
|| {
let runtime = HashMap::from([
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp")
.expect("mixed amqp target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp")
.expect("env amqp target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
@@ -565,6 +562,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_operation_block_reason_requires_audit_module_enable() {
with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("false"), || {
let reason =
@@ -575,6 +573,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_operation_block_reason_allows_when_audit_module_enabled() {
with_var(rustfs_config::ENV_AUDIT_ENABLE, Some("true"), || {
assert!(
@@ -585,6 +584,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
@@ -598,6 +598,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -622,6 +623,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
@@ -710,6 +712,14 @@ mod tests {
assert_eq!(target_type, AUDIT_KAFKA_SUB_SYS);
assert_eq!(target_name, "primary");
let supported_amqp_params = full_router
.at("/v3/audit/target/audit_amqp/primary")
.expect("route should match");
let (target_type, target_name) =
extract_target_params(&supported_amqp_params.params).expect("audit amqp target should be supported");
assert_eq!(target_type, AUDIT_AMQP_SUB_SYS);
assert_eq!(target_name, "primary");
let mut partial_router = Router::new();
partial_router
.insert("/v3/audit/target/{target_type}", ())
@@ -722,6 +732,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_audit_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
@@ -746,6 +757,7 @@ mod tests {
}
#[test]
#[serial]
fn audit_target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
AUDIT_WEBHOOK_SUB_SYS.to_string(),
rustfs/src/admin/handlers/event.rs
+105 -63
View File
@@ -15,7 +15,7 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
@@ -32,19 +32,16 @@ use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::{
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_MYSQL_KEYS, NOTIFY_MYSQL_SUB_SYS,
NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_POSTGRES_KEYS, NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_KEYS,
NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_DEFAULT_CHANNEL, NOTIFY_REDIS_KEYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_ROUTE_PREFIX,
NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors;
use rustfs_policy::policy::action::{Action, AdminAction};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, info, warn};
@@ -101,57 +98,15 @@ struct NotificationEndpointsResponse {
notification_endpoints: Vec<NotificationEndpoint>,
}
fn notification_target_specs() -> [AdminTargetSpec; 8] {
[
AdminTargetSpec {
subsystem: NOTIFY_WEBHOOK_SUB_SYS,
service: "webhook",
valid_keys: NOTIFY_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: NOTIFY_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: NOTIFY_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_MQTT_SUB_SYS,
service: "mqtt",
valid_keys: NOTIFY_MQTT_KEYS,
validator: AdminTargetValidator::Mqtt,
},
AdminTargetSpec {
subsystem: NOTIFY_MYSQL_SUB_SYS,
service: "mysql",
valid_keys: NOTIFY_MYSQL_KEYS,
validator: AdminTargetValidator::MySql,
},
AdminTargetSpec {
subsystem: NOTIFY_NATS_SUB_SYS,
service: "nats",
valid_keys: NOTIFY_NATS_KEYS,
validator: AdminTargetValidator::Nats(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_POSTGRES_SUB_SYS,
service: "postgres",
valid_keys: NOTIFY_POSTGRES_KEYS,
validator: AdminTargetValidator::Postgres(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_REDIS_SUB_SYS,
service: "redis",
valid_keys: NOTIFY_REDIS_KEYS,
validator: AdminTargetValidator::Redis(TargetDomain::Notify, NOTIFY_REDIS_DEFAULT_CHANNEL),
},
AdminTargetSpec {
subsystem: NOTIFY_PULSAR_SUB_SYS,
service: "pulsar",
valid_keys: NOTIFY_PULSAR_KEYS,
validator: AdminTargetValidator::Pulsar(TargetDomain::Notify),
},
]
static NOTIFICATION_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_notification_target_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
});
fn notification_target_specs() -> &'static [AdminTargetSpec] {
&NOTIFICATION_TARGET_SPECS
}
// --- Helper Functions ---
@@ -172,7 +127,7 @@ fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>>
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
&notification_target_specs(),
notification_target_specs(),
NOTIFY_ROUTE_PREFIX,
config,
target_type,
@@ -193,7 +148,7 @@ async fn notification_target_operation_block_reason(action: &str) -> Option<Stri
}
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
shared_merge_target_endpoints(&notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
shared_merge_target_endpoints(notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| NotificationEndpoint {
account_id: endpoint.account_id,
@@ -241,7 +196,7 @@ impl Operation for NotificationTarget {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?;
let specs = notification_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
notification_body
@@ -252,7 +207,7 @@ impl Operation for NotificationTarget {
target_type,
"target",
)?;
let spec = target_spec(&specs, target_type)
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR))
.await
@@ -397,7 +352,7 @@ fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str>
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
let target_type = extract_param(params, "target_type")?;
if target_service_name(&notification_target_specs(), target_type).is_none() {
if target_service_name(notification_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type));
}
let target_name = extract_param(params, "target_name")?;
@@ -409,8 +364,10 @@ mod tests {
use super::*;
use matchit::Router;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_ecstore::config::{KV, KVS};
use rustfs_targets::arn::TargetID;
use serial_test::serial;
use std::collections::{HashMap, HashSet};
use temp_env::{with_var, with_vars};
@@ -483,6 +440,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_env_and_mixed_sources() {
let config = Config(HashMap::from([
(
@@ -530,6 +488,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
NOTIFY_KAFKA_SUB_SYS.to_string(),
@@ -566,6 +525,44 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_amqp_env_and_mixed_sources() {
let config = Config(HashMap::from([(
NOTIFY_AMQP_SUB_SYS.to_string(),
HashMap::from([("mixed-amqp".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_AMQP_ENABLE_MIXED-AMQP", Some("on")),
("RUSTFS_NOTIFY_AMQP_URL_MIXED-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
("RUSTFS_NOTIFY_AMQP_ENABLE_ENV-AMQP", Some("on")),
("RUSTFS_NOTIFY_AMQP_URL_ENV-AMQP", Some("amqp://127.0.0.1:5672/%2f")),
],
|| {
let runtime = HashMap::from([
(("mixed-amqp".to_string(), "amqp".to_string()), "online".to_string()),
(("env-amqp".to_string(), "amqp".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-amqp" && entry.service == "amqp")
.expect("mixed amqp target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-amqp" && entry.service == "amqp")
.expect("env amqp target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
#[test]
#[serial]
fn target_mutation_block_reason_rejects_env_managed_target() {
with_vars(
[
@@ -582,6 +579,7 @@ mod tests {
}
#[test]
#[serial]
fn notification_target_operation_block_reason_requires_notify_module_enable() {
with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("false"), || {
let reason = futures::executor::block_on(notification_target_operation_block_reason(
@@ -593,6 +591,7 @@ mod tests {
}
#[test]
#[serial]
fn notification_target_operation_block_reason_allows_when_notify_module_enabled() {
with_var(rustfs_config::ENV_NOTIFY_ENABLE, Some("true"), || {
assert!(
@@ -605,6 +604,7 @@ mod tests {
}
#[test]
#[serial]
fn target_mutation_block_reason_rejects_mixed_target() {
with_var("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("https://example.com/hook"), || {
let config = Config(HashMap::from([(
@@ -628,6 +628,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_disabled_config_with_env_override_as_mixed() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -652,6 +653,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_includes_env_only_target_without_runtime_status() {
let config = Config(HashMap::new());
@@ -697,6 +699,7 @@ mod tests {
}
#[test]
#[serial]
fn merge_notification_endpoints_marks_mixed_with_case_insensitive_instance_id() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -734,6 +737,7 @@ mod tests {
}
#[test]
#[serial]
fn target_mutation_block_reason_allows_case_insensitive_config_target_lookup() {
let config = Config(HashMap::from([(
NOTIFY_WEBHOOK_SUB_SYS.to_string(),
@@ -811,6 +815,44 @@ mod tests {
assert_eq!(target_name, "streaming");
}
#[test]
fn extract_target_params_accepts_amqp_target_type() {
let mut router = Router::new();
router
.insert("/v3/target/{target_type}/{target_name}", ())
.expect("route should insert");
let params = router
.at("/v3/target/notify_amqp/rabbitmq")
.expect("route should match")
.params;
let (target_type, target_name) = extract_target_params(&params).expect("amqp target type should be accepted");
assert_eq!(target_type, NOTIFY_AMQP_SUB_SYS);
assert_eq!(target_name, "rabbitmq");
}
#[test]
fn collect_validated_key_values_accepts_amqp_keys() {
let specs = notification_target_specs();
let allowed_keys = allowed_target_keys(specs, NOTIFY_AMQP_SUB_SYS);
let kv_map = shared_collect_validated_key_values(
[
(rustfs_config::AMQP_URL, "amqp://127.0.0.1:5672/%2f"),
(rustfs_config::AMQP_EXCHANGE, "rustfs.events"),
(rustfs_config::AMQP_ROUTING_KEY, "objects"),
],
&allowed_keys,
NOTIFY_AMQP_SUB_SYS,
"target",
)
.expect("amqp keys should be accepted");
assert_eq!(kv_map.get(rustfs_config::AMQP_URL).map(String::as_str), Some("amqp://127.0.0.1:5672/%2f"));
assert!(allowed_keys.contains(rustfs_config::AMQP_MANDATORY));
assert!(allowed_keys.contains(rustfs_config::AMQP_PERSISTENT));
}
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
let start = src
.find(start_marker)
rustfs/src/admin/handlers/target_descriptor.rs
+240 -30
View File
@@ -15,16 +15,17 @@
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, HeaderValue, StatusCode};
use rustfs_config::{
ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
};
use rustfs_ecstore::config::Config;
use rustfs_targets::{
TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
check_mqtt_broker_available_with_tls, check_nats_server_available, check_postgres_server_available,
check_pulsar_broker_available, check_redis_server_available,
config::{
build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
build_amqp_args, build_kafka_args, build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args,
collect_env_target_instance_ids, validate_mysql_config, validate_redis_config,
},
target::{TargetType, mqtt::MQTTTlsConfig},
@@ -34,10 +35,13 @@ use serde::Serialize;
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::Path;
use std::sync::Arc;
use tokio::time::{Duration, sleep};
use url::Url;
pub(crate) type EndpointKey = (String, String);
type AdminRequestValidatorFn =
Arc<dyn Fn(&HashMap<String, String>, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
@@ -55,14 +59,14 @@ pub(crate) struct MergedTargetEndpoint {
pub source: TargetEndpointSource,
}
#[derive(Clone, Copy)]
#[derive(Clone, Copy, Debug)]
pub(crate) enum TargetDomain {
Notify,
Audit,
}
impl TargetDomain {
fn runtime_target_type(self) -> TargetType {
pub(crate) fn runtime_target_type(self) -> TargetType {
match self {
TargetDomain::Notify => TargetType::NotifyEvent,
TargetDomain::Audit => TargetType::AuditLog,
@@ -70,24 +74,98 @@ impl TargetDomain {
}
}
#[derive(Clone, Copy)]
pub(crate) enum AdminTargetValidator {
Webhook,
Mqtt,
Kafka(TargetDomain),
MySql,
Nats(TargetDomain),
Postgres(TargetDomain),
Pulsar(TargetDomain),
Redis(TargetDomain, &'static str),
impl From<TargetType> for TargetDomain {
fn from(value: TargetType) -> Self {
match value {
TargetType::NotifyEvent => TargetDomain::Notify,
TargetType::AuditLog => TargetDomain::Audit,
}
}
}
#[derive(Clone, Copy)]
#[derive(Clone)]
pub(crate) struct AdminTargetSpec {
pub subsystem: &'static str,
pub service: &'static str,
pub valid_keys: &'static [&'static str],
pub validator: AdminTargetValidator,
validator: AdminRequestValidatorFn,
}
pub(crate) fn admin_target_spec_from_builtin<E>(descriptor: &BuiltinTargetDescriptor<E>) -> AdminTargetSpec
where
E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
{
AdminTargetSpec {
subsystem: descriptor.subsystem(),
service: descriptor.plugin().target_type(),
valid_keys: descriptor.plugin().valid_fields(),
validator: match descriptor.request_validator() {
TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
TargetRequestValidator::Amqp(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_amqp_request_entry)
} else {
Arc::new(validate_notify_amqp_request_entry)
}
}
TargetRequestValidator::Kafka(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_kafka_request_entry)
} else {
Arc::new(validate_notify_kafka_request_entry)
}
}
TargetRequestValidator::MySql => Arc::new(validate_mysql_request_entry),
TargetRequestValidator::Nats(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_nats_request_entry)
} else {
Arc::new(validate_notify_nats_request_entry)
}
}
TargetRequestValidator::Postgres(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_postgres_request_entry)
} else {
Arc::new(validate_notify_postgres_request_entry)
}
}
TargetRequestValidator::Pulsar(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_pulsar_request_entry)
} else {
Arc::new(validate_notify_pulsar_request_entry)
}
}
TargetRequestValidator::Redis {
default_channel,
target_type,
} => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
validate_audit_redis_request_entry(default_channel)
} else {
validate_notify_redis_request_entry(default_channel)
}
}
},
}
}
impl std::fmt::Debug for AdminTargetSpec {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AdminTargetSpec")
.field("subsystem", &self.subsystem)
.field("service", &self.service)
.field("valid_keys", &self.valid_keys)
.finish_non_exhaustive()
}
}
impl AdminTargetSpec {
pub(crate) async fn validate_request(&self, kv_map: &HashMap<String, String>, default_queue_dir: &str) -> S3Result<()> {
(self.validator)(kv_map, default_queue_dir).await
}
}
pub(crate) fn normalized_endpoint_key(account_id: &str, service: &str) -> EndpointKey {
@@ -352,18 +430,7 @@ pub(crate) async fn validate_target_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> S3Result<()> {
match spec.validator {
AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await,
AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await,
AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::MySql => validate_mysql_request(kv_map, default_queue_dir).await,
AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Postgres(domain) => validate_postgres_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Redis(domain, default_channel) => {
validate_redis_request(kv_map, default_queue_dir, domain, default_channel).await
}
}
spec.validate_request(kv_map, default_queue_dir).await
}
fn config_enable_is_on(value: &str) -> bool {
@@ -420,6 +487,14 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
Ok(())
}
fn validate_webhook_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_webhook_request(&kv_map).await })
}
async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()> {
let endpoint = kv_map
.get(MQTT_BROKER)
@@ -464,6 +539,129 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
Ok(())
}
fn validate_mqtt_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_mqtt_request(&kv_map).await })
}
fn validate_notify_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_mysql_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir).await })
}
fn validate_notify_postgres_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_postgres_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await })
})
}
fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await })
})
}
async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir.as_str()).await?;
@@ -496,6 +694,18 @@ async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_
})
}
async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
let args = build_amqp_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
check_amqp_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "AMQP broker check failed: {}", e),
})
}
async fn validate_pulsar_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
rustfs/src/server/audit.rs
+2 -7
View File
@@ -35,12 +35,7 @@ pub fn is_audit_module_enabled() -> bool {
}
fn has_any_audit_targets(config: &rustfs_ecstore::config::Config) -> bool {
for subsystem in [
rustfs_config::audit::AUDIT_MQTT_SUB_SYS,
rustfs_config::audit::AUDIT_NATS_SUB_SYS,
rustfs_config::audit::AUDIT_PULSAR_SUB_SYS,
rustfs_config::audit::AUDIT_WEBHOOK_SUB_SYS,
] {
for &subsystem in rustfs_config::audit::AUDIT_SUB_SYSTEMS {
let Some(targets) = config.0.get(subsystem) else {
continue;
};
@@ -102,7 +97,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
if !has_targets {
info!(
target: "rustfs::main::start_audit_system",
"Audit subsystem (Webhook/MQTT/NATS/Pulsar) is not configured, and audit system initialization is skipped."
"Audit subsystem targets are not configured, and audit system initialization is skipped."
);
return Ok(());
}