gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 22:33:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(targets): unify endpoint source/merge logic and bump rustfs-kafka-async to v1.2.0 (#2654)

Browse Source 
Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
This commit is contained in:
houseme
2026-04-23 17:14:36 +08:00
committed by GitHub
parent bc37cc4001
commit 368ef0f16c
26 changed files with 1589 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/audit.rs
+104 -160
View File
@@ -15,10 +15,10 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, allowed_target_keys,
collect_config_entry_keys as shared_collect_config_entry_keys,
collect_configured_endpoint_keys as shared_collect_configured_endpoint_keys,
collect_env_endpoint_keys as shared_collect_env_endpoint_keys, normalized_endpoint_key, target_service_name, target_spec,
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
},
router::{AdminOperation, Operation, S3Router},
@@ -26,14 +26,13 @@ use crate::admin::{
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::audit::{
AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS, AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS,
AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS, AUDIT_NATS_KEYS, AUDIT_NATS_SUB_SYS,
AUDIT_PULSAR_KEYS, AUDIT_PULSAR_SUB_SYS, AUDIT_ROUTE_PREFIX, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS,
};
use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
@@ -84,7 +83,7 @@ struct AuditEndpoint {
account_id: String,
service: String,
status: String,
source: AuditEndpointSource,
source: TargetEndpointSource,
}
#[derive(Serialize, Debug)]
@@ -92,16 +91,7 @@ struct AuditEndpointsResponse {
audit_endpoints: Vec<AuditEndpoint>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
enum AuditEndpointSource {
Config,
Env,
Mixed,
Runtime,
}
fn audit_target_specs() -> [AdminTargetSpec; 4] {
fn audit_target_specs() -> [AdminTargetSpec; 5] {
[
AdminTargetSpec {
subsystem: AUDIT_WEBHOOK_SUB_SYS,
@@ -109,6 +99,12 @@ fn audit_target_specs() -> [AdminTargetSpec; 4] {
valid_keys: AUDIT_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: AUDIT_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: AUDIT_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Audit),
},
AdminTargetSpec {
subsystem: AUDIT_MQTT_SUB_SYS,
service: "mqtt",
@@ -161,140 +157,27 @@ fn has_any_audit_targets(config: &Config) -> bool {
false
}
fn collect_configured_audit_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
shared_collect_configured_endpoint_keys(&audit_target_specs(), config)
}
fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
shared_collect_config_entry_keys(&audit_target_specs(), config)
}
fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
shared_collect_env_endpoint_keys(&audit_target_specs(), AUDIT_ROUTE_PREFIX)
}
fn classify_audit_endpoint_source(
config_targets: &HbHashSet<EndpointKey>,
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> AuditEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
(true, true) => AuditEndpointSource::Mixed,
(true, false) => AuditEndpointSource::Config,
(false, true) => AuditEndpointSource::Env,
(false, false) => AuditEndpointSource::Runtime,
}
}
fn audit_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> AuditEndpointSource {
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let service = target_service_name(&audit_target_specs(), target_type).unwrap_or_default();
let key = normalized_endpoint_key(target_name, service);
classify_audit_endpoint_source(&config_targets, &env_targets, &key)
}
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
match audit_endpoint_source(config, target_type, target_name) {
AuditEndpointSource::Env => Some(format!(
"audit target '{}' is managed by environment variables and cannot be modified from the console",
target_name
)),
AuditEndpointSource::Mixed => Some(format!(
"audit target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
target_name
)),
AuditEndpointSource::Config | AuditEndpointSource::Runtime => None,
}
shared_target_mutation_block_reason(
&audit_target_specs(),
AUDIT_ROUTE_PREFIX,
config,
target_type,
target_name,
"audit target",
)
}
fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<AuditEndpoint> {
let mut audit_endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_audit_endpoint_keys(config);
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for key in configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
}
let status = normalized_runtime_statuses
.remove(&normalized)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
let source = classify_audit_endpoint_source(&config_targets, &env_targets, &normalized);
audit_endpoints.push(AuditEndpoint {
account_id: key.0,
service: key.1,
status,
source,
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if seen.insert(normalized.clone()) {
audit_endpoints.push(AuditEndpoint {
account_id,
service,
status,
source: classify_audit_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
}
for key in &env_targets {
if !seen.insert(key.clone()) {
continue;
}
audit_endpoints.push(AuditEndpoint {
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_audit_endpoint_source(&config_targets, &env_targets, key),
});
}
audit_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
audit_endpoints
}
fn collect_validated_key_values(
key_values: &[KeyValue],
allowed_keys: &HashSet<&str>,
target_type: &str,
) -> S3Result<HashMap<String, String>> {
let mut kv_map = HashMap::new();
let mut seen = HashSet::new();
for kv in key_values {
if !allowed_keys.contains(kv.key.as_str()) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for audit target type '{}'",
kv.key,
target_type
));
}
if !seen.insert(kv.key.as_str()) {
return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
}
kv_map.insert(kv.key.clone(), kv.value.clone());
}
Ok(kv_map)
shared_merge_target_endpoints(&audit_target_specs(), AUDIT_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| AuditEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect()
}
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
@@ -406,7 +289,12 @@ impl Operation for AuditTargetConfig {
let specs = audit_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let kv_map = collect_validated_key_values(&audit_body.key_values, &allowed_keys, target_type)?;
let kv_map = shared_collect_validated_key_values(
audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
target_type,
"audit target",
)?;
let spec = target_spec(&specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?;
@@ -577,19 +465,55 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "mixed-target")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, AuditEndpointSource::Env);
assert_eq!(env_only.source, TargetEndpointSource::Env);
let config_only = merged
.iter()
.find(|entry| entry.account_id == "config-target")
.expect("config target should be present");
assert_eq!(config_only.source, AuditEndpointSource::Config);
assert_eq!(config_only.source, TargetEndpointSource::Config);
},
);
}
#[test]
fn merge_audit_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
AUDIT_KAFKA_SUB_SYS.to_string(),
HashMap::from([("mixed-kafka".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_AUDIT_KAFKA_ENABLE_MIXED-KAFKA", Some("on")),
("RUSTFS_AUDIT_KAFKA_BROKERS_MIXED-KAFKA", Some("127.0.0.1:9092")),
("RUSTFS_AUDIT_KAFKA_ENABLE_ENV-KAFKA", Some("on")),
("RUSTFS_AUDIT_KAFKA_BROKERS_ENV-KAFKA", Some("127.0.0.1:9093")),
],
|| {
let runtime = HashMap::from([
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
]);
let merged = merge_audit_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-kafka" && entry.service == "kafka")
.expect("mixed kafka target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-kafka" && entry.service == "kafka")
.expect("env kafka target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
@@ -641,7 +565,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
assert_eq!(mixed.status, "offline");
},
);
@@ -662,7 +586,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, AuditEndpointSource::Env);
assert_eq!(env_only.source, TargetEndpointSource::Env);
assert_eq!(env_only.status, "offline");
},
);
@@ -671,7 +595,7 @@ mod tests {
#[test]
fn collect_validated_key_values_rejects_duplicate_keys() {
let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
let key_values = vec![
let key_values = [
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/one".to_string(),
@@ -682,19 +606,31 @@ mod tests {
},
];
let err = collect_validated_key_values(&key_values, &allowed_keys, AUDIT_WEBHOOK_SUB_SYS).unwrap_err();
let err = shared_collect_validated_key_values(
key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
AUDIT_WEBHOOK_SUB_SYS,
"audit target",
)
.unwrap_err();
assert!(err.to_string().contains("duplicate key"));
}
#[test]
fn collect_validated_key_values_rejects_unsupported_key() {
let allowed_keys: HashSet<&str> = AUDIT_WEBHOOK_KEYS.iter().copied().collect();
let key_values = vec![KeyValue {
let key_values = [KeyValue {
key: "not_a_real_key".to_string(),
value: "/tmp/rustfs-audit".to_string(),
}];
let err = collect_validated_key_values(&key_values, &allowed_keys, AUDIT_WEBHOOK_SUB_SYS).unwrap_err();
let err = shared_collect_validated_key_values(
key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
AUDIT_WEBHOOK_SUB_SYS,
"audit target",
)
.unwrap_err();
assert!(err.to_string().contains("not allowed for audit target type"));
}
@@ -711,11 +647,19 @@ mod tests {
.insert("/v3/audit/target/{target_type}/{target_name}", ())
.expect("route should insert");
let unsupported_type_params = full_router
.at("/v3/audit/target/audit_kafka/primary")
.at("/v3/audit/target/audit_unknown/primary")
.expect("route should match");
let unsupported_type = extract_target_params(&unsupported_type_params.params).unwrap_err();
assert!(unsupported_type.to_string().contains("unsupported audit target type"));
let supported_kafka_params = full_router
.at("/v3/audit/target/audit_kafka/primary")
.expect("route should match");
let (target_type, target_name) =
extract_target_params(&supported_kafka_params.params).expect("audit kafka target should be supported");
assert_eq!(target_type, AUDIT_KAFKA_SUB_SYS);
assert_eq!(target_name, "primary");
let mut partial_router = Router::new();
partial_router
.insert("/v3/audit/target/{target_type}", ())
@@ -746,7 +690,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
.expect("mixed target should be present");
assert_eq!(mixed.source, AuditEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
},
);
}
rustfs/src/admin/handlers/event.rs
+111 -161
View File
@@ -15,10 +15,10 @@
use crate::admin::{
auth::validate_admin_request,
handlers::target_descriptor::{
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, allowed_target_keys,
collect_config_entry_keys as shared_collect_config_entry_keys,
collect_configured_endpoint_keys as shared_collect_configured_endpoint_keys,
collect_env_endpoint_keys as shared_collect_env_endpoint_keys, normalized_endpoint_key, target_service_name, target_spec,
AdminTargetSpec, AdminTargetValidator, EndpointKey, TargetDomain, TargetEndpointSource, allowed_target_keys,
collect_validated_key_values as shared_collect_validated_key_values,
merge_target_endpoints as shared_merge_target_endpoints,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
},
router::{AdminOperation, Operation, S3Router},
@@ -26,13 +26,12 @@ use crate::admin::{
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use futures::stream::{FuturesUnordered, StreamExt};
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::{
NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS, NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS,
NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
NOTIFY_KAFKA_KEYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_KEYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_NATS_KEYS, NOTIFY_NATS_SUB_SYS,
NOTIFY_PULSAR_KEYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_ROUTE_PREFIX, NOTIFY_WEBHOOK_KEYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
@@ -89,7 +88,7 @@ struct NotificationEndpoint {
account_id: String,
service: String,
status: String,
source: NotificationEndpointSource,
source: TargetEndpointSource,
}
#[derive(Serialize, Debug)]
@@ -97,16 +96,7 @@ struct NotificationEndpointsResponse {
notification_endpoints: Vec<NotificationEndpoint>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
enum NotificationEndpointSource {
Config,
Env,
Mixed,
Runtime,
}
fn notification_target_specs() -> [AdminTargetSpec; 4] {
fn notification_target_specs() -> [AdminTargetSpec; 5] {
[
AdminTargetSpec {
subsystem: NOTIFY_WEBHOOK_SUB_SYS,
@@ -114,6 +104,12 @@ fn notification_target_specs() -> [AdminTargetSpec; 4] {
valid_keys: NOTIFY_WEBHOOK_KEYS,
validator: AdminTargetValidator::Webhook,
},
AdminTargetSpec {
subsystem: NOTIFY_KAFKA_SUB_SYS,
service: "kafka",
valid_keys: NOTIFY_KAFKA_KEYS,
validator: AdminTargetValidator::Kafka(TargetDomain::Notify),
},
AdminTargetSpec {
subsystem: NOTIFY_MQTT_SUB_SYS,
service: "mqtt",
@@ -160,112 +156,27 @@ fn build_response(status: StatusCode, body: Body, request_id: Option<&http::Head
S3Response::with_headers((status, body), header)
}
fn collect_configured_endpoint_keys(config: &Config) -> Vec<EndpointKey> {
shared_collect_configured_endpoint_keys(&notification_target_specs(), config)
}
fn collect_config_entry_keys(config: &Config) -> HbHashSet<EndpointKey> {
shared_collect_config_entry_keys(&notification_target_specs(), config)
}
fn collect_env_endpoint_keys() -> HbHashSet<EndpointKey> {
shared_collect_env_endpoint_keys(&notification_target_specs(), NOTIFY_ROUTE_PREFIX)
}
fn classify_notification_endpoint_source(
config_targets: &HbHashSet<EndpointKey>,
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> NotificationEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
(true, true) => NotificationEndpointSource::Mixed,
(true, false) => NotificationEndpointSource::Config,
(false, true) => NotificationEndpointSource::Env,
(false, false) => NotificationEndpointSource::Runtime,
}
}
fn notification_endpoint_source(config: &Config, target_type: &str, target_name: &str) -> NotificationEndpointSource {
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let service = target_service_name(&notification_target_specs(), target_type).unwrap_or_default();
let key = normalized_endpoint_key(target_name, service);
classify_notification_endpoint_source(&config_targets, &env_targets, &key)
}
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
match notification_endpoint_source(config, target_type, target_name) {
NotificationEndpointSource::Env => Some(format!(
"target '{}' is managed by environment variables and cannot be modified from the console",
target_name
)),
NotificationEndpointSource::Mixed => Some(format!(
"target '{}' is configured by both persisted config and environment variables; remove the environment variables first",
target_name
)),
NotificationEndpointSource::Config | NotificationEndpointSource::Runtime => None,
}
shared_target_mutation_block_reason(
&notification_target_specs(),
NOTIFY_ROUTE_PREFIX,
config,
target_type,
target_name,
"target",
)
}
fn merge_notification_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey, String>) -> Vec<NotificationEndpoint> {
let mut notification_endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_endpoint_keys(config);
let config_targets = collect_config_entry_keys(config);
let env_targets = collect_env_endpoint_keys();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for key in configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
}
let status = normalized_runtime_statuses
.remove(&normalized)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
let source = classify_notification_endpoint_source(&config_targets, &env_targets, &normalized);
notification_endpoints.push(NotificationEndpoint {
account_id: key.0,
service: key.1,
status,
source,
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if seen.insert(normalized.clone()) {
notification_endpoints.push(NotificationEndpoint {
account_id,
service,
status,
source: classify_notification_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
}
for key in &env_targets {
if !seen.insert(key.clone()) {
continue;
}
notification_endpoints.push(NotificationEndpoint {
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_notification_endpoint_source(&config_targets, &env_targets, key),
});
}
notification_endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
notification_endpoints
shared_merge_target_endpoints(&notification_target_specs(), NOTIFY_ROUTE_PREFIX, config, runtime_statuses)
.into_iter()
.map(|endpoint| NotificationEndpoint {
account_id: endpoint.account_id,
service: endpoint.service,
status: endpoint.status,
source: endpoint.source,
})
.collect()
}
fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets::arn::TargetID, String)>) -> Vec<String> {
@@ -275,34 +186,6 @@ fn collect_online_target_arns(region: &str, target_statuses: Vec<(rustfs_targets
.collect()
}
fn collect_validated_key_values(
key_values: &[KeyValue],
allowed_keys: &HashSet<&str>,
target_type: &str,
) -> S3Result<HashMap<String, String>> {
let mut kv_map = HashMap::new();
let mut seen = HashSet::new();
for kv in key_values {
if !allowed_keys.contains(kv.key.as_str()) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for target type '{}'",
kv.key,
target_type
));
}
if !seen.insert(kv.key.as_str()) {
return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", kv.key));
}
kv_map.insert(kv.key.clone(), kv.value.clone());
}
Ok(kv_map)
}
// --- Operations ---
pub struct NotificationTarget {}
@@ -332,7 +215,15 @@ impl Operation for NotificationTarget {
let specs = notification_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(&specs, target_type);
let kv_map = collect_validated_key_values(&notification_body.key_values, &allowed_keys, target_type)?;
let kv_map = shared_collect_validated_key_values(
notification_body
.key_values
.iter()
.map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
target_type,
"target",
)?;
let spec = target_spec(&specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR))
@@ -478,6 +369,7 @@ fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &
#[cfg(test)]
mod tests {
use super::*;
use matchit::Router;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_ecstore::config::{KV, KVS};
use rustfs_targets::arn::TargetID;
@@ -513,14 +405,14 @@ mod tests {
.find(|entry| entry.account_id == "mqtt-a" && entry.service == "mqtt")
.expect("mqtt-a should be present");
assert_eq!(mqtt.status, "offline");
assert_eq!(mqtt.source, NotificationEndpointSource::Config);
assert_eq!(mqtt.source, TargetEndpointSource::Config);
let webhook = merged
.iter()
.find(|entry| entry.account_id == "webhook-a" && entry.service == "webhook")
.expect("webhook-a should be present");
assert_eq!(webhook.status, "online");
assert_eq!(webhook.source, NotificationEndpointSource::Config);
assert_eq!(webhook.source, TargetEndpointSource::Config);
}
#[test]
@@ -542,14 +434,14 @@ mod tests {
.find(|entry| entry.account_id == "env-only" && entry.service == "mqtt")
.expect("env-only should be present");
assert_eq!(env_only.status, "offline");
assert_eq!(env_only.source, NotificationEndpointSource::Runtime);
assert_eq!(env_only.source, TargetEndpointSource::Runtime);
let enabled = merged
.iter()
.find(|entry| entry.account_id == "webhook-enabled" && entry.service == "webhook")
.expect("webhook-enabled should be present");
assert_eq!(enabled.status, "online");
assert_eq!(enabled.source, NotificationEndpointSource::Config);
assert_eq!(enabled.source, TargetEndpointSource::Config);
}
#[test]
@@ -582,19 +474,55 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "mixed-target")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, NotificationEndpointSource::Env);
assert_eq!(env_only.source, TargetEndpointSource::Env);
let config_only = merged
.iter()
.find(|entry| entry.account_id == "config-target")
.expect("config target should be present");
assert_eq!(config_only.source, NotificationEndpointSource::Config);
assert_eq!(config_only.source, TargetEndpointSource::Config);
},
);
}
#[test]
fn merge_notification_endpoints_marks_kafka_env_and_mixed_sources() {
let config = Config(HashMap::from([(
NOTIFY_KAFKA_SUB_SYS.to_string(),
HashMap::from([("mixed-kafka".to_string(), enabled_kvs("on"))]),
)]));
with_vars(
[
("RUSTFS_NOTIFY_KAFKA_ENABLE_MIXED-KAFKA", Some("on")),
("RUSTFS_NOTIFY_KAFKA_BROKERS_MIXED-KAFKA", Some("127.0.0.1:9092")),
("RUSTFS_NOTIFY_KAFKA_ENABLE_ENV-KAFKA", Some("on")),
("RUSTFS_NOTIFY_KAFKA_BROKERS_ENV-KAFKA", Some("127.0.0.1:9093")),
],
|| {
let runtime = HashMap::from([
(("mixed-kafka".to_string(), "kafka".to_string()), "online".to_string()),
(("env-kafka".to_string(), "kafka".to_string()), "online".to_string()),
]);
let merged = merge_notification_endpoints(&config, runtime);
let mixed = merged
.iter()
.find(|entry| entry.account_id == "mixed-kafka" && entry.service == "kafka")
.expect("mixed kafka target should be present");
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
let env_only = merged
.iter()
.find(|entry| entry.account_id == "env-kafka" && entry.service == "kafka")
.expect("env kafka target should be present");
assert_eq!(env_only.source, TargetEndpointSource::Env);
},
);
}
@@ -656,7 +584,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "mixed-disabled")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
assert_eq!(mixed.status, "offline");
},
);
@@ -677,7 +605,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "env-only")
.expect("env-only target should be present");
assert_eq!(env_only.source, NotificationEndpointSource::Env);
assert_eq!(env_only.source, TargetEndpointSource::Env);
assert_eq!(env_only.status, "offline");
},
);
@@ -686,7 +614,7 @@ mod tests {
#[test]
fn collect_validated_key_values_rejects_duplicate_keys() {
let allowed_keys: HashSet<&str> = ["endpoint", "auth_token"].into_iter().collect();
let key_values = vec![
let key_values = [
KeyValue {
key: "endpoint".to_string(),
value: "https://example.com/one".to_string(),
@@ -697,7 +625,13 @@ mod tests {
},
];
let err = collect_validated_key_values(&key_values, &allowed_keys, NOTIFY_WEBHOOK_SUB_SYS).unwrap_err();
let err = shared_collect_validated_key_values(
key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
NOTIFY_WEBHOOK_SUB_SYS,
"target",
)
.unwrap_err();
assert!(err.to_string().contains("duplicate key"));
}
@@ -720,7 +654,7 @@ mod tests {
.iter()
.find(|entry| entry.account_id == "PrimaryCase" && entry.service == "webhook")
.expect("mixed target should be present");
assert_eq!(mixed.source, NotificationEndpointSource::Mixed);
assert_eq!(mixed.source, TargetEndpointSource::Mixed);
},
);
}
@@ -785,6 +719,22 @@ mod tests {
);
}
#[test]
fn extract_target_params_accepts_kafka_target_type() {
let mut router = Router::new();
router
.insert("/v3/target/{target_type}/{target_name}", ())
.expect("route should insert");
let params = router
.at("/v3/target/notify_kafka/streaming")
.expect("route should match")
.params;
let (target_type, target_name) = extract_target_params(&params).expect("kafka target type should be accepted");
assert_eq!(target_type, NOTIFY_KAFKA_SUB_SYS);
assert_eq!(target_name, "streaming");
}
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
let start = src
.find(start_marker)
rustfs/src/admin/handlers/target_descriptor.rs
+193 -4
View File
@@ -14,16 +14,19 @@
use hashbrown::HashSet as HbHashSet;
use rustfs_config::{
ENABLE_KEY, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST,
ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST,
};
use rustfs_ecstore::config::Config;
use rustfs_targets::{
TargetError, check_mqtt_broker_available_with_tls, check_nats_server_available, check_pulsar_broker_available,
config::{build_nats_args, build_pulsar_args, collect_env_target_instance_ids},
TargetError, check_kafka_broker_available, check_mqtt_broker_available_with_tls, check_nats_server_available,
check_pulsar_broker_available,
config::{build_kafka_args, build_nats_args, build_pulsar_args, collect_env_target_instance_ids},
target::{TargetType, mqtt::MQTTTlsConfig},
};
use s3s::{S3Result, s3_error};
use serde::Serialize;
use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::Path;
@@ -32,6 +35,22 @@ use url::Url;
pub(crate) type EndpointKey = (String, String);
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub(crate) enum TargetEndpointSource {
Config,
Env,
Mixed,
Runtime,
}
pub(crate) struct MergedTargetEndpoint {
pub account_id: String,
pub service: String,
pub status: String,
pub source: TargetEndpointSource,
}
#[derive(Clone, Copy)]
pub(crate) enum TargetDomain {
Notify,
@@ -51,6 +70,7 @@ impl TargetDomain {
pub(crate) enum AdminTargetValidator {
Webhook,
Mqtt,
Kafka(TargetDomain),
Nats(TargetDomain),
Pulsar(TargetDomain),
}
@@ -125,12 +145,160 @@ pub(crate) fn collect_env_endpoint_keys(specs: &[AdminTargetSpec], route_prefix:
endpoints
}
pub(crate) fn classify_endpoint_source(
config_targets: &HbHashSet<EndpointKey>,
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> TargetEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
(true, true) => TargetEndpointSource::Mixed,
(true, false) => TargetEndpointSource::Config,
(false, true) => TargetEndpointSource::Env,
(false, false) => TargetEndpointSource::Runtime,
}
}
pub(crate) fn endpoint_source(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
target_type: &str,
target_name: &str,
) -> TargetEndpointSource {
let config_targets = collect_config_entry_keys(specs, config);
let env_targets = collect_env_endpoint_keys(specs, route_prefix);
let service = target_service_name(specs, target_type).unwrap_or_default();
let key = normalized_endpoint_key(target_name, service);
classify_endpoint_source(&config_targets, &env_targets, &key)
}
pub(crate) fn target_mutation_block_reason(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
target_type: &str,
target_name: &str,
target_label: &str,
) -> Option<String> {
match endpoint_source(specs, route_prefix, config, target_type, target_name) {
TargetEndpointSource::Env => Some(format!(
"{} '{}' is managed by environment variables and cannot be modified from the console",
target_label, target_name
)),
TargetEndpointSource::Mixed => Some(format!(
"{} '{}' is configured by both persisted config and environment variables; remove the environment variables first",
target_label, target_name
)),
TargetEndpointSource::Config | TargetEndpointSource::Runtime => None,
}
}
pub(crate) fn merge_target_endpoints(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
) -> Vec<MergedTargetEndpoint> {
let mut endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_endpoint_keys(specs, config);
let config_targets = collect_config_entry_keys(specs, config);
let env_targets = collect_env_endpoint_keys(specs, route_prefix);
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for key in configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
}
let status = normalized_runtime_statuses
.remove(&normalized)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
endpoints.push(MergedTargetEndpoint {
account_id: key.0,
service: key.1,
status,
source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if seen.insert(normalized.clone()) {
endpoints.push(MergedTargetEndpoint {
account_id,
service,
status,
source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
});
}
}
for key in &env_targets {
if !seen.insert(key.clone()) {
continue;
}
endpoints.push(MergedTargetEndpoint {
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_endpoint_source(&config_targets, &env_targets, key),
});
}
endpoints.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
endpoints
}
pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
target_spec(specs, target_type)
.map(|spec| spec.valid_keys.iter().copied().collect())
.unwrap_or_default()
}
pub(crate) fn collect_validated_key_values<'a, I>(
key_values: I,
allowed_keys: &HashSet<&str>,
target_type: &str,
target_label: &str,
) -> S3Result<HashMap<String, String>>
where
I: IntoIterator<Item = (&'a str, &'a str)>,
{
let mut kv_map = HashMap::new();
let mut seen = HashSet::new();
for (key, value) in key_values {
if !allowed_keys.contains(key) {
return Err(s3_error!(
InvalidArgument,
"key '{}' not allowed for {} type '{}'",
key,
target_label,
target_type
));
}
if !seen.insert(key) {
return Err(s3_error!(InvalidArgument, "duplicate key '{}' in request body", key));
}
kv_map.insert(key.to_string(), value.to_string());
}
Ok(kv_map)
}
pub(crate) async fn validate_queue_dir(queue_dir: &str) -> S3Result<()> {
if !queue_dir.is_empty() {
if !Path::new(queue_dir).is_absolute() {
@@ -159,6 +327,7 @@ pub(crate) async fn validate_target_request(
match spec.validator {
AdminTargetValidator::Webhook => validate_webhook_request(kv_map).await,
AdminTargetValidator::Mqtt => validate_mqtt_request(kv_map).await,
AdminTargetValidator::Kafka(domain) => validate_kafka_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Nats(domain) => validate_nats_request(kv_map, default_queue_dir, domain).await,
AdminTargetValidator::Pulsar(domain) => validate_pulsar_request(kv_map, default_queue_dir, domain).await,
}
@@ -274,6 +443,26 @@ async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_d
})
}
async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(KAFKA_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
if !kv_map.contains_key(KAFKA_BROKERS) {
return Err(s3_error!(InvalidArgument, "Kafka brokers are required"));
}
if !kv_map.contains_key(KAFKA_TOPIC) {
return Err(s3_error!(InvalidArgument, "Kafka topic is required"));
}
let args = build_kafka_args(&to_kvs(kv_map), default_queue_dir, domain.runtime_target_type())
.map_err(|e| s3_error!(InvalidArgument, "{}", e))?;
check_kafka_broker_available(&args).await.map_err(|e| match e {
TargetError::Configuration(_) => s3_error!(InvalidArgument, "{}", e),
_ => s3_error!(InvalidArgument, "Kafka broker check failed: {}", e),
})
}
async fn validate_pulsar_request(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,