gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-07 05:43:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(targets): unify queue/connectivity handling and coverage (#2953)

Browse Source 
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
Co-authored-by: marshawcoco <marshawcoco@gmail.com>
This commit is contained in:
houseme
2026-05-14 12:31:23 +08:00
committed by GitHub
parent bdb98598d2
commit 81754d80b3
64 changed files with 9613 additions and 2800 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/audit.rs
+20 -143
View File
@@ -14,12 +14,12 @@
use crate::admin::{
auth::validate_admin_request,
handlers::audit_runtime_config::{load_server_config_from_store, update_audit_config_and_reload},
handlers::target_descriptor::{
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
build_json_response, collect_runtime_statuses, extract_supported_target_params,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
target_mutation_block_reason as shared_target_mutation_block_reason,
},
router::{AdminOperation, Operation, S3Router},
};
@@ -27,23 +27,19 @@ use crate::auth::{check_key_valid, get_session_token};
use crate::server::{
ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, refresh_audit_module_enabled, refresh_persisted_module_switches_from_store,
};
use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_audit::factory::builtin_target_descriptors as builtin_audit_target_descriptors;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_audit::audit_system;
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
use rustfs_config::{AUDIT_DEFAULT_DIR, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_config::{AUDIT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_policy::policy::action::{Action, AdminAction};
use rustfs_targets::catalog::builtin::builtin_audit_target_admin_descriptors;
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, warn};
pub fn register_audit_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -93,7 +89,7 @@ struct AuditEndpointsResponse {
}
static AUDIT_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_audit_target_descriptors()
builtin_audit_target_admin_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
@@ -113,18 +109,6 @@ async fn authorize_audit_admin_request(req: &S3Request<Body>, action: AdminActio
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
}
fn has_any_audit_targets(config: &Config) -> bool {
for spec in audit_target_specs() {
let Some(targets) = config.0.get(spec.subsystem) else {
continue;
};
if targets.keys().any(|key| key != DEFAULT_DELIMITER) {
return true;
}
}
false
}
fn audit_target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
audit_target_specs(),
@@ -160,85 +144,7 @@ fn merge_audit_endpoints(config: &Config, runtime_statuses: HashMap<EndpointKey,
}
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
let target_type = params
.get("target_type")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
if target_service_name(audit_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type));
}
let target_name = params
.get("target_name")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
Ok((target_type, target_name))
}
async fn load_server_config_from_store() -> S3Result<Config> {
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Ok(Config::new());
};
rustfs_ecstore::config::com::read_config_without_migrate(store)
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
}
async fn apply_audit_runtime_config(config: Config) -> S3Result<()> {
let has_targets = has_any_audit_targets(&config);
if let Some(system) = audit_system() {
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
if has_targets {
system
.reload_config(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
} else {
system
.close()
.await
.map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
}
}
AuditSystemState::Stopped | AuditSystemState::Stopping => {
if has_targets {
system
.start(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
}
}
} else if has_targets {
start_global_audit_system(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
Ok(())
}
async fn update_audit_config_and_reload<F>(mut modifier: F) -> S3Result<()>
where
F: FnMut(&mut Config) -> bool,
{
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
if !modifier(&mut config) {
return Ok(());
}
rustfs_ecstore::config::com::save_server_config(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
apply_audit_runtime_config(config).await
extract_supported_target_params(audit_target_specs(), params, "audit")
}
pub struct AuditTargetConfig {}
@@ -269,28 +175,16 @@ impl Operation for AuditTargetConfig {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for audit target config: {}", e))?;
let specs = audit_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
let kvs = build_enabled_target_kvs(
specs,
audit_body.key_values.iter().map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
target_type,
AUDIT_DEFAULT_DIR,
"audit target",
)?;
)
.await?;
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported audit target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, AUDIT_DEFAULT_DIR))
.await
.map_err(|_| s3_error!(InvalidArgument, "audit target validation timed out"))??;
let mut kvs = rustfs_ecstore::config::KVS::new();
for (key, value) in kv_map {
kvs.insert(key, value);
}
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
update_audit_config_and_reload(|config| {
update_audit_config_and_reload(audit_target_specs(), |config| {
config
.0
.entry(target_type.to_lowercase())
@@ -315,25 +209,7 @@ impl Operation for ListAuditTargets {
let mut runtime_statuses = HashMap::new();
if let Some(system) = audit_system() {
let targets = system.get_target_values().await;
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
((target.id().id, target.id().name), status.to_string())
});
}
while let Some((key, status)) = futures.next().await {
runtime_statuses.insert(key, status);
}
runtime_statuses = collect_runtime_statuses(system.get_target_values().await).await;
}
let config = load_server_config_from_store().await?;
@@ -363,7 +239,7 @@ impl Operation for RemoveAuditTarget {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
update_audit_config_and_reload(|config| {
update_audit_config_and_reload(audit_target_specs(), |config| {
let mut changed = false;
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
if targets.remove(&target_name.to_lowercase()).is_some() {
@@ -384,9 +260,10 @@ impl Operation for RemoveAuditTarget {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::handlers::target_descriptor::collect_validated_key_values as shared_collect_validated_key_values;
use matchit::Router;
use rustfs_config::ENV_PREFIX;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_WEBHOOK_KEYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX};
use rustfs_ecstore::config::{KV, KVS};
use serial_test::serial;
use std::collections::{HashMap, HashSet};
rustfs/src/admin/handlers/audit_runtime_config.rs
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::handlers::target_descriptor::AdminTargetSpec;
use rustfs_audit::{audit_system, start_audit_system as start_global_audit_system, system::AuditSystemState};
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_ecstore::config::Config;
use s3s::{S3Result, s3_error};
pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Ok(Config::new());
};
rustfs_ecstore::config::com::read_config_without_migrate(store)
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
}
fn has_any_audit_targets(specs: &[AdminTargetSpec], config: &Config) -> bool {
specs.iter().any(|spec| {
config
.0
.get(spec.subsystem)
.is_some_and(|targets| targets.keys().any(|key| key != DEFAULT_DELIMITER))
})
}
pub(crate) async fn apply_audit_runtime_config(specs: &[AdminTargetSpec], config: Config) -> S3Result<()> {
let has_targets = has_any_audit_targets(specs, &config);
if let Some(system) = audit_system() {
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
if has_targets {
system
.reload_config(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to reload audit config: {}", e))?;
} else {
system
.close()
.await
.map_err(|e| s3_error!(InternalError, "failed to stop audit system: {}", e))?;
}
}
AuditSystemState::Stopped | AuditSystemState::Stopping => {
if has_targets {
system
.start(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
}
}
} else if has_targets {
start_global_audit_system(config)
.await
.map_err(|e| s3_error!(InternalError, "failed to start audit system: {}", e))?;
}
Ok(())
}
pub(crate) async fn update_audit_config_and_reload<F>(specs: &[AdminTargetSpec], mut modifier: F) -> S3Result<()>
where
F: FnMut(&mut Config) -> bool,
{
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let mut config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
if !modifier(&mut config) {
return Ok(());
}
rustfs_ecstore::config::com::save_server_config(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
apply_audit_runtime_config(specs, config).await
}
pub(crate) async fn set_audit_target_config(
specs: &[AdminTargetSpec],
subsystem: &str,
target_name: &str,
kvs: rustfs_ecstore::config::KVS,
) -> S3Result<()> {
update_audit_config_and_reload(specs, |config| {
config
.0
.entry(subsystem.to_lowercase())
.or_default()
.insert(target_name.to_lowercase(), kvs.clone());
true
})
.await
}
pub(crate) async fn remove_audit_target_config(specs: &[AdminTargetSpec], subsystem: &str, target_name: &str) -> S3Result<()> {
update_audit_config_and_reload(specs, |config| {
let mut changed = false;
if let Some(targets) = config.0.get_mut(&subsystem.to_lowercase()) {
if targets.remove(&target_name.to_lowercase()).is_some() {
changed = true;
}
if targets.is_empty() {
config.0.remove(&subsystem.to_lowercase());
}
}
changed
})
.await
}
rustfs/src/admin/handlers/event.rs
+27 -92
View File
@@ -14,12 +14,12 @@
use crate::admin::{
auth::validate_admin_request,
handlers::notify_runtime_access::{get_notification_system, load_notification_config_snapshot},
handlers::target_descriptor::{
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, allowed_target_keys,
build_json_response, collect_validated_key_values as shared_collect_validated_key_values,
AdminTargetSpec, EndpointKey, TargetEndpointSource, admin_target_spec_from_builtin, build_enabled_target_kvs,
build_json_response, collect_runtime_statuses, extract_supported_target_params,
merge_target_endpoints as shared_merge_target_endpoints, target_module_disabled_reason,
target_mutation_block_reason as shared_target_mutation_block_reason, target_service_name, target_spec,
validate_target_request,
target_mutation_block_reason as shared_target_mutation_block_reason,
},
router::{AdminOperation, Operation, S3Router},
};
@@ -28,22 +28,18 @@ use crate::server::{
ADMIN_PREFIX, RemoteAddr, is_notify_module_enabled, refresh_notify_module_enabled,
refresh_persisted_module_switches_from_store,
};
use futures::stream::{FuturesUnordered, StreamExt};
use http::StatusCode;
use hyper::Method;
use matchit::Params;
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
use rustfs_config::{ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_config::{EVENT_DEFAULT_DIR, MAX_ADMIN_REQUEST_BODY_SIZE};
use rustfs_ecstore::config::Config;
use rustfs_notify::factory::builtin_target_descriptors as builtin_notification_target_descriptors;
use rustfs_policy::policy::action::{Action, AdminAction};
use rustfs_targets::catalog::builtin::builtin_notify_target_admin_descriptors;
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::LazyLock;
use tokio::sync::Semaphore;
use tokio::time::{Duration, timeout};
use tracing::{Span, info, warn};
pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
@@ -99,7 +95,7 @@ struct NotificationEndpointsResponse {
}
static NOTIFICATION_TARGET_SPECS: LazyLock<Vec<AdminTargetSpec>> = LazyLock::new(|| {
builtin_notification_target_descriptors()
builtin_notify_target_admin_descriptors()
.into_iter()
.map(|descriptor| admin_target_spec_from_builtin(&descriptor))
.collect()
@@ -121,10 +117,6 @@ async fn authorize_notification_admin_request(req: &S3Request<Body>, action: Adm
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await
}
fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
}
fn target_mutation_block_reason(config: &Config, target_type: &str, target_name: &str) -> Option<String> {
shared_target_mutation_block_reason(
notification_target_specs(),
@@ -180,8 +172,7 @@ impl Operation for NotificationTarget {
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let ns = get_notification_system()?;
let config_snapshot = ns.config.read().await.clone();
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -196,28 +187,17 @@ impl Operation for NotificationTarget {
.map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?;
let specs = notification_target_specs();
let allowed_keys: HashSet<&str> = allowed_target_keys(specs, target_type);
let kv_map = shared_collect_validated_key_values(
let kvs = build_enabled_target_kvs(
specs,
notification_body
.key_values
.iter()
.map(|kv| (kv.key.as_str(), kv.value.as_str())),
&allowed_keys,
target_type,
EVENT_DEFAULT_DIR,
"target",
)?;
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, EVENT_DEFAULT_DIR))
.await
.map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??;
let mut kvs = rustfs_ecstore::config::KVS::new();
for (key, value) in kv_map {
kvs.insert(key, value);
}
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
)
.await?;
info!("Setting target config for type '{}', name '{}'", target_type, target_name);
ns.set_target_config(target_type, target_name, kvs)
@@ -235,29 +215,8 @@ impl Operation for ListNotificationTargets {
let span = Span::current();
let _enter = span.enter();
authorize_notification_admin_request(&req, AdminAction::GetBucketTargetAction).await?;
let ns = get_notification_system()?;
let targets = ns.get_target_values().await;
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
((target.id().id, target.id().name), status.to_string())
});
}
let mut runtime_statuses = HashMap::new();
while let Some((key, status)) = futures.next().await {
runtime_statuses.insert(key, status);
}
let config = ns.config.read().await.clone();
let (ns, config) = load_notification_config_snapshot().await?;
let runtime_statuses = collect_runtime_statuses(ns.get_target_values().await).await;
let notification_endpoints = merge_notification_endpoints(&config, runtime_statuses);
let data = serde_json::to_vec(&NotificationEndpointsResponse { notification_endpoints })
@@ -283,30 +242,15 @@ impl Operation for ListTargetsArns {
}
let ns = get_notification_system()?;
let targets = ns.get_target_values().await;
let region = req
.region
.clone()
.ok_or_else(|| s3_error!(InvalidRequest, "region not found"))?;
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = FuturesUnordered::new();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
(target.id(), status.to_string())
});
}
let mut target_statuses = Vec::new();
while let Some(target_status) = futures.next().await {
target_statuses.push(target_status);
}
let target_statuses = collect_runtime_statuses(ns.get_target_values().await)
.await
.into_iter()
.map(|((account_id, service), status)| (rustfs_targets::arn::TargetID::new(account_id, service), status))
.collect();
let data_target_arn_list = collect_online_target_arns(region.as_str(), target_statuses);
@@ -329,8 +273,7 @@ impl Operation for RemoveNotificationTarget {
if let Some(reason) = notification_target_operation_block_reason("managing notification targets from the console").await {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
let ns = get_notification_system()?;
let config_snapshot = ns.config.read().await.clone();
let (ns, config_snapshot) = load_notification_config_snapshot().await?;
if let Some(reason) = target_mutation_block_reason(&config_snapshot, target_type, target_name) {
return Err(s3_error!(InvalidRequest, "{reason}"));
}
@@ -344,27 +287,19 @@ impl Operation for RemoveNotificationTarget {
}
}
fn extract_param<'a>(params: &'a Params<'_, '_>, key: &str) -> S3Result<&'a str> {
params
.get(key)
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: '{}'", key))
}
fn extract_target_params<'a>(params: &'a Params<'_, '_>) -> S3Result<(&'a str, &'a str)> {
let target_type = extract_param(params, "target_type")?;
if target_service_name(notification_target_specs(), target_type).is_none() {
return Err(s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type));
}
let target_name = extract_param(params, "target_name")?;
Ok((target_type, target_name))
extract_supported_target_params(notification_target_specs(), params, "notification")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::handlers::target_descriptor::{
allowed_target_keys, collect_validated_key_values as shared_collect_validated_key_values,
};
use matchit::Router;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::notify::{NOTIFY_AMQP_SUB_SYS, NOTIFY_KAFKA_SUB_SYS, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS};
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY};
use rustfs_ecstore::config::{KV, KVS};
use rustfs_targets::arn::TargetID;
use serial_test::serial;
rustfs/src/admin/handlers/mod.rs
+9
View File
@@ -14,6 +14,7 @@
pub mod account_info;
pub mod audit;
mod audit_runtime_config;
pub mod bucket_meta;
pub mod event;
pub mod group;
@@ -27,7 +28,10 @@ pub mod kms_keys;
pub mod kms_management;
pub mod metrics;
pub mod module_switch;
mod notify_runtime_access;
pub mod oidc;
pub mod plugins_catalog;
pub mod plugins_instances;
pub mod policies;
pub mod pools;
pub mod profile;
@@ -57,6 +61,11 @@ mod tests {
let _account_handler = account_info::AccountInfoHandler {};
let _list_audit_targets = audit::ListAuditTargets {};
let _get_module_switches = module_switch::GetModuleSwitchesHandler {};
let _get_plugin_catalog = plugins_catalog::GetPluginCatalogHandler {};
let _list_plugin_instances = plugins_instances::ListPluginInstancesHandler {};
let _get_plugin_instance = plugins_instances::GetPluginInstanceHandler {};
let _put_plugin_instance = plugins_instances::PutPluginInstanceHandler {};
let _delete_plugin_instance = plugins_instances::DeletePluginInstanceHandler {};
let _update_module_switches = module_switch::UpdateModuleSwitchesHandler {};
let _service_handler = system::ServiceHandle {};
let _server_info_handler = system::ServerInfoHandler {};
rustfs/src/admin/handlers/notify_runtime_access.rs
+47
View File
@@ -0,0 +1,47 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::config::Config;
use s3s::{S3Result, s3_error};
use std::sync::Arc;
pub(crate) fn get_notification_system() -> S3Result<Arc<rustfs_notify::NotificationSystem>> {
rustfs_notify::notification_system().ok_or_else(|| s3_error!(InternalError, "notification system not initialized"))
}
pub(crate) async fn load_notification_config_snapshot() -> S3Result<(Arc<rustfs_notify::NotificationSystem>, Config)> {
let system = get_notification_system()?;
let config = system.config.read().await.clone();
Ok((system, config))
}
pub(crate) async fn set_notification_target_config(
subsystem: &str,
target_name: &str,
kvs: rustfs_ecstore::config::KVS,
) -> S3Result<()> {
let system = get_notification_system()?;
system
.set_target_config(subsystem, target_name, kvs)
.await
.map_err(|e| s3_error!(InternalError, "failed to set notification target config: {}", e))
}
pub(crate) async fn remove_notification_target_config(subsystem: &str, target_name: &str) -> S3Result<()> {
let system = get_notification_system()?;
system
.remove_target_config(subsystem, target_name)
.await
.map_err(|e| s3_error!(InternalError, "failed to remove notification target config: {}", e))
}
rustfs/src/admin/handlers/plugins_catalog.rs
+265
View File
@@ -0,0 +1,265 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::{
auth::validate_admin_request,
plugin_contract::{
PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind,
PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract,
},
router::{AdminOperation, Operation, S3Router},
};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
use http::{HeaderMap, HeaderValue, StatusCode};
use hyper::Method;
use matchit::Params;
use rustfs_policy::policy::action::{Action, AdminAction};
use rustfs_targets::catalog::{
builtin::builtin_audit_target_admin_descriptors, builtin::builtin_notify_target_admin_descriptors,
};
use rustfs_targets::{
BuiltinTargetAdminDescriptor, builtin_target_marketplace_manifest, builtin_target_plugin_installation,
catalog::example_external_webhook_plugin,
};
use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::Serialize;
use std::collections::HashMap;
pub fn register_plugin_catalog_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
r.insert(
Method::GET,
format!("{}{}", ADMIN_PREFIX, "/v4/plugins/catalog").as_str(),
AdminOperation(&GetPluginCatalogHandler {}),
)?;
Ok(())
}
fn target_domain_name_from_subsystem(subsystem: &str) -> PluginContractDomain {
if subsystem.starts_with("audit_") {
PluginContractDomain::Audit
} else {
PluginContractDomain::Notify
}
}
fn build_catalog_response() -> PluginCatalogResponse {
let mut plugins: HashMap<&'static str, PluginCatalogEntry> = HashMap::new();
for descriptor in builtin_notify_target_admin_descriptors()
.into_iter()
.chain(builtin_audit_target_admin_descriptors())
{
merge_catalog_descriptor(&mut plugins, &descriptor);
}
let mut plugins = plugins.into_values().collect::<Vec<_>>();
plugins.push(example_external_webhook_plugin_entry());
plugins.sort_by(|a, b| a.target_type.cmp(&b.target_type));
for plugin in &mut plugins {
plugin.supported_domains.sort();
plugin.domain_configs.sort_by_key(|a| a.domain);
}
PluginCatalogResponse { plugins }
}
fn example_external_webhook_plugin_entry() -> PluginCatalogEntry {
let example = example_external_webhook_plugin();
let manifest = example.manifest;
PluginCatalogEntry {
plugin_id: manifest.plugin_id.to_string(),
target_type: manifest.target_type.to_string(),
display_name: manifest.display_name.to_string(),
provider: manifest.provider.to_string(),
version: manifest.version.to_string(),
packaging: PluginContractPackaging::from(manifest.packaging),
entrypoint_kind: PluginContractEntrypointKind::from(manifest.entrypoint_kind),
api_compatibility_version: manifest.api_compatibility_version.to_string(),
runtime_contract: PluginRuntimeContract::from(manifest.runtime_contract),
distribution: manifest.distribution.map(PluginDistributionContract::from),
supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(),
secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(),
domain_configs: vec![PluginCatalogDomainEntry {
domain: PluginContractDomain::Notify,
subsystem: "notify_webhook".to_string(),
valid_fields: example.valid_fields,
}],
installation: Some(example.installation.into()),
}
}
fn merge_catalog_descriptor(plugins: &mut HashMap<&'static str, PluginCatalogEntry>, descriptor: &BuiltinTargetAdminDescriptor) {
let manifest = descriptor.manifest();
let marketplace = builtin_target_marketplace_manifest(manifest.target_type);
let domain = target_domain_name_from_subsystem(descriptor.admin_metadata().subsystem());
let domain_entry = PluginCatalogDomainEntry {
domain,
subsystem: descriptor.admin_metadata().subsystem().to_string(),
valid_fields: descriptor.valid_fields().iter().map(|field| (*field).to_string()).collect(),
};
let entry = plugins.entry(manifest.plugin_id).or_insert_with(|| PluginCatalogEntry {
plugin_id: manifest.plugin_id.to_string(),
target_type: manifest.target_type.to_string(),
display_name: manifest.display_name.to_string(),
provider: manifest.provider.to_string(),
version: manifest.version.to_string(),
packaging: PluginContractPackaging::from(marketplace.packaging),
entrypoint_kind: PluginContractEntrypointKind::from(marketplace.entrypoint_kind),
api_compatibility_version: marketplace.api_compatibility_version.to_string(),
runtime_contract: PluginRuntimeContract::from(marketplace.runtime_contract),
distribution: marketplace.distribution.map(PluginDistributionContract::from),
supported_domains: manifest.supported_domains.iter().copied().map(Into::into).collect(),
secret_fields: manifest.secret_fields.iter().map(|field| (*field).to_string()).collect(),
domain_configs: Vec::new(),
installation: Some(builtin_target_plugin_installation(manifest).into()),
});
if !entry.domain_configs.iter().any(|existing| existing.domain == domain) {
entry.domain_configs.push(domain_entry);
}
}
async fn authorize_plugin_catalog_request(req: &S3Request<Body>) -> S3Result<()> {
let Some(input_cred) = &req.credentials else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
let (cred, owner) =
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
validate_admin_request(
&req.headers,
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0)),
)
.await
}
fn build_json_response(
status: StatusCode,
body: &impl Serialize,
request_id: Option<&HeaderValue>,
) -> S3Result<S3Response<(StatusCode, Body)>> {
let data = serde_json::to_vec(body).map_err(|e| s3_error!(InternalError, "failed to serialize response: {}", e))?;
let mut header = HeaderMap::new();
header.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
if let Some(value) = request_id {
header.insert("x-request-id", value.clone());
}
Ok(S3Response::with_headers((status, Body::from(data)), header))
}
pub struct GetPluginCatalogHandler {}
#[async_trait::async_trait]
impl Operation for GetPluginCatalogHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
authorize_plugin_catalog_request(&req).await?;
build_json_response(StatusCode::OK, &build_catalog_response(), req.headers.get("x-request-id"))
}
}
#[cfg(test)]
mod tests {
use super::build_catalog_response;
use crate::admin::plugin_contract::{
PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginRuntimeTransport,
};
#[test]
fn plugin_catalog_handlers_require_admin_authorization_contract() {
let src = include_str!("plugins_catalog.rs");
let handler_block = extract_block_between_markers(src, "impl Operation for GetPluginCatalogHandler", "#[cfg(test)]");
assert!(
handler_block.contains("authorize_plugin_catalog_request(&req).await?;"),
"plugin catalog GET should require admin authorization"
);
}
#[test]
fn plugin_catalog_contains_representative_builtin_targets() {
let response = build_catalog_response();
let webhook = response
.plugins
.iter()
.find(|plugin| plugin.plugin_id == "builtin:webhook")
.expect("builtin webhook plugin should be present");
assert_eq!(webhook.target_type, "webhook");
assert_eq!(webhook.display_name, "Webhook");
assert_eq!(webhook.packaging, PluginContractPackaging::Builtin);
assert_eq!(webhook.entrypoint_kind, PluginContractEntrypointKind::Builtin);
assert_eq!(webhook.api_compatibility_version, "rustfs.target-plugin.v1");
assert_eq!(webhook.runtime_contract.protocol_version, "rustfs.target-runtime.v1");
assert_eq!(webhook.runtime_contract.transport, PluginRuntimeTransport::InProcess);
assert_eq!(webhook.distribution, None);
assert!(webhook.supported_domains.contains(&PluginContractDomain::Audit));
assert!(webhook.supported_domains.contains(&PluginContractDomain::Notify));
assert!(
webhook
.domain_configs
.iter()
.any(|domain| domain.subsystem == "audit_webhook")
);
assert!(
webhook
.domain_configs
.iter()
.any(|domain| domain.subsystem == "notify_webhook")
);
let kafka = response
.plugins
.iter()
.find(|plugin| plugin.plugin_id == "builtin:kafka")
.expect("builtin kafka plugin should be present");
assert_eq!(kafka.target_type, "kafka");
assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "audit_kafka"));
assert!(kafka.domain_configs.iter().any(|domain| domain.subsystem == "notify_kafka"));
}
#[test]
fn plugin_catalog_exposes_secret_fields_only_as_metadata() {
let response = build_catalog_response();
let webhook = response
.plugins
.iter()
.find(|plugin| plugin.plugin_id == "builtin:webhook")
.expect("builtin webhook plugin should be present");
assert!(webhook.secret_fields.contains(&"auth_token".to_string()));
assert!(!webhook.secret_fields.iter().any(|field| field.contains("https://")));
assert!(!webhook.secret_fields.iter().any(|field| field.contains("password=")));
}
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
let start = src
.find(start_marker)
.unwrap_or_else(|| panic!("Expected marker `{start_marker}` in source"));
let after_start = &src[start..];
let end = after_start
.find(end_marker)
.unwrap_or_else(|| panic!("Expected end marker `{end_marker}` in source"));
&after_start[..end]
}
}
rustfs/src/admin/handlers/plugins_instances.rs
+1630
View File
File diff suppressed because it is too large Load Diff
rustfs/src/admin/handlers/target_descriptor.rs
+404 -279
View File
@@ -12,22 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use futures::StreamExt;
use futures::future::BoxFuture;
use hashbrown::HashSet as HbHashSet;
use http::{HeaderMap, HeaderValue, StatusCode};
use rustfs_config::{
AMQP_QUEUE_DIR, ENABLE_KEY, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_TLS_CA,
MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME,
MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
AMQP_QUEUE_DIR, ENABLE_KEY, EnableState, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_TOPIC, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS,
MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC,
MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_QUEUE_DIR, POSTGRES_QUEUE_DIR, REDIS_QUEUE_DIR,
};
use rustfs_ecstore::config::Config;
use rustfs_ecstore::config::{Config, KVS};
use rustfs_targets::SharedTarget;
use rustfs_targets::{
BuiltinTargetDescriptor, TargetError, TargetRequestValidator, check_amqp_broker_available, check_kafka_broker_available,
check_mqtt_broker_available_with_tls, check_mysql_server_available, check_nats_server_available,
check_postgres_server_available, check_pulsar_broker_available, check_redis_server_available,
BuiltinTargetAdminDescriptor, TargetAdminMetadata, TargetDomain, TargetError, TargetRequestValidator,
check_amqp_broker_available, check_kafka_broker_available, check_mqtt_broker_available_with_tls,
check_mysql_server_available, check_nats_server_available, check_postgres_server_available, check_pulsar_broker_available,
check_redis_server_available,
config::{
build_amqp_args, build_kafka_args, build_mysql_args, build_nats_args, build_postgres_args, build_pulsar_args,
build_redis_args, collect_env_target_instance_ids, validate_redis_config,
TargetPluginInstanceCompatDescriptor, TargetPluginInstanceRecord, build_amqp_args, build_kafka_args, build_mysql_args,
build_nats_args, build_postgres_args, build_pulsar_args, build_redis_args, normalize_target_plugin_instances,
validate_redis_config,
},
manifest::builtin_target_manifest,
target::{TargetType, mqtt::MQTTTlsConfig},
};
use s3s::{Body, S3Response, S3Result, header::CONTENT_TYPE, s3_error};
@@ -36,12 +42,14 @@ use std::collections::{HashMap, HashSet};
use std::io::{Error, ErrorKind};
use std::path::Path;
use std::sync::Arc;
use tokio::time::{Duration, sleep};
use tokio::sync::Semaphore;
use tokio::time::{Duration, sleep, timeout};
use url::Url;
pub(crate) type EndpointKey = (String, String);
type AdminRequestValidatorFn =
Arc<dyn Fn(&HashMap<String, String>, &str) -> futures::future::BoxFuture<'static, S3Result<()>> + Send + Sync>;
Arc<dyn for<'a> Fn(&'a HashMap<String, String>, &'a str) -> BoxFuture<'a, S3Result<()>> + Send + Sync>;
type DomainScopedValidatorFn = for<'a> fn(&'a HashMap<String, String>, &'a str, TargetDomain) -> BoxFuture<'a, S3Result<()>>;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
@@ -59,28 +67,26 @@ pub(crate) struct MergedTargetEndpoint {
pub source: TargetEndpointSource,
}
#[derive(Clone, Copy, Debug)]
pub(crate) enum TargetDomain {
Notify,
Audit,
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TargetInstanceReadModel {
pub canonical_id: String,
pub plugin_id: String,
pub domain: TargetDomain,
pub subsystem: String,
pub account_id: String,
pub service: String,
pub status: String,
pub runtime_present: bool,
pub source: TargetEndpointSource,
pub enabled: bool,
pub config: KVS,
}
impl TargetDomain {
pub(crate) fn runtime_target_type(self) -> TargetType {
match self {
TargetDomain::Notify => TargetType::NotifyEvent,
TargetDomain::Audit => TargetType::AuditLog,
}
}
}
impl From<TargetType> for TargetDomain {
fn from(value: TargetType) -> Self {
match value {
TargetType::NotifyEvent => TargetDomain::Notify,
TargetType::AuditLog => TargetDomain::Audit,
}
}
struct TargetEndpointSnapshot {
normalized_instances: Vec<TargetPluginInstanceRecord>,
configured_keys: Vec<EndpointKey>,
config_targets: HbHashSet<EndpointKey>,
env_targets: HbHashSet<EndpointKey>,
}
#[derive(Clone)]
@@ -91,69 +97,55 @@ pub(crate) struct AdminTargetSpec {
validator: AdminRequestValidatorFn,
}
pub(crate) fn admin_target_spec_from_builtin<E>(descriptor: &BuiltinTargetDescriptor<E>) -> AdminTargetSpec
where
E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
{
pub(crate) fn admin_target_spec_from_builtin(descriptor: &BuiltinTargetAdminDescriptor) -> AdminTargetSpec {
let admin = descriptor.admin_metadata();
AdminTargetSpec {
subsystem: descriptor.subsystem(),
service: descriptor.plugin().target_type(),
valid_keys: descriptor.plugin().valid_fields(),
validator: match descriptor.request_validator() {
TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
TargetRequestValidator::Amqp(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_amqp_request_entry)
} else {
Arc::new(validate_notify_amqp_request_entry)
}
}
TargetRequestValidator::Kafka(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_kafka_request_entry)
} else {
Arc::new(validate_notify_kafka_request_entry)
}
}
TargetRequestValidator::MySql(target_type) => {
Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type))
}
TargetRequestValidator::Nats(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_nats_request_entry)
} else {
Arc::new(validate_notify_nats_request_entry)
}
}
TargetRequestValidator::Postgres(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_postgres_request_entry)
} else {
Arc::new(validate_notify_postgres_request_entry)
}
}
TargetRequestValidator::Pulsar(target_type) => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
Arc::new(validate_audit_pulsar_request_entry)
} else {
Arc::new(validate_notify_pulsar_request_entry)
}
}
TargetRequestValidator::Redis {
default_channel,
target_type,
} => {
if matches!(TargetDomain::from(target_type), TargetDomain::Audit) {
validate_audit_redis_request_entry(default_channel)
} else {
validate_notify_redis_request_entry(default_channel)
}
}
},
subsystem: admin.subsystem(),
service: descriptor.manifest().target_type,
valid_keys: descriptor.valid_fields(),
validator: validator_from_metadata(admin),
}
}
fn validator_from_metadata(metadata: TargetAdminMetadata) -> AdminRequestValidatorFn {
match metadata.request_validator() {
TargetRequestValidator::Webhook => Arc::new(validate_webhook_request_entry),
TargetRequestValidator::Mqtt => Arc::new(validate_mqtt_request_entry),
TargetRequestValidator::Amqp(target_type) => {
domain_request_validator(TargetDomain::from(target_type), validate_amqp_request)
}
TargetRequestValidator::Kafka(target_type) => {
domain_request_validator(TargetDomain::from(target_type), validate_kafka_request)
}
TargetRequestValidator::MySql(target_type) => {
Arc::new(move |kv_map, default_queue_dir| validate_mysql_request_entry(kv_map, default_queue_dir, target_type))
}
TargetRequestValidator::Nats(target_type) => {
domain_request_validator(TargetDomain::from(target_type), validate_nats_request)
}
TargetRequestValidator::Postgres(target_type) => {
domain_request_validator(TargetDomain::from(target_type), validate_postgres_request)
}
TargetRequestValidator::Pulsar(target_type) => {
domain_request_validator(TargetDomain::from(target_type), validate_pulsar_request)
}
TargetRequestValidator::Redis {
default_channel,
target_type,
} => redis_request_validator(TargetDomain::from(target_type), default_channel),
}
}
fn domain_request_validator(domain: TargetDomain, validator: DomainScopedValidatorFn) -> AdminRequestValidatorFn {
Arc::new(move |kv_map, default_queue_dir| validator(kv_map, default_queue_dir, domain))
}
fn redis_request_validator(domain: TargetDomain, default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map, default_queue_dir| {
Box::pin(validate_redis_request(kv_map, default_queue_dir, domain, default_channel))
})
}
impl std::fmt::Debug for AdminTargetSpec {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AdminTargetSpec")
@@ -182,54 +174,26 @@ pub(crate) fn target_service_name(specs: &[AdminTargetSpec], target_type: &str)
target_spec(specs, target_type).map(|spec| spec.service)
}
pub(crate) fn collect_configured_endpoint_keys(specs: &[AdminTargetSpec], config: &Config) -> Vec<EndpointKey> {
let mut endpoints = Vec::new();
for spec in specs {
let Some(targets) = config.0.get(spec.subsystem) else {
continue;
};
for (target_name, kvs) in targets {
if target_name == rustfs_config::DEFAULT_DELIMITER {
continue;
}
let enabled = kvs.lookup(ENABLE_KEY).as_deref().map(config_enable_is_on).unwrap_or(false);
if enabled {
endpoints.push((target_name.clone(), spec.service.to_string()));
}
}
pub(crate) fn extract_supported_target_params<'a>(
specs: &[AdminTargetSpec],
params: &'a matchit::Params<'_, '_>,
unsupported_target_label: &str,
) -> S3Result<(&'a str, &'a str)> {
let target_type = params
.get("target_type")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_type'"))?;
if target_service_name(specs, target_type).is_none() {
return Err(s3_error!(
InvalidArgument,
"unsupported {} target type: '{}'",
unsupported_target_label,
target_type
));
}
endpoints
}
pub(crate) fn collect_config_entry_keys(specs: &[AdminTargetSpec], config: &Config) -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for spec in specs {
let Some(targets) = config.0.get(spec.subsystem) else {
continue;
};
for target_name in targets.keys() {
if target_name == rustfs_config::DEFAULT_DELIMITER {
continue;
}
endpoints.insert(normalized_endpoint_key(target_name, spec.service));
}
}
endpoints
}
pub(crate) fn collect_env_endpoint_keys(specs: &[AdminTargetSpec], route_prefix: &str) -> HbHashSet<EndpointKey> {
let mut endpoints = HbHashSet::new();
for spec in specs {
let valid_keys = spec.valid_keys.iter().map(|key| (*key).to_string()).collect::<HashSet<_>>();
for instance_id in collect_env_target_instance_ids(route_prefix, spec.service, &valid_keys) {
if instance_id != rustfs_config::DEFAULT_DELIMITER && !instance_id.is_empty() {
endpoints.insert(normalized_endpoint_key(&instance_id, spec.service));
}
}
}
endpoints
let target_name = params
.get("target_name")
.ok_or_else(|| s3_error!(InvalidArgument, "missing required parameter: 'target_name'"))?;
Ok((target_type, target_name))
}
pub(crate) fn classify_endpoint_source(
@@ -237,7 +201,11 @@ pub(crate) fn classify_endpoint_source(
env_targets: &HbHashSet<EndpointKey>,
key: &EndpointKey,
) -> TargetEndpointSource {
match (config_targets.contains(key), env_targets.contains(key)) {
classify_endpoint_source_flags(config_targets.contains(key), env_targets.contains(key))
}
fn classify_endpoint_source_flags(has_config_source: bool, has_env_source: bool) -> TargetEndpointSource {
match (has_config_source, has_env_source) {
(true, true) => TargetEndpointSource::Mixed,
(true, false) => TargetEndpointSource::Config,
(false, true) => TargetEndpointSource::Env,
@@ -252,11 +220,10 @@ pub(crate) fn endpoint_source(
target_type: &str,
target_name: &str,
) -> TargetEndpointSource {
let config_targets = collect_config_entry_keys(specs, config);
let env_targets = collect_env_endpoint_keys(specs, route_prefix);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
let service = target_service_name(specs, target_type).unwrap_or_default();
let key = normalized_endpoint_key(target_name, service);
classify_endpoint_source(&config_targets, &env_targets, &key)
classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &key)
}
pub(crate) fn target_mutation_block_reason(
@@ -301,6 +268,33 @@ pub(crate) fn build_json_response(
S3Response::with_headers((status, body), header)
}
pub(crate) async fn collect_runtime_statuses<E>(targets: Vec<SharedTarget<E>>) -> HashMap<EndpointKey, String>
where
E: Send + Sync + 'static + Clone + serde::Serialize + serde::de::DeserializeOwned,
{
let semaphore = Arc::new(Semaphore::new(10));
let mut futures = futures::stream::FuturesUnordered::new();
for target in targets {
let sem = Arc::clone(&semaphore);
futures.push(async move {
let _permit = sem.acquire().await;
let status = match tokio::time::timeout(Duration::from_secs(3), target.is_active()).await {
Ok(Ok(true)) => "online",
_ => "offline",
};
((target.id().id, target.id().name), status.to_string())
});
}
let mut runtime_statuses = HashMap::new();
while let Some((key, status)) = futures.next().await {
runtime_statuses.insert(key, status);
}
runtime_statuses
}
pub(crate) fn merge_target_endpoints(
specs: &[AdminTargetSpec],
route_prefix: &str,
@@ -309,9 +303,7 @@ pub(crate) fn merge_target_endpoints(
) -> Vec<MergedTargetEndpoint> {
let mut endpoints = Vec::new();
let mut seen = HashSet::new();
let configured_keys = collect_configured_endpoint_keys(specs, config);
let config_targets = collect_config_entry_keys(specs, config);
let env_targets = collect_env_endpoint_keys(specs, route_prefix);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
for ((account_id, service), status) in runtime_statuses {
@@ -321,7 +313,7 @@ pub(crate) fn merge_target_endpoints(
.or_insert((account_id, service, status));
}
for key in configured_keys {
for key in snapshot.configured_keys {
let normalized = normalized_endpoint_key(&key.0, &key.1);
if !seen.insert(normalized.clone()) {
continue;
@@ -336,7 +328,7 @@ pub(crate) fn merge_target_endpoints(
account_id: key.0,
service: key.1,
status,
source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
});
}
@@ -346,12 +338,12 @@ pub(crate) fn merge_target_endpoints(
account_id,
service,
status,
source: classify_endpoint_source(&config_targets, &env_targets, &normalized),
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, &normalized),
});
}
}
for key in &env_targets {
for key in &snapshot.env_targets {
if !seen.insert(key.clone()) {
continue;
}
@@ -360,7 +352,7 @@ pub(crate) fn merge_target_endpoints(
account_id: key.0.clone(),
service: key.1.clone(),
status: "offline".to_string(),
source: classify_endpoint_source(&config_targets, &env_targets, key),
source: classify_endpoint_source(&snapshot.config_targets, &snapshot.env_targets, key),
});
}
@@ -368,6 +360,96 @@ pub(crate) fn merge_target_endpoints(
endpoints
}
pub(crate) fn canonical_target_instance_id(plugin_id: &str, domain: TargetDomain, instance_id: &str) -> String {
format!("{plugin_id}:{}:{}", canonical_domain_label(domain), instance_id.to_lowercase())
}
pub(crate) fn collect_target_instances(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
) -> Vec<TargetInstanceReadModel> {
let mut instances = Vec::new();
let mut seen = HashSet::new();
let mut normalized_runtime_statuses: HashMap<EndpointKey, (String, String, String)> = HashMap::new();
let domain = inferred_target_domain(route_prefix);
let snapshot = collect_endpoint_snapshot(specs, route_prefix, config);
for ((account_id, service), status) in runtime_statuses {
let normalized = normalized_endpoint_key(&account_id, &service);
normalized_runtime_statuses
.entry(normalized)
.or_insert((account_id, service, status));
}
for instance in snapshot.normalized_instances {
let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type);
if !seen.insert(key.clone()) {
continue;
}
let runtime_present = normalized_runtime_statuses.contains_key(&key);
let status = normalized_runtime_statuses
.remove(&key)
.map(|(_, _, status)| status)
.unwrap_or_else(|| "offline".to_string());
let source = classify_endpoint_source_flags(instance_has_config_entry(&instance), instance_has_env_entry(&instance));
instances.push(TargetInstanceReadModel {
canonical_id: canonical_target_instance_id(&instance.plugin_id, domain, &instance.instance_id),
plugin_id: instance.plugin_id,
domain,
subsystem: instance.subsystem,
account_id: instance.instance_id,
service: instance.target_type,
status,
runtime_present,
source,
enabled: instance.enabled,
config: instance.effective_config,
});
}
for (normalized, (account_id, service, status)) in normalized_runtime_statuses {
if !seen.insert(normalized) {
continue;
}
let (plugin_id, subsystem): (String, String) = target_spec_by_service(specs, &service)
.map(|spec| (builtin_target_manifest(spec.service).plugin_id.to_string(), spec.subsystem.to_string()))
.unwrap_or_else(|| ("custom:target".to_string(), format!("{}_{}", canonical_domain_label(domain), service)));
instances.push(TargetInstanceReadModel {
canonical_id: canonical_target_instance_id(&plugin_id, domain, &account_id),
plugin_id,
domain,
subsystem,
account_id,
service,
status,
runtime_present: true,
source: TargetEndpointSource::Runtime,
enabled: true,
config: KVS::new(),
});
}
instances.sort_by(|a, b| a.service.cmp(&b.service).then_with(|| a.account_id.cmp(&b.account_id)));
instances
}
pub(crate) fn find_target_instance(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
runtime_statuses: HashMap<EndpointKey, String>,
canonical_id: &str,
) -> Option<TargetInstanceReadModel> {
collect_target_instances(specs, route_prefix, config, runtime_statuses)
.into_iter()
.find(|instance| instance.canonical_id == canonical_id)
}
pub(crate) fn allowed_target_keys(specs: &[AdminTargetSpec], target_type: &str) -> HashSet<&'static str> {
target_spec(specs, target_type)
.map(|spec| spec.valid_keys.iter().copied().collect())
@@ -435,8 +517,109 @@ pub(crate) async fn validate_target_request(
spec.validate_request(kv_map, default_queue_dir).await
}
fn config_enable_is_on(value: &str) -> bool {
matches!(value.trim().to_ascii_lowercase().as_str(), "on" | "true" | "yes" | "1")
pub(crate) async fn build_enabled_target_kvs<'a, I>(
specs: &[AdminTargetSpec],
key_values: I,
target_type: &str,
default_queue_dir: &str,
target_label: &str,
) -> S3Result<KVS>
where
I: IntoIterator<Item = (&'a str, &'a str)>,
{
let allowed_keys = allowed_target_keys(specs, target_type);
let kv_map = collect_validated_key_values(key_values, &allowed_keys, target_type, target_label)?;
let spec = target_spec(specs, target_type)
.ok_or_else(|| s3_error!(InvalidArgument, "unsupported target type: '{}'", target_type))?;
timeout(Duration::from_secs(10), validate_target_request(spec, &kv_map, default_queue_dir))
.await
.map_err(|_| s3_error!(InvalidArgument, "target validation timed out"))??;
let mut kvs = KVS::new();
for (key, value) in kv_map {
kvs.insert(key, value);
}
kvs.insert(ENABLE_KEY.to_string(), EnableState::On.to_string());
Ok(kvs)
}
fn instance_has_config_entry(instance: &TargetPluginInstanceRecord) -> bool {
instance.source_hints.has_file_instance
}
fn instance_has_env_entry(instance: &TargetPluginInstanceRecord) -> bool {
instance.source_hints.has_env_instance
}
fn normalized_target_instances(
specs: &[AdminTargetSpec],
route_prefix: &str,
config: &Config,
) -> Vec<TargetPluginInstanceRecord> {
specs
.iter()
.flat_map(|spec| {
normalize_target_plugin_instances(
config,
&TargetPluginInstanceCompatDescriptor {
domain: inferred_target_domain(route_prefix),
plugin_id: builtin_target_manifest(spec.service).plugin_id,
target_type: spec.service,
subsystem: spec.subsystem,
route_prefix,
valid_fields: spec.valid_keys,
},
)
})
.collect()
}
fn inferred_target_domain(route_prefix: &str) -> TargetDomain {
match route_prefix {
rustfs_config::notify::NOTIFY_ROUTE_PREFIX => TargetDomain::Notify,
rustfs_config::audit::AUDIT_ROUTE_PREFIX => TargetDomain::Audit,
_ => TargetDomain::Notify,
}
}
fn canonical_domain_label(domain: TargetDomain) -> &'static str {
match domain {
TargetDomain::Notify => "notify",
TargetDomain::Audit => "audit",
}
}
fn target_spec_by_service<'a>(specs: &'a [AdminTargetSpec], service: &str) -> Option<&'a AdminTargetSpec> {
specs.iter().find(|spec| spec.service == service)
}
fn collect_endpoint_snapshot(specs: &[AdminTargetSpec], route_prefix: &str, config: &Config) -> TargetEndpointSnapshot {
let normalized_instances = normalized_target_instances(specs, route_prefix, config);
let mut configured_keys = Vec::new();
let mut config_targets = HbHashSet::new();
let mut env_targets = HbHashSet::new();
for instance in &normalized_instances {
let key = normalized_endpoint_key(&instance.instance_id, &instance.target_type);
if instance_has_config_entry(instance) {
config_targets.insert(key.clone());
if instance.enabled {
configured_keys.push((instance.instance_id.clone(), instance.target_type.clone()));
}
}
if instance_has_env_entry(instance) {
env_targets.insert(key);
}
}
TargetEndpointSnapshot {
normalized_instances,
configured_keys,
config_targets,
env_targets,
}
}
async fn retry_with_backoff<F, Fut, T>(mut operation: F, max_attempts: usize, base_delay: Duration) -> Result<T, Error>
@@ -489,12 +672,11 @@ async fn validate_webhook_request(kv_map: &HashMap<String, String>) -> S3Result<
Ok(())
}
fn validate_webhook_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_webhook_request(&kv_map).await })
fn validate_webhook_request_entry<'a>(
kv_map: &'a HashMap<String, String>,
_default_queue_dir: &'a str,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(validate_webhook_request(kv_map))
}
async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()> {
@@ -541,131 +723,34 @@ async fn validate_mqtt_request(kv_map: &HashMap<String, String>) -> S3Result<()>
Ok(())
}
fn validate_mqtt_request_entry(
kv_map: &HashMap<String, String>,
_default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
Box::pin(async move { validate_mqtt_request(&kv_map).await })
fn validate_mqtt_request_entry<'a>(
kv_map: &'a HashMap<String, String>,
_default_queue_dir: &'a str,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(validate_mqtt_request(kv_map))
}
fn validate_notify_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_nats_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_nats_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_kafka_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_kafka_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_amqp_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_amqp_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_pulsar_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_pulsar_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_mysql_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
fn validate_mysql_request_entry<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
target_type: TargetType,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_mysql_request(&kv_map, &default_queue_dir, target_type).await })
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(validate_mysql_request(kv_map, default_queue_dir, target_type))
}
fn validate_notify_postgres_request_entry(
fn validate_nats_request<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
domain: TargetDomain,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(async move { validate_nats_request_impl(kv_map, default_queue_dir, domain).await })
}
async fn validate_nats_request_impl(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Notify).await })
}
fn validate_audit_postgres_request_entry(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
) -> futures::future::BoxFuture<'static, S3Result<()>> {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_postgres_request(&kv_map, &default_queue_dir, TargetDomain::Audit).await })
}
fn validate_notify_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Notify, default_channel).await })
})
}
fn validate_audit_redis_request_entry(default_channel: &'static str) -> AdminRequestValidatorFn {
Arc::new(move |kv_map: &HashMap<String, String>, default_queue_dir: &str| {
let kv_map = kv_map.clone();
let default_queue_dir = default_queue_dir.to_string();
Box::pin(async move { validate_redis_request(&kv_map, &default_queue_dir, TargetDomain::Audit, default_channel).await })
})
}
async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
domain: TargetDomain,
) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get("queue_dir") {
validate_queue_dir(queue_dir.as_str()).await?;
}
@@ -677,7 +762,19 @@ async fn validate_nats_request(kv_map: &HashMap<String, String>, default_queue_d
})
}
async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
fn validate_kafka_request<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
domain: TargetDomain,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(async move { validate_kafka_request_impl(kv_map, default_queue_dir, domain).await })
}
async fn validate_kafka_request_impl(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(KAFKA_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
@@ -697,7 +794,19 @@ async fn validate_kafka_request(kv_map: &HashMap<String, String>, default_queue_
})
}
async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_dir: &str, domain: TargetDomain) -> S3Result<()> {
fn validate_amqp_request<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
domain: TargetDomain,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(async move { validate_amqp_request_impl(kv_map, default_queue_dir, domain).await })
}
async fn validate_amqp_request_impl(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
) -> S3Result<()> {
if let Some(queue_dir) = kv_map.get(AMQP_QUEUE_DIR) {
validate_queue_dir(queue_dir.as_str()).await?;
}
@@ -709,7 +818,15 @@ async fn validate_amqp_request(kv_map: &HashMap<String, String>, default_queue_d
})
}
async fn validate_pulsar_request(
fn validate_pulsar_request<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
domain: TargetDomain,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(async move { validate_pulsar_request_impl(kv_map, default_queue_dir, domain).await })
}
async fn validate_pulsar_request_impl(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
@@ -742,7 +859,15 @@ async fn validate_mysql_request(
})
}
async fn validate_postgres_request(
fn validate_postgres_request<'a>(
kv_map: &'a HashMap<String, String>,
default_queue_dir: &'a str,
domain: TargetDomain,
) -> BoxFuture<'a, S3Result<()>> {
Box::pin(async move { validate_postgres_request_impl(kv_map, default_queue_dir, domain).await })
}
async fn validate_postgres_request_impl(
kv_map: &HashMap<String, String>,
default_queue_dir: &str,
domain: TargetDomain,
rustfs/src/admin/mod.rs
+5 -2
View File
@@ -15,6 +15,7 @@
mod auth;
pub mod console;
pub mod handlers;
mod plugin_contract;
pub mod router;
pub mod service;
pub mod site_replication_identity;
@@ -26,8 +27,8 @@ mod console_test;
mod route_registration_test;
use handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication,
site_replication, sts, system, tier, user,
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin, quota,
rebalance, replication, site_replication, sts, system, tier, user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
@@ -58,6 +59,8 @@ pub fn make_admin_route(console_enabled: bool) -> std::io::Result<impl S3Route>
bucket_meta::register_bucket_meta_route(&mut r)?;
audit::register_audit_target_route(&mut r)?;
module_switch::register_module_switch_route(&mut r)?;
plugins_catalog::register_plugin_catalog_route(&mut r)?;
plugins_instances::register_plugin_instance_route(&mut r)?;
replication::register_replication_route(&mut r)?;
site_replication::register_site_replication_route(&mut r)?;
rustfs/src/admin/plugin_contract.rs
+577
View File
@@ -0,0 +1,577 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_targets::{
TargetDomain, TargetPluginArtifactManifest, TargetPluginDistributionManifest, TargetPluginEnableState,
TargetPluginEntrypointKind, TargetPluginExternalRuntimeContract, TargetPluginInstallState, TargetPluginInstallation,
TargetPluginOperationalState, TargetPluginPackaging, TargetPluginRuntimeState, TargetPluginRuntimeTransport,
};
use serde::Serialize;
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginContractDomain {
Audit,
Notify,
}
impl From<TargetDomain> for PluginContractDomain {
fn from(value: TargetDomain) -> Self {
match value {
TargetDomain::Audit => Self::Audit,
TargetDomain::Notify => Self::Notify,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginContractPackaging {
Builtin,
External,
}
impl From<TargetPluginPackaging> for PluginContractPackaging {
fn from(value: TargetPluginPackaging) -> Self {
match value {
TargetPluginPackaging::Builtin => Self::Builtin,
TargetPluginPackaging::External => Self::External,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginContractEntrypointKind {
Builtin,
Sidecar,
Wasm,
}
impl From<TargetPluginEntrypointKind> for PluginContractEntrypointKind {
fn from(value: TargetPluginEntrypointKind) -> Self {
match value {
TargetPluginEntrypointKind::Builtin => Self::Builtin,
TargetPluginEntrypointKind::Sidecar => Self::Sidecar,
TargetPluginEntrypointKind::Wasm => Self::Wasm,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginRuntimeTransport {
InProcess,
Grpc,
WasmHost,
}
impl From<TargetPluginRuntimeTransport> for PluginRuntimeTransport {
fn from(value: TargetPluginRuntimeTransport) -> Self {
match value {
TargetPluginRuntimeTransport::InProcess => Self::InProcess,
TargetPluginRuntimeTransport::Grpc => Self::Grpc,
TargetPluginRuntimeTransport::WasmHost => Self::WasmHost,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginInstallState {
NotInstalled,
Installed,
InstallFailed,
}
impl From<TargetPluginInstallState> for PluginInstallState {
fn from(value: TargetPluginInstallState) -> Self {
match value {
TargetPluginInstallState::NotInstalled => Self::NotInstalled,
TargetPluginInstallState::Installed => Self::Installed,
TargetPluginInstallState::InstallFailed => Self::InstallFailed,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginEnableState {
Enabled,
Disabled,
}
impl From<TargetPluginEnableState> for PluginEnableState {
fn from(value: TargetPluginEnableState) -> Self {
match value {
TargetPluginEnableState::Enabled => Self::Enabled,
TargetPluginEnableState::Disabled => Self::Disabled,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginOperationalRuntimeState {
Running,
Offline,
Error,
Unknown,
}
impl From<TargetPluginRuntimeState> for PluginOperationalRuntimeState {
fn from(value: TargetPluginRuntimeState) -> Self {
match value {
TargetPluginRuntimeState::Running => Self::Running,
TargetPluginRuntimeState::Offline => Self::Offline,
TargetPluginRuntimeState::Error => Self::Error,
TargetPluginRuntimeState::Unknown => Self::Unknown,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginRevisionContract {
pub version: String,
pub digest_sha256: Option<String>,
pub source: String,
pub installed_at: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub artifact_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginInstallationContract {
pub install_state: PluginInstallState,
pub current_revision: Option<PluginRevisionContract>,
pub previous_revision: Option<PluginRevisionContract>,
#[serde(skip_serializing_if = "Option::is_none")]
pub validation_error: Option<String>,
}
impl From<TargetPluginInstallation> for PluginInstallationContract {
fn from(value: TargetPluginInstallation) -> Self {
Self {
install_state: PluginInstallState::from(value.install_state),
current_revision: value.current_revision.map(|revision| PluginRevisionContract {
version: revision.version,
digest_sha256: revision.digest_sha256,
source: revision.source,
installed_at: revision.installed_at,
artifact_id: revision.artifact_id,
}),
previous_revision: value.previous_revision.map(|revision| PluginRevisionContract {
version: revision.version,
digest_sha256: revision.digest_sha256,
source: revision.source,
installed_at: revision.installed_at,
artifact_id: revision.artifact_id,
}),
validation_error: value.validation_error,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginOperationalStateContract {
pub install_state: PluginInstallState,
pub enable_state: PluginEnableState,
pub runtime_state: PluginOperationalRuntimeState,
}
impl From<TargetPluginOperationalState> for PluginOperationalStateContract {
fn from(value: TargetPluginOperationalState) -> Self {
Self {
install_state: PluginInstallState::from(value.install_state),
enable_state: PluginEnableState::from(value.enable_state),
runtime_state: PluginOperationalRuntimeState::from(value.runtime_state),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginRuntimeContract {
pub protocol_version: String,
pub transport: PluginRuntimeTransport,
}
impl From<TargetPluginExternalRuntimeContract> for PluginRuntimeContract {
fn from(value: TargetPluginExternalRuntimeContract) -> Self {
Self {
protocol_version: value.protocol_version.to_string(),
transport: PluginRuntimeTransport::from(value.transport),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginArtifactContract {
pub artifact_id: String,
pub target_triple: String,
pub download_uri: String,
pub digest_sha256: String,
pub size_bytes: u64,
}
impl From<TargetPluginArtifactManifest> for PluginArtifactContract {
fn from(value: TargetPluginArtifactManifest) -> Self {
Self {
artifact_id: value.artifact_id.to_string(),
target_triple: value.target_triple.to_string(),
download_uri: value.download_uri.to_string(),
digest_sha256: value.digest_sha256.to_string(),
size_bytes: value.size_bytes,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginDistributionContract {
pub artifacts: Vec<PluginArtifactContract>,
}
impl From<TargetPluginDistributionManifest> for PluginDistributionContract {
fn from(value: TargetPluginDistributionManifest) -> Self {
Self {
artifacts: value.artifacts.iter().copied().map(PluginArtifactContract::from).collect(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginInstanceSource {
Config,
Env,
Mixed,
Runtime,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginCatalogDomainEntry {
pub domain: PluginContractDomain,
pub subsystem: String,
pub valid_fields: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginCatalogEntry {
pub plugin_id: String,
pub target_type: String,
pub display_name: String,
pub provider: String,
pub version: String,
pub packaging: PluginContractPackaging,
pub entrypoint_kind: PluginContractEntrypointKind,
pub api_compatibility_version: String,
pub runtime_contract: PluginRuntimeContract,
pub distribution: Option<PluginDistributionContract>,
pub supported_domains: Vec<PluginContractDomain>,
pub secret_fields: Vec<String>,
pub domain_configs: Vec<PluginCatalogDomainEntry>,
#[serde(skip_serializing_if = "Option::is_none")]
pub installation: Option<PluginInstallationContract>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PluginCatalogResponse {
pub plugins: Vec<PluginCatalogEntry>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginInstanceEntry {
pub id: String,
pub plugin_id: String,
pub domain: PluginContractDomain,
pub subsystem: String,
pub account_id: String,
pub service: String,
pub status: String,
pub source: PluginInstanceSource,
pub enabled: bool,
pub config: HashMap<String, String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub operational_state: Option<PluginOperationalStateContract>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub diagnostic_codes: Vec<PluginInstanceDiagnosticCode>,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum PluginInstanceDiagnosticCode {
ModuleDisabled,
InstanceDisabled,
EnvironmentManaged,
MixedSource,
NotLoadedInRuntime,
RuntimeOffline,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginInstanceDiagnostic {
pub code: PluginInstanceDiagnosticCode,
pub message: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) struct PluginInstanceDiagnosticCount {
pub code: PluginInstanceDiagnosticCode,
pub count: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PluginInstanceDetail {
#[serde(flatten)]
pub instance: PluginInstanceEntry,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub diagnostics: Vec<PluginInstanceDiagnostic>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct PluginInstancesResponse {
pub instances: Vec<PluginInstanceEntry>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub diagnostic_counts: Vec<PluginInstanceDiagnosticCount>,
pub truncated: bool,
pub next_marker: Option<String>,
}
#[cfg(test)]
mod tests {
use super::{
PluginArtifactContract, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain,
PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginInstanceDetail,
PluginInstanceDiagnostic, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry,
PluginInstanceSource, PluginInstancesResponse, PluginRuntimeContract, PluginRuntimeTransport,
};
use serde_json::json;
use std::collections::HashMap;
#[test]
fn plugin_catalog_contract_serializes_stable_json_shape() {
let response = PluginCatalogResponse {
plugins: vec![PluginCatalogEntry {
plugin_id: "builtin:webhook".to_string(),
target_type: "webhook".to_string(),
display_name: "Webhook".to_string(),
provider: "rustfs".to_string(),
version: "1.0.0".to_string(),
packaging: PluginContractPackaging::Builtin,
entrypoint_kind: PluginContractEntrypointKind::Builtin,
api_compatibility_version: "rustfs.target-plugin.v1".to_string(),
runtime_contract: PluginRuntimeContract {
protocol_version: "rustfs.target-runtime.v1".to_string(),
transport: PluginRuntimeTransport::InProcess,
},
distribution: None,
supported_domains: vec![PluginContractDomain::Audit, PluginContractDomain::Notify],
secret_fields: vec!["auth_token".to_string()],
domain_configs: vec![PluginCatalogDomainEntry {
domain: PluginContractDomain::Notify,
subsystem: "notify_webhook".to_string(),
valid_fields: vec!["endpoint".to_string(), "auth_token".to_string()],
}],
installation: None,
}],
};
let value = serde_json::to_value(response).expect("catalog response should serialize");
assert_eq!(
value,
json!({
"plugins": [{
"plugin_id": "builtin:webhook",
"target_type": "webhook",
"display_name": "Webhook",
"provider": "rustfs",
"version": "1.0.0",
"packaging": "builtin",
"entrypoint_kind": "builtin",
"api_compatibility_version": "rustfs.target-plugin.v1",
"runtime_contract": {
"protocol_version": "rustfs.target-runtime.v1",
"transport": "in_process"
},
"distribution": null,
"supported_domains": ["audit", "notify"],
"secret_fields": ["auth_token"],
"domain_configs": [{
"domain": "notify",
"subsystem": "notify_webhook",
"valid_fields": ["endpoint", "auth_token"]
}]
}]
})
);
}
#[test]
fn plugin_instance_contract_serializes_stable_json_shape() {
let response = PluginInstancesResponse {
instances: vec![PluginInstanceEntry {
id: "builtin:webhook:notify:primary".to_string(),
plugin_id: "builtin:webhook".to_string(),
domain: PluginContractDomain::Notify,
subsystem: "notify_webhook".to_string(),
account_id: "primary".to_string(),
service: "webhook".to_string(),
status: "offline".to_string(),
source: PluginInstanceSource::Config,
enabled: true,
config: HashMap::from([
("enable".to_string(), "on".to_string()),
("endpoint".to_string(), "https://example.com/hook".to_string()),
]),
operational_state: None,
diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime],
}],
diagnostic_counts: vec![PluginInstanceDiagnosticCount {
code: PluginInstanceDiagnosticCode::NotLoadedInRuntime,
count: 1,
}],
truncated: false,
next_marker: None,
};
let value = serde_json::to_value(response).expect("instance response should serialize");
assert_eq!(
value,
json!({
"instances": [{
"id": "builtin:webhook:notify:primary",
"plugin_id": "builtin:webhook",
"domain": "notify",
"subsystem": "notify_webhook",
"account_id": "primary",
"service": "webhook",
"status": "offline",
"source": "config",
"enabled": true,
"config": {
"enable": "on",
"endpoint": "https://example.com/hook"
},
"diagnostic_codes": ["not_loaded_in_runtime"]
}],
"diagnostic_counts": [{
"code": "not_loaded_in_runtime",
"count": 1
}],
"truncated": false,
"next_marker": null
})
);
}
#[test]
fn plugin_instance_detail_contract_serializes_diagnostics_when_present() {
let detail = PluginInstanceDetail {
instance: PluginInstanceEntry {
id: "builtin:webhook:notify:primary".to_string(),
plugin_id: "builtin:webhook".to_string(),
domain: PluginContractDomain::Notify,
subsystem: "notify_webhook".to_string(),
account_id: "primary".to_string(),
service: "webhook".to_string(),
status: "offline".to_string(),
source: PluginInstanceSource::Config,
enabled: true,
config: HashMap::from([("endpoint".to_string(), "https://example.com/hook".to_string())]),
operational_state: None,
diagnostic_codes: vec![PluginInstanceDiagnosticCode::NotLoadedInRuntime],
},
diagnostics: vec![PluginInstanceDiagnostic {
code: PluginInstanceDiagnosticCode::NotLoadedInRuntime,
message: "plugin instance is enabled in config but not currently loaded in runtime".to_string(),
}],
};
let value = serde_json::to_value(detail).expect("instance detail should serialize");
assert_eq!(
value,
json!({
"id": "builtin:webhook:notify:primary",
"plugin_id": "builtin:webhook",
"domain": "notify",
"subsystem": "notify_webhook",
"account_id": "primary",
"service": "webhook",
"status": "offline",
"source": "config",
"enabled": true,
"config": {
"endpoint": "https://example.com/hook"
},
"diagnostic_codes": ["not_loaded_in_runtime"],
"diagnostics": [{
"code": "not_loaded_in_runtime",
"message": "plugin instance is enabled in config but not currently loaded in runtime"
}]
})
);
}
#[test]
fn plugin_catalog_distribution_contract_serializes_when_present() {
let entry = PluginCatalogEntry {
plugin_id: "external:webhook".to_string(),
target_type: "webhook".to_string(),
display_name: "Webhook+".to_string(),
provider: "example".to_string(),
version: "1.2.3".to_string(),
packaging: PluginContractPackaging::Builtin,
entrypoint_kind: PluginContractEntrypointKind::Sidecar,
api_compatibility_version: "rustfs.target-plugin.v1".to_string(),
runtime_contract: PluginRuntimeContract {
protocol_version: "rustfs.target-runtime.v1".to_string(),
transport: PluginRuntimeTransport::Grpc,
},
distribution: Some(PluginDistributionContract {
artifacts: vec![PluginArtifactContract {
artifact_id: "sidecar-linux-amd64".to_string(),
target_triple: "x86_64-unknown-linux-gnu".to_string(),
download_uri: "https://plugins.example.test/webhook.tar.zst".to_string(),
digest_sha256: "0123456789abcdef".to_string(),
size_bytes: 4096,
}],
}),
supported_domains: vec![PluginContractDomain::Notify],
secret_fields: Vec::new(),
domain_configs: Vec::new(),
installation: None,
};
let value = serde_json::to_value(entry).expect("catalog entry should serialize");
assert_eq!(value["distribution"]["artifacts"][0]["artifact_id"], "sidecar-linux-amd64");
assert_eq!(value["distribution"]["artifacts"][0]["target_triple"], "x86_64-unknown-linux-gnu");
assert_eq!(
value["distribution"]["artifacts"][0]["download_uri"],
"https://plugins.example.test/webhook.tar.zst"
);
assert_eq!(value["distribution"]["artifacts"][0]["digest_sha256"], "0123456789abcdef");
assert_eq!(value["distribution"]["artifacts"][0]["size_bytes"], 4096);
}
}
rustfs/src/admin/route_registration_test.rs
+9 -2
View File
@@ -14,8 +14,8 @@
use crate::admin::{
handlers::{
audit, bucket_meta, heal, health, kms, module_switch, oidc, pools, profile_admin, quota, rebalance, replication,
site_replication, sts, system, tier, user,
audit, bucket_meta, heal, health, kms, module_switch, oidc, plugins_catalog, plugins_instances, pools, profile_admin,
quota, rebalance, replication, site_replication, sts, system, tier, user,
},
router::{AdminOperation, S3Router},
};
@@ -54,6 +54,8 @@ fn register_admin_routes(router: &mut S3Router<AdminOperation>) {
bucket_meta::register_bucket_meta_route(router).expect("register bucket meta route");
audit::register_audit_target_route(router).expect("register audit target route");
module_switch::register_module_switch_route(router).expect("register module switch route");
plugins_catalog::register_plugin_catalog_route(router).expect("register plugin catalog route");
plugins_instances::register_plugin_instance_route(router).expect("register plugin instances route");
replication::register_replication_route(router).expect("register replication route");
site_replication::register_site_replication_route(router).expect("register site replication route");
profile_admin::register_profiling_route(router).expect("register profile route");
@@ -101,6 +103,11 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::GET, &admin_path("/v3/audit/target/list"));
assert_route(&router, Method::GET, &admin_path("/v3/module-switches"));
assert_route(&router, Method::PUT, &admin_path("/v3/module-switches"));
assert_route(&router, Method::GET, &admin_path("/v4/plugins/catalog"));
assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances"));
assert_route(&router, Method::GET, &admin_path("/v4/plugins/instances/example-id"));
assert_route(&router, Method::PUT, &admin_path("/v4/plugins/instances/example-id"));
assert_route(&router, Method::DELETE, &admin_path("/v4/plugins/instances/example-id"));
assert_route(&router, Method::PUT, &admin_path("/v3/audit/target/audit_webhook/test-audit"));
assert_route(&router, Method::DELETE, &admin_path("/v3/audit/target/audit_webhook/test-audit/reset"));
assert_route(&router, Method::GET, &admin_path("/v3/accountinfo"));
rustfs/src/admin/router.rs
+5 -8
View File
@@ -510,7 +510,7 @@ fn build_object_lambda_get_request(req: &S3Request<Body>, bucket: &str, object:
})
.transpose()?;
let version_id = query_value_exact(&filtered_uri, "versionId").filter(|value| !value.is_empty());
let range = parse_optional_header(&req.headers, http::header::RANGE)?
let range = parse_optional_header(&req.headers, header::RANGE)?
.map(|value| Range::parse(&value).map_err(|_| s3_error!(InvalidArgument, "Range header is invalid")))
.transpose()?;
@@ -520,13 +520,10 @@ fn build_object_lambda_get_request(req: &S3Request<Body>, bucket: &str, object:
.part_number(part_number)
.version_id(version_id)
.range(range)
.if_match(parse_optional_etag_condition_header::<IfMatch>(&req.headers, http::header::IF_MATCH)?)
.if_none_match(parse_optional_etag_condition_header::<IfNoneMatch>(
&req.headers,
http::header::IF_NONE_MATCH,
)?)
.if_modified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_MODIFIED_SINCE)?)
.if_unmodified_since(parse_optional_timestamp_header(&req.headers, http::header::IF_UNMODIFIED_SINCE)?);
.if_match(parse_optional_etag_condition_header::<IfMatch>(&req.headers, header::IF_MATCH)?)
.if_none_match(parse_optional_etag_condition_header::<IfNoneMatch>(&req.headers, header::IF_NONE_MATCH)?)
.if_modified_since(parse_optional_timestamp_header(&req.headers, header::IF_MODIFIED_SINCE)?)
.if_unmodified_since(parse_optional_timestamp_header(&req.headers, header::IF_UNMODIFIED_SINCE)?);
builder = builder.sse_customer_algorithm(parse_optional_header(
&req.headers,
rustfs/src/server/audit.rs
+14 -42
View File
@@ -72,13 +72,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
// 1. Get the global configuration loaded by ecstore
let server_config = match server_config_from_context() {
Some(config) => {
info!(
target: "rustfs::main::start_audit_system",
"Global server configuration loads successfully: {:?}", config
);
config
}
Some(config) => config,
None => {
warn!(
target: "rustfs::main::start_audit_system",
@@ -92,9 +86,8 @@ pub async fn start_audit_system() -> AuditResult<()> {
target: "rustfs::main::start_audit_system",
"The global server configuration is loaded"
);
// 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
let has_targets = has_any_audit_targets(&server_config);
if !has_targets {
// 2. Check if the audit subsystem exists in the configuration, and skip initialization if it doesn't
if !has_any_audit_targets(&server_config) {
info!(
target: "rustfs::main::start_audit_system",
"Audit subsystem targets are not configured, and audit system initialization is skipped."
@@ -107,35 +100,16 @@ pub async fn start_audit_system() -> AuditResult<()> {
"Audit subsystem configuration detected and started initializing the audit system."
);
if let Some(system) = audit_system() {
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
// Match notify behavior: prefer reloading the existing singleton
// instead of constructing a second lifecycle path on re-enable.
match system.reload_config(server_config).await {
Ok(()) => {
info!(
target: "rustfs::main::start_audit_system",
"Audit system reloaded successfully with time: {}.",
jiff::Zoned::now()
);
Ok(())
}
Err(e) => {
warn!(
target: "rustfs::main::start_audit_system",
"Audit system reload failed: {:?}",
e
);
Err(e)
}
}
}
AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await {
let system = audit_system().unwrap_or_else(init_audit_system);
match system.get_state().await {
AuditSystemState::Running | AuditSystemState::Paused | AuditSystemState::Starting => {
// Match notify behavior: prefer reloading the existing singleton
// instead of constructing a second lifecycle path on re-enable.
match system.reload_config(server_config).await {
Ok(()) => {
info!(
target: "rustfs::main::start_audit_system",
"Audit system started successfully with time: {}.",
"Audit system reloaded successfully with time: {}.",
jiff::Zoned::now()
);
Ok(())
@@ -143,16 +117,14 @@ pub async fn start_audit_system() -> AuditResult<()> {
Err(e) => {
warn!(
target: "rustfs::main::start_audit_system",
"Audit system startup failed: {:?}",
"Audit system reload failed: {:?}",
e
);
Err(e)
}
},
}
}
} else {
let system = init_audit_system();
match system.start(server_config).await {
AuditSystemState::Stopped | AuditSystemState::Stopping => match system.start(server_config).await {
Ok(()) => {
info!(
target: "rustfs::main::start_audit_system",
@@ -169,7 +141,7 @@ pub async fn start_audit_system() -> AuditResult<()> {
);
Err(e)
}
}
},
}
}
rustfs/src/server/event.rs
+85 -25
View File
@@ -17,6 +17,7 @@ use crate::app::context::resolve_server_config;
use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
use rustfs_notify::EventArgs as NotifyEventArgs;
use rustfs_s3_common::EventName;
use std::net::SocketAddr;
use std::sync::atomic::{AtomicBool, Ordering};
use tokio::spawn;
use tracing::{error, info, instrument, warn};
@@ -39,13 +40,7 @@ pub fn is_notify_module_enabled() -> bool {
fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default();
let (host, port) = match args.host.rsplit_once(':') {
Some((host, port)) => match port.parse::<u16>() {
Ok(port) => (host.to_string(), port),
Err(_) => (args.host, 0),
},
None => (args.host, 0),
};
let (host, port) = parse_host_and_port(args.host);
let req_params = args.req_params.into_iter().collect();
let resp_elements = args.resp_elements.into_iter().collect();
@@ -62,6 +57,24 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> NotifyEventArgs {
}
}
fn parse_host_and_port(host: String) -> (String, u16) {
if let Ok(addr) = host.parse::<SocketAddr>() {
return (addr.ip().to_string(), addr.port());
}
if host.chars().filter(|&c| c == ':').count() != 1 {
return (host, 0);
}
match host.split_once(':') {
Some((base, port)) if !base.is_empty() => match port.parse::<u16>() {
Ok(port) => (base.to_string(), port),
Err(_) => (host, 0),
},
_ => (host, 0),
}
}
fn install_ecstore_event_dispatch_hook() {
let installed = register_event_dispatch_hook(|args| {
let notify_args = convert_ecstore_event_args(args);
@@ -75,6 +88,23 @@ fn install_ecstore_event_dispatch_hook() {
}
}
fn ensure_live_events_initialized() -> bool {
if rustfs_notify::notification_system().is_some() {
return true;
}
match rustfs_notify::initialize_live_events() {
Ok(()) => {
install_ecstore_event_dispatch_hook();
true
}
Err(e) => {
error!("Failed to initialize live event stream support: {}", e);
false
}
}
}
/// Shuts down the event notifier system gracefully
pub async fn shutdown_event_notifier() {
info!("Shutting down event notifier system...");
@@ -110,17 +140,11 @@ pub async fn init_event_notifier() {
"Notify module is disabled, initializing live event stream support only. Set {}=true to enable notification targets.",
rustfs_config::ENV_NOTIFY_ENABLE
);
if rustfs_notify::notification_system().is_none() {
match rustfs_notify::initialize_live_events() {
Ok(()) => {
install_ecstore_event_dispatch_hook();
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
}
Err(e) => error!("Failed to initialize live event stream support: {}", e),
}
if ensure_live_events_initialized() {
info!(
target: "rustfs::main::init_event_notifier",
"Live event stream support initialized successfully."
);
}
return;
}
@@ -155,13 +179,49 @@ pub async fn init_event_notifier() {
"Event notifier system reloaded successfully."
);
}
} else if let Err(e) = rustfs_notify::initialize(server_config).await {
error!("Failed to initialize event notifier system: {}", e);
} else {
install_ecstore_event_dispatch_hook();
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier system initialized successfully."
);
match rustfs_notify::initialize(server_config).await {
Ok(()) => {
install_ecstore_event_dispatch_hook();
info!(
target: "rustfs::main::init_event_notifier",
"Event notifier system initialized successfully."
);
}
Err(e) => error!("Failed to initialize event notifier system: {}", e),
}
}
}
#[cfg(test)]
mod tests {
use super::parse_host_and_port;
#[test]
fn parse_host_and_port_with_ipv4_and_port() {
let (host, port) = parse_host_and_port("127.0.0.1:9000".to_string());
assert_eq!(host, "127.0.0.1");
assert_eq!(port, 9000);
}
#[test]
fn parse_host_and_port_with_bracketed_ipv6_and_port() {
let (host, port) = parse_host_and_port("[::1]:9000".to_string());
assert_eq!(host, "::1");
assert_eq!(port, 9000);
}
#[test]
fn parse_host_and_port_with_ipv6_without_port() {
let (host, port) = parse_host_and_port("::1".to_string());
assert_eq!(host, "::1");
assert_eq!(port, 0);
}
#[test]
fn parse_host_and_port_with_hostname_and_port() {
let (host, port) = parse_host_and_port("localhost:9001".to_string());
assert_eq!(host, "localhost");
assert_eq!(port, 9001);
}
}