gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 16:28:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve code for notify

Browse Source
This commit is contained in:
houseme
2025-06-23 03:34:05 +08:00
parent c7af6587f5
commit 928453db62
30 changed files with 521 additions and 989 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+2 -1
View File
@@ -8390,11 +8390,11 @@ dependencies = [
"chrono",
"const-str",
"ecstore",
"form_urlencoded",
"once_cell",
"quick-xml",
"reqwest",
"rumqttc",
"rustfs-config",
"rustfs-utils",
"serde",
"serde_json",
@@ -8524,6 +8524,7 @@ dependencies = [
"sha2 0.10.9",
"siphasher 1.0.1",
"snap",
"sysinfo",
"tempfile",
"tokio",
"tracing",
Cargo.toml
+1
View File
@@ -94,6 +94,7 @@ dirs = "6.0.0"
dotenvy = "0.15.7"
flatbuffers = "25.2.10"
flexi_logger = { version = "0.30.2", features = ["trc", "dont_minimize_extra_stacks"] }
form_urlencoded = "1.2.1"
futures = "0.3.31"
futures-core = "0.3.31"
futures-util = "0.3.31"
crates/config/Cargo.toml
-1
View File
@@ -18,6 +18,5 @@ workspace = true
[features]
default = []
constants = ["dep:const-str"]
notify = []
observability = []
crates/config/src/lib.rs
-3
View File
@@ -3,8 +3,5 @@ pub mod constants;
#[cfg(feature = "constants")]
pub use constants::app::*;
#[cfg(feature = "notify")]
pub mod notify;
#[cfg(feature = "observability")]
pub mod observability;
crates/config/src/notify/config.rs
-53
View File
@@ -1,53 +0,0 @@
use crate::notify::mqtt::MQTTArgs;
use crate::notify::webhook::WebhookArgs;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// Config - notification target configuration structure, holds
/// information about various notification targets.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NotifyConfig {
pub mqtt: HashMap<String, MQTTArgs>,
pub webhook: HashMap<String, WebhookArgs>,
}
impl NotifyConfig {
/// Create a new configuration with default values.
pub fn new() -> Self {
let mut config = NotifyConfig {
webhook: HashMap::new(),
mqtt: HashMap::new(),
};
// Insert default target for each backend
config.webhook.insert("1".to_string(), WebhookArgs::new());
config.mqtt.insert("1".to_string(), MQTTArgs::new());
config
}
}
impl Default for NotifyConfig {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use crate::notify::config::NotifyConfig;
#[test]
fn test_notify_config_new() {
let config = NotifyConfig::new();
assert_eq!(config.webhook.len(), 1);
assert_eq!(config.mqtt.len(), 1);
assert!(config.webhook.contains_key("1"));
assert!(config.mqtt.contains_key("1"));
}
#[test]
fn test_notify_config_default() {
let config = NotifyConfig::default();
assert_eq!(config.webhook.len(), 1);
assert_eq!(config.mqtt.len(), 1);
}
}
crates/config/src/notify/help.rs
-26
View File
@@ -1,26 +0,0 @@
/// Help text for Webhook configuration.
pub const HELP_WEBHOOK: &str = r#"
Webhook configuration:
- enable: Enable or disable the webhook target (true/false)
- endpoint: Webhook server endpoint (e.g., http://localhost:8080/rustfs/events)
- auth_token: Opaque string or JWT authorization token (optional)
- queue_dir: Absolute path for persistent event queue (optional)
- queue_limit: Maximum number of events to queue (optional, default: 0)
- client_cert: Path to client certificate file (optional)
- client_key: Path to client private key file (optional)
"#;
/// Help text for MQTT configuration.
pub const HELP_MQTT: &str = r#"
MQTT configuration:
- enable: Enable or disable the MQTT target (true/false)
- broker: MQTT broker address (e.g., tcp://localhost:1883)
- topic: MQTT topic (e.g., rustfs/events)
- qos: Quality of Service level (0, 1, or 2)
- username: Username for MQTT authentication (optional)
- password: Password for MQTT authentication (optional)
- reconnect_interval: Reconnect interval in milliseconds (optional)
- keep_alive_interval: Keep alive interval in milliseconds (optional)
- queue_dir: Absolute path for persistent event queue (optional)
- queue_limit: Maximum number of events to queue (optional, default: 0)
"#;
crates/config/src/notify/legacy.rs
-268
View File
@@ -1,268 +0,0 @@
use crate::notify::mqtt::MQTTArgs;
use crate::notify::webhook::WebhookArgs;
use std::collections::HashMap;
/// Convert legacy webhook configuration to the new WebhookArgs struct.
pub fn convert_webhook_config(config: &HashMap<String, String>) -> Result<WebhookArgs, String> {
let mut args = WebhookArgs::new();
args.enable = config.get("enable").map_or(false, |v| v == "true");
args.endpoint = config.get("endpoint").unwrap_or(&"".to_string()).clone();
args.auth_token = config.get("auth_token").unwrap_or(&"".to_string()).clone();
args.queue_dir = config.get("queue_dir").unwrap_or(&"".to_string()).clone();
args.queue_limit = config.get("queue_limit").map_or(0, |v| v.parse().unwrap_or(0));
args.client_cert = config.get("client_cert").unwrap_or(&"".to_string()).clone();
args.client_key = config.get("client_key").unwrap_or(&"".to_string()).clone();
Ok(args)
}
/// Convert legacy MQTT configuration to the new MQTTArgs struct.
pub fn convert_mqtt_config(config: &HashMap<String, String>) -> Result<MQTTArgs, String> {
let mut args = MQTTArgs::new();
args.enable = config.get("enable").map_or(false, |v| v == "true");
args.broker = config.get("broker").unwrap_or(&"".to_string()).clone();
args.topic = config.get("topic").unwrap_or(&"".to_string()).clone();
args.qos = config.get("qos").map_or(0, |v| v.parse().unwrap_or(0));
args.username = config.get("username").unwrap_or(&"".to_string()).clone();
args.password = config.get("password").unwrap_or(&"".to_string()).clone();
args.reconnect_interval = config.get("reconnect_interval").map_or(0, |v| v.parse().unwrap_or(0));
args.keep_alive_interval = config.get("keep_alive_interval").map_or(0, |v| v.parse().unwrap_or(0));
args.queue_dir = config.get("queue_dir").unwrap_or(&"".to_string()).clone();
args.queue_limit = config.get("queue_limit").map_or(0, |v| v.parse().unwrap_or(0));
Ok(args)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_convert_webhook_config_invalid() {
let mut old_config = HashMap::new();
old_config.insert("max_retries".to_string(), "invalid".to_string());
let result = convert_webhook_config(&old_config);
assert!(result.is_err());
}
#[test]
fn test_convert_mqtt_config_invalid() {
let mut old_config = HashMap::new();
old_config.insert("port".to_string(), "invalid".to_string());
let result = convert_mqtt_config(&old_config);
assert!(result.is_err());
}
#[test]
fn test_convert_empty_config() {
let empty_config = HashMap::new();
let webhook_result = convert_webhook_config(&empty_config);
assert!(webhook_result.is_ok());
let mqtt_result = convert_mqtt_config(&empty_config);
assert!(mqtt_result.is_ok());
}
#[test]
fn test_convert_config_with_extra_fields() {
let mut extra_config = HashMap::new();
extra_config.insert("endpoint".to_string(), "http://example.com".to_string());
extra_config.insert("extra_field".to_string(), "extra_value".to_string());
let webhook_result = convert_webhook_config(&extra_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut extra_mqtt_config = HashMap::new();
extra_mqtt_config.insert("broker".to_string(), "mqtt.example.com".to_string());
extra_mqtt_config.insert("extra_field".to_string(), "extra_value".to_string());
let mqtt_result = convert_mqtt_config(&extra_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
#[test]
fn test_convert_config_with_empty_values() {
let mut empty_values_config = HashMap::new();
empty_values_config.insert("endpoint".to_string(), "".to_string());
let webhook_result = convert_webhook_config(&empty_values_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "");
let mut empty_mqtt_config = HashMap::new();
empty_mqtt_config.insert("broker".to_string(), "".to_string());
let mqtt_result = convert_mqtt_config(&empty_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "");
}
#[test]
fn test_convert_config_with_whitespace_values() {
let mut whitespace_config = HashMap::new();
whitespace_config.insert("endpoint".to_string(), " http://example.com ".to_string());
let webhook_result = convert_webhook_config(&whitespace_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, " http://example.com ");
let mut whitespace_mqtt_config = HashMap::new();
whitespace_mqtt_config.insert("broker".to_string(), " mqtt.example.com ".to_string());
let mqtt_result = convert_mqtt_config(&whitespace_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, " mqtt.example.com ");
}
#[test]
fn test_convert_config_with_special_characters() {
let mut special_chars_config = HashMap::new();
special_chars_config.insert("endpoint".to_string(), "http://example.com/path?param=value&other=123".to_string());
let webhook_result = convert_webhook_config(&special_chars_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com/path?param=value&other=123");
let mut special_chars_mqtt_config = HashMap::new();
special_chars_mqtt_config.insert("broker".to_string(), "mqtt.example.com:1883".to_string());
let mqtt_result = convert_mqtt_config(&special_chars_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com:1883");
}
#[test]
fn test_convert_config_with_boolean_values() {
let mut boolean_config = HashMap::new();
boolean_config.insert("enable".to_string(), "true".to_string());
let webhook_result = convert_webhook_config(&boolean_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, ""); // default value
let mut boolean_mqtt_config = HashMap::new();
boolean_mqtt_config.insert("enable".to_string(), "false".to_string());
let mqtt_result = convert_mqtt_config(&boolean_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "localhost"); // default value
}
#[test]
fn test_convert_config_with_null_values() {
let mut null_config = HashMap::new();
null_config.insert("endpoint".to_string(), "null".to_string());
let webhook_result = convert_webhook_config(&null_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "null");
let mut null_mqtt_config = HashMap::new();
null_mqtt_config.insert("broker".to_string(), "null".to_string());
let mqtt_result = convert_mqtt_config(&null_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "null");
}
#[test]
fn test_convert_config_with_duplicate_keys() {
let mut duplicate_config = HashMap::new();
duplicate_config.insert("endpoint".to_string(), "http://example.org".to_string());
let webhook_result = convert_webhook_config(&duplicate_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.org"); // last value wins
let mut duplicate_mqtt_config = HashMap::new();
duplicate_mqtt_config.insert("broker".to_string(), "mqtt.example.org".to_string());
let mqtt_result = convert_mqtt_config(&duplicate_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.org"); // last value wins
}
#[test]
fn test_convert_config_with_case_insensitive_keys() {
let mut case_insensitive_config = HashMap::new();
case_insensitive_config.insert("ENDPOINT".to_string(), "http://example.com".to_string());
let webhook_result = convert_webhook_config(&case_insensitive_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut case_insensitive_mqtt_config = HashMap::new();
case_insensitive_mqtt_config.insert("BROKER".to_string(), "mqtt.example.com".to_string());
let mqtt_result = convert_mqtt_config(&case_insensitive_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
#[test]
fn test_convert_config_with_mixed_case_keys() {
let mut mixed_case_config = HashMap::new();
mixed_case_config.insert("EndPoint".to_string(), "http://example.com".to_string());
let webhook_result = convert_webhook_config(&mixed_case_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut mixed_case_mqtt_config = HashMap::new();
mixed_case_mqtt_config.insert("BroKer".to_string(), "mqtt.example.com".to_string());
let mqtt_result = convert_mqtt_config(&mixed_case_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
#[test]
fn test_convert_config_with_snake_case_keys() {
let mut snake_case_config = HashMap::new();
snake_case_config.insert("end_point".to_string(), "http://example.com".to_string());
let webhook_result = convert_webhook_config(&snake_case_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut snake_case_mqtt_config = HashMap::new();
snake_case_mqtt_config.insert("bro_ker".to_string(), "mqtt.example.com".to_string());
let mqtt_result = convert_mqtt_config(&snake_case_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
#[test]
fn test_convert_config_with_kebab_case_keys() {
let mut kebab_case_config = HashMap::new();
kebab_case_config.insert("end-point".to_string(), "http://example.com".to_string());
let webhook_result = convert_webhook_config(&kebab_case_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut kebab_case_mqtt_config = HashMap::new();
kebab_case_mqtt_config.insert("bro-ker".to_string(), "mqtt.example.com".to_string());
let mqtt_result = convert_mqtt_config(&kebab_case_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
#[test]
fn test_convert_config_with_camel_case_keys() {
let mut camel_case_config = HashMap::new();
camel_case_config.insert("endPoint".to_string(), "http://example.com".to_string());
let webhook_result = convert_webhook_config(&camel_case_config);
assert!(webhook_result.is_ok());
let args = webhook_result.unwrap();
assert_eq!(args.endpoint, "http://example.com");
let mut camel_case_mqtt_config = HashMap::new();
camel_case_mqtt_config.insert("broKer".to_string(), "mqtt.example.com".to_string());
let mqtt_result = convert_mqtt_config(&camel_case_mqtt_config);
assert!(mqtt_result.is_ok());
let args = mqtt_result.unwrap();
assert_eq!(args.broker, "mqtt.example.com");
}
}
crates/config/src/notify/mod.rs
-5
View File
@@ -1,5 +0,0 @@
pub mod config;
pub mod help;
pub mod legacy;
pub mod mqtt;
pub mod webhook;
crates/config/src/notify/mqtt.rs
-114
View File
@@ -1,114 +0,0 @@
use serde::{Deserialize, Serialize};
/// MQTTArgs - MQTT target arguments.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MQTTArgs {
pub enable: bool,
pub broker: String,
pub topic: String,
pub qos: u8,
pub username: String,
pub password: String,
pub reconnect_interval: u64,
pub keep_alive_interval: u64,
#[serde(skip)]
pub root_cas: Option<()>, // Placeholder for *x509.CertPool
pub queue_dir: String,
pub queue_limit: u64,
}
impl MQTTArgs {
/// Create a new configuration with default values.
pub fn new() -> Self {
Self {
enable: false,
broker: "".to_string(),
topic: "".to_string(),
qos: 0,
username: "".to_string(),
password: "".to_string(),
reconnect_interval: 0,
keep_alive_interval: 0,
root_cas: None,
queue_dir: "".to_string(),
queue_limit: 0,
}
}
/// Validate MQTTArgs fields
pub fn validate(&self) -> Result<(), String> {
if !self.enable {
return Ok(());
}
if self.broker.trim().is_empty() {
return Err("MQTT broker cannot be empty".to_string());
}
if self.topic.trim().is_empty() {
return Err("MQTT topic cannot be empty".to_string());
}
if self.queue_dir != "" && !self.queue_dir.starts_with('/') {
return Err("queueDir path should be absolute".to_string());
}
if self.qos == 0 && self.queue_dir != "" {
return Err("qos should be set to 1 or 2 if queueDir is set".to_string());
}
Ok(())
}
}
impl Default for MQTTArgs {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_mqtt_args_new() {
let args = MQTTArgs::new();
assert_eq!(args.broker, "");
assert_eq!(args.topic, "");
assert_eq!(args.qos, 0);
assert_eq!(args.username, "");
assert_eq!(args.password, "");
assert_eq!(args.reconnect_interval, 0);
assert_eq!(args.keep_alive_interval, 0);
assert!(args.root_cas.is_none());
assert_eq!(args.queue_dir, "");
assert_eq!(args.queue_limit, 0);
assert!(!args.enable);
}
#[test]
fn test_mqtt_args_validate() {
let mut args = MQTTArgs::new();
assert!(args.validate().is_ok());
args.broker = "".to_string();
assert!(args.validate().is_err());
args.broker = "localhost".to_string();
args.topic = "".to_string();
assert!(args.validate().is_err());
args.topic = "mqtt_topic".to_string();
args.reconnect_interval = 10001;
assert!(args.validate().is_err());
args.reconnect_interval = 1000;
args.keep_alive_interval = 10001;
assert!(args.validate().is_err());
args.keep_alive_interval = 1000;
args.queue_limit = 10001;
assert!(args.validate().is_err());
args.queue_dir = "invalid_path".to_string();
assert!(args.validate().is_err());
args.queue_dir = "/valid_path".to_string();
assert!(args.validate().is_ok());
args.qos = 0;
assert!(args.validate().is_err());
args.qos = 1;
assert!(args.validate().is_ok());
args.qos = 2;
assert!(args.validate().is_ok());
}
}
crates/config/src/notify/webhook.rs
-81
View File
@@ -1,81 +0,0 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
/// WebhookArgs - Webhook target arguments.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebhookArgs {
pub enable: bool,
pub endpoint: String,
pub auth_token: String,
#[serde(skip)]
pub custom_headers: Option<HashMap<String, String>>,
pub queue_dir: String,
pub queue_limit: u64,
pub client_cert: String,
pub client_key: String,
}
impl WebhookArgs {
/// Create a new configuration with default values.
pub fn new() -> Self {
Self {
enable: false,
endpoint: "".to_string(),
auth_token: "".to_string(),
custom_headers: None,
queue_dir: "".to_string(),
queue_limit: 0,
client_cert: "".to_string(),
client_key: "".to_string(),
}
}
/// Validate WebhookArgs fields
pub fn validate(&self) -> Result<(), String> {
if !self.enable {
return Ok(());
}
if self.endpoint.trim().is_empty() {
return Err("endpoint empty".to_string());
}
if self.queue_dir != "" && !self.queue_dir.starts_with('/') {
return Err("queueDir path should be absolute".to_string());
}
if (self.client_cert != "" && self.client_key == "") || (self.client_cert == "" && self.client_key != "") {
return Err("cert and key must be specified as a pair".to_string());
}
Ok(())
}
}
impl Default for WebhookArgs {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use crate::notify::webhook::WebhookArgs;
#[test]
fn test_webhook_args_new() {
let args = WebhookArgs::new();
assert_eq!(args.endpoint, "");
assert_eq!(args.auth_token, "");
assert!(args.custom_headers.is_none());
assert_eq!(args.queue_dir, "");
assert_eq!(args.queue_limit, 0);
assert_eq!(args.client_cert, "");
assert_eq!(args.client_key, "");
assert!(!args.enable);
}
#[test]
fn test_webhook_args_validate() {
let mut args = WebhookArgs::new();
assert!(args.validate().is_err());
args.endpoint = "http://example.com".to_string();
assert!(args.validate().is_ok());
}
}
crates/notify/Cargo.toml
+4 -2
View File
@@ -7,11 +7,12 @@ rust-version.workspace = true
version.workspace = true
[dependencies]
rustfs-config = { workspace = true, features = ["constants", "notify"] }
rustfs-utils = { workspace = true, features = ["path", "sys"] }
async-trait = { workspace = true }
chrono = { workspace = true, features = ["serde"] }
const-str = { workspace = true }
ecstore = { workspace = true }
form_urlencoded = { workspace = true }
once_cell = { workspace = true }
quick-xml = { workspace = true, features = ["serialize", "async-tokio"] }
reqwest = { workspace = true }
@@ -28,11 +29,12 @@ url = { workspace = true }
urlencoding = { workspace = true }
wildmatch = { workspace = true, features = ["serde"] }
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
axum = { workspace = true }
rustfs-utils = { workspace = true, features = ["path"] }
[lints]
workspace = true
crates/notify/examples/full_demo.rs
+15 -7
View File
@@ -1,7 +1,7 @@
use ecstore::config::{Config, KV, KVS};
use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS};
use rustfs_notify::arn::TargetID;
use rustfs_notify::factory::{
DEFAULT_TARGET, ENABLE, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
};
use rustfs_notify::global::notification_system;
@@ -9,12 +9,20 @@ use rustfs_notify::store::DEFAULT_LIMIT;
use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError};
use std::time::Duration;
use tracing::info;
use tracing_subscriber::util::SubscriberInitExt;
#[tokio::main]
async fn main() -> Result<(), NotificationError> {
init_logger(LogLevel::Debug);
let system = notification_system();
let system = match notification_system() {
Some(sys) => sys,
None => {
let config = Config::new();
notification_system::initialize(config).await?;
notification_system().expect("Failed to initialize notification system")
}
};
// --- Initial configuration (Webhook and MQTT) ---
let mut config = Config::new();
@@ -23,8 +31,8 @@ async fn main() -> Result<(), NotificationError> {
let webhook_kvs_vec = vec![
KV {
key: ENABLE.to_string(),
value: "on".to_string(),
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
@@ -62,8 +70,8 @@ async fn main() -> Result<(), NotificationError> {
// MQTT target configuration
let mqtt_kvs_vec = vec![
KV {
key: ENABLE.to_string(),
value: "on".to_string(),
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
crates/notify/examples/full_demo_one.rs
+15 -7
View File
@@ -1,8 +1,8 @@
use ecstore::config::{Config, KV, KVS};
use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS};
// Using Global Accessories
use rustfs_notify::arn::TargetID;
use rustfs_notify::factory::{
DEFAULT_TARGET, ENABLE, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME,
NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
};
use rustfs_notify::global::notification_system;
@@ -10,13 +10,21 @@ use rustfs_notify::store::DEFAULT_LIMIT;
use rustfs_notify::{init_logger, BucketNotificationConfig, Event, EventName, LogLevel, NotificationError};
use std::time::Duration;
use tracing::info;
use tracing_subscriber::util::SubscriberInitExt;
#[tokio::main]
async fn main() -> Result<(), NotificationError> {
init_logger(LogLevel::Debug);
// Get global NotificationSystem instance
let system = notification_system();
let system = match notification_system() {
Some(sys) => sys,
None => {
let config = Config::new();
notification_system::initialize(config).await?;
notification_system().expect("Failed to initialize notification system")
}
};
// --- Initial configuration ---
let mut config = Config::new();
@@ -24,8 +32,8 @@ async fn main() -> Result<(), NotificationError> {
// Webhook target
let webhook_kvs_vec = vec![
KV {
key: ENABLE.to_string(),
value: "on".to_string(),
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
@@ -72,8 +80,8 @@ async fn main() -> Result<(), NotificationError> {
let mqtt_kvs_vec = vec![
KV {
key: ENABLE.to_string(),
value: "on".to_string(),
key: ENABLE_KEY.to_string(),
value: ENABLE_ON.to_string(),
hidden_if_empty: false,
},
KV {
crates/notify/src/args.rs
-110
View File
@@ -1,110 +0,0 @@
use crate::{Event, EventName};
use std::collections::HashMap;
/// 事件参数
#[derive(Debug, Clone)]
pub struct EventArgs {
pub event_name: EventName,
pub bucket_name: String,
pub object_name: String,
pub object_size: Option<i64>,
pub object_etag: Option<String>,
pub object_version_id: Option<String>,
pub object_content_type: Option<String>,
pub object_user_metadata: Option<HashMap<String, String>>,
pub req_params: HashMap<String, String>,
pub resp_elements: HashMap<String, String>,
pub host: String,
pub user_agent: String,
}
impl EventArgs {
/// 转换为通知事件
pub fn to_event(&self) -> Event {
let event_time = chrono::Utc::now();
let unique_id = format!("{:X}", event_time.timestamp_nanos_opt().unwrap_or(0));
let mut resp_elements = HashMap::new();
if let Some(request_id) = self.resp_elements.get("requestId") {
resp_elements.insert("x-amz-request-id".to_string(), request_id.clone());
}
if let Some(node_id) = self.resp_elements.get("nodeId") {
resp_elements.insert("x-amz-id-2".to_string(), node_id.clone());
}
// RustFS 特定的自定义元素
// 注意：这里需要获取 endpoint 的逻辑在 Rust 中可能需要单独实现
resp_elements.insert("x-rustfs-origin-endpoint".to_string(), "".to_string());
// 添加 deployment ID
resp_elements.insert("x-rustfs-deployment-id".to_string(), "".to_string());
if let Some(content_length) = self.resp_elements.get("content-length") {
resp_elements.insert("content-length".to_string(), content_length.clone());
}
let key_name = &self.object_name;
// 注意：这里可能需要根据 escape 参数进行 URL 编码
let mut event = Event {
event_version: "2.0".to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: self.req_params.get("region").cloned().unwrap_or_default(),
event_time,
event_name: self.event_name,
user_identity: crate::event::Identity {
principal_id: self
.req_params
.get("principalId")
.cloned()
.unwrap_or_default(),
},
request_parameters: self.req_params.clone(),
response_elements: resp_elements,
s3: crate::event::Metadata {
schema_version: "1.0".to_string(),
configuration_id: "Config".to_string(),
bucket: crate::event::Bucket {
name: self.bucket_name.clone(),
owner_identity: crate::event::Identity {
principal_id: self
.req_params
.get("principalId")
.cloned()
.unwrap_or_default(),
},
arn: format!("arn:aws:s3:::{}", self.bucket_name),
},
object: crate::event::Object {
key: key_name.clone(),
version_id: self.object_version_id.clone(),
sequencer: unique_id,
size: self.object_size,
etag: self.object_etag.clone(),
content_type: self.object_content_type.clone(),
user_metadata: Some(self.object_user_metadata.clone().unwrap_or_default()),
},
},
source: crate::event::Source {
host: self.host.clone(),
port: "".to_string(),
user_agent: self.user_agent.clone(),
},
};
// 检查是否为删除事件，如果是删除事件，某些字段应当为空
let is_removed_event = matches!(
self.event_name,
EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated
);
if is_removed_event {
event.s3.object.etag = None;
event.s3.object.size = None;
event.s3.object.content_type = None;
event.s3.object.user_metadata = None;
}
event
}
}
crates/notify/src/error.rs
+6
View File
@@ -92,6 +92,12 @@ pub enum NotificationError {
#[error("System initialization error: {0}")]
Initialization(String),
#[error("Notification system has already been initialized")]
AlreadyInitialized,
#[error("Io error: {0}")]
Io(std::io::Error),
}
impl From<url::ParseError> for TargetError {
crates/notify/src/event.rs
+102 -1
View File
@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use url::form_urlencoded;
/// Error returned when parsing event name string fails。
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -296,7 +297,7 @@ pub struct Bucket {
}
/// Represents the object that the event occurred on
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct Object {
/// The key (name) of the object
pub key: String,
@@ -323,6 +324,7 @@ pub struct Object {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Metadata {
/// The schema version of the event
#[serde(rename = "s3SchemaVersion")]
pub schema_version: String,
/// The ID of the configuration that triggered the event
pub configuration_id: String,
@@ -340,11 +342,13 @@ pub struct Source {
/// The port on the host
pub port: String,
/// The user agent that caused the event
#[serde(rename = "userAgent")]
pub user_agent: String,
}
/// Represents a storage event
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Event {
/// The version of the event
pub event_version: String,
@@ -432,6 +436,85 @@ impl Event {
pub fn mask(&self) -> u64 {
self.event_name.mask()
}
pub fn new(args: EventArgs) -> Self {
let event_time = Utc::now().naive_local();
let unique_id = match args.object.mod_time {
Some(t) => format!("{:X}", t.unix_timestamp_nanos()),
None => format!("{:X}", event_time.and_utc().timestamp_nanos_opt().unwrap_or(0)),
};
let mut resp_elements = args.resp_elements.clone();
resp_elements
.entry("x-amz-request-id".to_string())
.or_insert_with(|| "".to_string());
resp_elements
.entry("x-amz-id-2".to_string())
.or_insert_with(|| "".to_string());
// ... Filling of other response elements
// URL encoding of object keys
let key_name = form_urlencoded::byte_serialize(args.object.name.as_bytes()).collect::<String>();
let principal_id = args.req_params.get("principalId").cloned().unwrap_or_default();
let owner_identity = Identity {
principal_id: principal_id.clone(),
};
let user_identity = Identity { principal_id };
let mut s3_metadata = Metadata {
schema_version: "1.0".to_string(),
configuration_id: "Config".to_string(), // or from args
bucket: Bucket {
name: args.bucket_name.clone(),
owner_identity,
arn: format!("arn:aws:s3:::{}", args.bucket_name),
},
object: Object {
key: key_name,
version_id: Some(args.object.version_id.unwrap().to_string()),
sequencer: unique_id,
..Default::default()
},
};
let is_removed_event = matches!(
args.event_name,
EventName::ObjectRemovedDelete | EventName::ObjectRemovedDeleteMarkerCreated
);
if !is_removed_event {
s3_metadata.object.size = Some(args.object.size);
s3_metadata.object.etag = args.object.etag.clone();
s3_metadata.object.content_type = args.object.content_type.clone();
// Filter out internal reserved metadata
let user_metadata = args
.object
.user_defined
.iter()
.filter(|&(k, v)| !k.to_lowercase().starts_with("x-amz-meta-internal-"))
.map(|(k, v)| (k.clone(), v.clone()))
.collect::<HashMap<String, String>>();
s3_metadata.object.user_metadata = Some(user_metadata);
}
Self {
event_version: "2.1".to_string(),
event_source: "rustfs:s3".to_string(),
aws_region: args.req_params.get("region").cloned().unwrap_or_default(),
event_time: event_time.and_utc(),
event_name: args.event_name,
user_identity,
request_parameters: args.req_params,
response_elements: resp_elements,
s3: s3_metadata,
source: Source {
host: args.host,
port: "".to_string(),
user_agent: args.user_agent,
},
}
}
}
/// Represents a log of events for sending to targets
@@ -444,3 +527,21 @@ pub struct EventLog {
/// The list of events
pub records: Vec<Event>,
}
#[derive(Debug, Clone)]
pub struct EventArgs {
pub event_name: EventName,
pub bucket_name: String,
pub object: ecstore::store_api::ObjectInfo,
pub req_params: HashMap<String, String>,
pub resp_elements: HashMap<String, String>,
pub host: String,
pub user_agent: String,
}
impl EventArgs {
// Helper function to check if it is a copy request
pub fn is_replication_request(&self) -> bool {
self.req_params.contains_key("x-rustfs-source-replication-request")
}
}
crates/notify/src/factory.rs
+12 -10
View File
@@ -4,7 +4,7 @@ use crate::{
target::{mqtt::MQTTArgs, webhook::WebhookArgs, Target},
};
use async_trait::async_trait;
use ecstore::config::KVS;
use ecstore::config::{ENABLE_KEY, ENABLE_ON, KVS};
use rumqttc::QoS;
use std::time::Duration;
use tracing::warn;
@@ -13,7 +13,6 @@ use url::Url;
// --- Configuration Constants ---
// General
pub const ENABLE: &str = "enable";
pub const DEFAULT_TARGET: &str = "1";
@@ -37,6 +36,9 @@ pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
pub const NOTIFY_WEBHOOK_SUB_SYS: &str = "notify_webhook";
#[allow(dead_code)]
pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS];
// Webhook Keys
pub const WEBHOOK_ENDPOINT: &str = "endpoint";
pub const WEBHOOK_AUTH_TOKEN: &str = "auth_token";
@@ -111,8 +113,8 @@ impl TargetFactory for WebhookTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target + Send + Sync>, TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config);
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE)
.map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true"))
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
@@ -151,8 +153,8 @@ impl TargetFactory for WebhookTargetFactory {
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config);
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE)
.map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true"))
let enable = get(ENV_WEBHOOK_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
@@ -189,8 +191,8 @@ impl TargetFactory for MQTTTargetFactory {
async fn create_target(&self, id: String, config: &KVS) -> Result<Box<dyn Target + Send + Sync>, TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(&id, base_env_key, config_key, config);
let enable = get(ENV_MQTT_ENABLE, ENABLE)
.map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true"))
let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
@@ -252,8 +254,8 @@ impl TargetFactory for MQTTTargetFactory {
fn validate_config(&self, id: &str, config: &KVS) -> Result<(), TargetError> {
let get = |base_env_key: &str, config_key: &str| get_config_value(id, base_env_key, config_key, config);
let enable = get(ENV_MQTT_ENABLE, ENABLE)
.map(|v| v.eq_ignore_ascii_case("on") || v.eq_ignore_ascii_case("true"))
let enable = get(ENV_MQTT_ENABLE, ENABLE_KEY)
.map(|v| v.eq_ignore_ascii_case(ENABLE_ON) || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
if !enable {
crates/notify/src/global.rs
+56 -8
View File
@@ -1,12 +1,60 @@
use crate::NotificationSystem;
use crate::{Event, EventArgs, NotificationError, NotificationSystem};
use ecstore::config::Config;
use once_cell::sync::Lazy;
use std::sync::Arc;
use std::sync::{Arc, OnceLock};
static NOTIFICATION_SYSTEM: Lazy<Arc<NotificationSystem>> =
Lazy::new(|| Arc::new(NotificationSystem::new()));
static NOTIFICATION_SYSTEM: OnceLock<Arc<NotificationSystem>> = OnceLock::new();
// Create a globally unique Notifier instance
pub static GLOBAL_NOTIFIER: Lazy<Notifier> = Lazy::new(|| Notifier {});
/// Returns the handle to the global NotificationSystem instance.
/// This function can be called anywhere you need to interact with the notification system。
pub fn notification_system() -> Arc<NotificationSystem> {
NOTIFICATION_SYSTEM.clone()
/// Initialize the global notification system with the given configuration.
/// This function should only be called once throughout the application life cycle.
pub async fn initialize(config: Config) -> Result<(), NotificationError> {
// `new` is synchronous and responsible for creating instances
let system = NotificationSystem::new(config);
// `init` is asynchronous and responsible for performing I/O-intensive initialization
system.init().await?;
match NOTIFICATION_SYSTEM.set(Arc::new(system)) {
Ok(_) => Ok(()),
Err(_) => Err(NotificationError::AlreadyInitialized),
}
}
/// Returns a handle to the global NotificationSystem instance.
/// Return None if the system has not been initialized.
pub fn notification_system() -> Option<Arc<NotificationSystem>> {
NOTIFICATION_SYSTEM.get().cloned()
}
pub struct Notifier {
// Notifier can hold state, but in this design we make it stateless,
// Rely on getting an instance of NotificationSystem from the outside.
}
impl crate::notifier::Notifier {
/// Notify an event asynchronously.
/// This is the only entry point for all event notifications in the system.
pub async fn notify(&self, args: EventArgs) {
// Dependency injection or service positioning mode obtain NotificationSystem instance
let notification_sys = match notification_system() {
// If the notification system itself cannot be retrieved, it will be returned directly
Some(sys) => sys,
None => {
tracing::error!("Notification system is not initialized.");
return;
}
};
// Avoid generating notifications for replica creation events
if args.is_replication_request() {
return;
}
// Create an event and send it
let event = Event::new(args.clone());
notification_sys
.send_event(&args.bucket_name, &args.event_name.as_str(), &args.object.name.clone(), event)
.await;
}
}
crates/notify/src/integration.rs
+89 -55
View File
@@ -40,7 +40,7 @@ impl NotificationMetrics {
}
}
// 提供公共方法增加计数
// Provide public methods to increase count
pub fn increment_processing(&self) {
self.processing_events.fetch_add(1, Ordering::Relaxed);
}
@@ -55,7 +55,7 @@ impl NotificationMetrics {
self.failed_events.fetch_add(1, Ordering::Relaxed);
}
// 提供公共方法获取计数
// Provide public methods to get count
pub fn processing_count(&self) -> usize {
self.processing_events.load(Ordering::Relaxed)
}
@@ -89,19 +89,13 @@ pub struct NotificationSystem {
metrics: Arc<NotificationMetrics>,
}
impl Default for NotificationSystem {
fn default() -> Self {
Self::new()
}
}
impl NotificationSystem {
/// Creates a new NotificationSystem
pub fn new() -> Self {
pub fn new(config: Config) -> Self {
NotificationSystem {
notifier: Arc::new(EventNotifier::new()),
registry: Arc::new(TargetRegistry::new()),
config: Arc::new(RwLock::new(Config::new())),
config: Arc::new(RwLock::new(config)),
stream_cancellers: Arc::new(RwLock::new(HashMap::new())),
concurrency_limiter: Arc::new(Semaphore::new(
std::env::var("RUSTFS_TARGET_STREAM_CONCURRENCY")
@@ -194,49 +188,43 @@ impl NotificationSystem {
pub async fn remove_target(&self, target_id: &TargetID, target_type: &str) -> Result<(), NotificationError> {
info!("Attempting to remove target: {}", target_id);
// Step 1: Stop the event stream (if present)
let mut cancellers_guard = self.stream_cancellers.write().await;
if let Some(cancel_tx) = cancellers_guard.remove(target_id) {
info!("Stopping event stream for target {}", target_id);
// Send a stop signal and continue execution even if it fails, because the receiver may have been closed
if let Err(e) = cancel_tx.send(()).await {
error!("Failed to send stop signal to target {} stream: {}", target_id, e);
}
} else {
info!("No active event stream found for target {}, skipping stop.", target_id);
}
drop(cancellers_guard);
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
// Step 2: Remove the Target instance from the activity list of Notifier
// TargetList::remove_target_only will call target.close()
let target_list = self.notifier.target_list();
let mut target_list_guard = target_list.write().await;
if target_list_guard.remove_target_only(target_id).await.is_some() {
info!("Removed target {} from the active list.", target_id);
} else {
warn!("Target {} was not found in the active list.", target_id);
}
drop(target_list_guard);
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
// Step 3: Remove Target from persistent configuration
let mut config_guard = self.config.write().await;
let mut changed = false;
if let Some(targets_of_type) = config_guard.0.get_mut(target_type) {
if let Some(targets_of_type) = new_config.0.get_mut(target_type) {
if targets_of_type.remove(&target_id.name).is_some() {
info!("Removed target {} from the configuration.", target_id);
changed = true;
}
// If there are no targets under this type, remove the entry for this type
if targets_of_type.is_empty() {
config_guard.0.remove(target_type);
new_config.0.remove(target_type);
}
}
if !changed {
warn!("Target {} was not found in the configuration.", target_id);
return Ok(());
}
Ok(())
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save config for target removal: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save config: {}", e)));
}
info!(
"Configuration updated and persisted for target {} removal. Reloading system...",
target_id
);
self.reload_config(new_config).await
}
/// Set or update a Target configuration.
@@ -253,18 +241,49 @@ impl NotificationSystem {
/// If the target configuration is invalid, it returns Err(NotificationError::Configuration).
pub async fn set_target_config(&self, target_type: &str, target_name: &str, kvs: KVS) -> Result<(), NotificationError> {
info!("Setting config for target {} of type {}", target_name, target_type);
let mut config_guard = self.config.write().await;
config_guard
// 1. Get the storage handle
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
// 2. Read the latest configuration from storage
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
// 3. Modify the configuration copy
new_config
.0
.entry(target_type.to_string())
.or_default()
.insert(target_name.to_string(), kvs);
let new_config = config_guard.clone();
// Release the lock before calling reload_config
drop(config_guard);
// 4. Persist the new configuration
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save notification config: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save notification config: {}", e)));
}
self.reload_config(new_config).await
// 5. After the persistence is successful, the system will be reloaded to apply changes.
match self.reload_config(new_config).await {
Ok(_) => {
info!(
"Target {} of type {} configuration updated and reloaded successfully",
target_name, target_type
);
Ok(())
}
Err(e) => {
error!("Failed to reload config for target {} of type {}: {}", target_name, target_type, e);
Err(NotificationError::Configuration(format!(
"Configuration saved, but failed to reload: {}",
e
)))
}
}
}
/// Removes all notification configurations for a bucket.
@@ -286,27 +305,42 @@ impl NotificationSystem {
/// If the target configuration does not exist, it returns Ok(()) without making any changes.
pub async fn remove_target_config(&self, target_type: &str, target_name: &str) -> Result<(), NotificationError> {
info!("Removing config for target {} of type {}", target_name, target_type);
let mut config_guard = self.config.write().await;
let mut changed = false;
let Some(store) = ecstore::global::new_object_layer_fn() else {
return Err(NotificationError::Io(std::io::Error::new(
std::io::ErrorKind::Other,
"errServerNotInitialized",
)));
};
if let Some(targets) = config_guard.0.get_mut(target_type) {
let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| NotificationError::Configuration(format!("Failed to read notification config: {}", e)))?;
let mut changed = false;
if let Some(targets) = new_config.0.get_mut(target_type) {
if targets.remove(target_name).is_some() {
changed = true;
}
if targets.is_empty() {
config_guard.0.remove(target_type);
new_config.0.remove(target_type);
}
}
if changed {
let new_config = config_guard.clone();
// Release the lock before calling reload_config
drop(config_guard);
self.reload_config(new_config).await
} else {
if !changed {
info!("Target {} of type {} not found, no changes made.", target_name, target_type);
Ok(())
return Ok(());
}
if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await {
error!("Failed to save config for target removal: {}", e);
return Err(NotificationError::Configuration(format!("Failed to save config: {}", e)));
}
info!(
"Configuration updated and persisted for target {} removal. Reloading system...",
target_name
);
self.reload_config(new_config).await
}
/// Enhanced event stream startup function, including monitoring and concurrency control
crates/notify/src/lib.rs
+2 -3
View File
@@ -4,7 +4,6 @@
//! similar to RustFS's notification system. It supports sending events to various targets
//! (like Webhook and MQTT) and includes features like event persistence and retry on failure.
pub mod args;
pub mod arn;
pub mod error;
pub mod event;
@@ -17,11 +16,11 @@ pub mod rules;
pub mod store;
pub mod stream;
pub mod target;
pub mod utils;
// Re-exports
pub use error::{NotificationError, StoreError, TargetError};
pub use event::{Event, EventLog, EventName};
pub use event::{Event, EventArgs, EventLog, EventName};
pub use global::{initialize, notification_system};
pub use integration::NotificationSystem;
pub use rules::BucketNotificationConfig;
use std::io::IsTerminal;
crates/notify/src/notifier.rs
+22 -45
View File
@@ -96,57 +96,42 @@ impl EventNotifier {
let target_ids_len = target_ids.len();
let mut handles = vec![];
// 使用作用域来限制 target_list 的借用范围
// Use scope to limit the borrow scope of target_list
{
let target_list_guard = self.target_list.read().await;
info!("Sending event to targets: {:?}", target_ids);
for target_id in target_ids {
// `get` now returns Option<Arc<dyn Target + Send + Sync>>
if let Some(target_arc) = target_list_guard.get(&target_id) {
// 克隆 Arc<Box<dyn Target>> (target_list 存储的就是这个类型) 以便移入异步任务
// Clone an Arc<Box<dyn Target>> (which is where target_list is stored) to move into an asynchronous task
// target_arc is already Arc, clone it for the async task
let cloned_target_for_task = target_arc.clone();
let event_clone = event.clone();
let target_name_for_task = cloned_target_for_task.name(); // 在生成任务前获取名称
debug!(
"Preparing to send event to target: {}",
target_name_for_task
);
// 在闭包中使用克隆的数据，避免借用冲突
let target_name_for_task = cloned_target_for_task.name(); // Get the name before generating the task
debug!("Preparing to send event to target: {}", target_name_for_task);
// Use cloned data in closures to avoid borrowing conflicts
let handle = tokio::spawn(async move {
if let Err(e) = cloned_target_for_task.save(event_clone).await {
error!(
"Failed to send event to target {}: {}",
target_name_for_task, e
);
error!("Failed to send event to target {}: {}", target_name_for_task, e);
} else {
debug!(
"Successfully saved event to target {}",
target_name_for_task
);
debug!("Successfully saved event to target {}", target_name_for_task);
}
});
handles.push(handle);
} else {
warn!(
"Target ID {:?} found in rules but not in target list.",
target_id
);
warn!("Target ID {:?} found in rules but not in target list.", target_id);
}
}
// target_list 在这里自动释放
// target_list is automatically released here
}
// 等待所有任务完成
// Wait for all tasks to be completed
for handle in handles {
if let Err(e) = handle.await {
error!("Task for sending/saving event failed: {}", e);
}
}
info!(
"Event processing initiated for {} targets for bucket: {}",
target_ids_len, bucket_name
);
info!("Event processing initiated for {} targets for bucket: {}", target_ids_len, bucket_name);
} else {
debug!("No rules found for bucket: {}", bucket_name);
}
@@ -158,22 +143,22 @@ impl EventNotifier {
&self,
targets_to_init: Vec<Box<dyn Target + Send + Sync>>,
) -> Result<(), NotificationError> {
// 当前激活的、更简单的逻辑：
let mut target_list_guard = self.target_list.write().await; // 获取 TargetList 的写锁
// Currently active, simpler logic
let mut target_list_guard = self.target_list.write().await; //Gets a write lock for the TargetList
for target_boxed in targets_to_init {
// 遍历传入的 Box<dyn Target>
// Traverse the incoming Box<dyn Target >
debug!("init bucket target: {}", target_boxed.name());
// TargetList::add 方法期望 Arc<dyn Target + Send + Sync>
// 因此，需要将 Box<dyn Target + Send + Sync> 转换为 Arc<dyn Target + Send + Sync>
// TargetList::add method expectations Arc<dyn Target + Send + Sync>
// Therefore, you need to convert Box<dyn Target + Send + Sync> to Arc<dyn Target + Send + Sync>
let target_arc: Arc<dyn Target + Send + Sync> = Arc::from(target_boxed);
target_list_guard.add(target_arc)?; // 将 Arc<dyn Target> 添加到列表中
target_list_guard.add(target_arc)?; // Add Arc<dyn Target> to the list
}
info!(
"Initialized {} targets, list size: {}", // 更清晰的日志
"Initialized {} targets, list size: {}", // Clearer logs
target_list_guard.len(),
target_list_guard.len()
);
Ok(()) // 确保返回 Result
Ok(()) // Make sure to return a Result
}
}
@@ -191,9 +176,7 @@ impl Default for TargetList {
impl TargetList {
/// Creates a new TargetList
pub fn new() -> Self {
TargetList {
targets: HashMap::new(),
}
TargetList { targets: HashMap::new() }
}
/// Adds a target to the list
@@ -201,10 +184,7 @@ impl TargetList {
let id = target.id();
if self.targets.contains_key(&id) {
// Potentially update or log a warning/error if replacing an existing target.
warn!(
"Target with ID {} already exists in TargetList. It will be overwritten.",
id
);
warn!("Target with ID {} already exists in TargetList. It will be overwritten.", id);
}
self.targets.insert(id, target);
Ok(())
@@ -212,10 +192,7 @@ impl TargetList {
/// Removes a target by ID. Note: This does not stop its associated event stream.
/// Stream cancellation should be handled by EventNotifier.
pub async fn remove_target_only(
&mut self,
id: &TargetID,
) -> Option<Arc<dyn Target + Send + Sync>> {
pub async fn remove_target_only(&mut self, id: &TargetID) -> Option<Arc<dyn Target + Send + Sync>> {
if let Some(target_arc) = self.targets.remove(id) {
if let Err(e) = target_arc.close().await {
// Target's own close logic
crates/notify/src/registry.rs
+2 -8
View File
@@ -4,7 +4,7 @@ use crate::{
factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory},
target::Target,
};
use ecstore::config::{Config, KVS};
use ecstore::config::{Config, ENABLE_KEY, ENABLE_OFF, ENABLE_ON, KVS};
use std::collections::HashMap;
use tracing::{error, info};
@@ -74,7 +74,7 @@ impl TargetRegistry {
// Iterate through subsections (each representing a target instance)
for (target_id, target_config) in subsections {
// Skip disabled targets
if target_config.lookup("enable").unwrap_or_else(|| "off".to_string()) != "on" {
if target_config.lookup(ENABLE_KEY).unwrap_or_else(|| ENABLE_OFF.to_string()) != ENABLE_ON {
continue;
}
@@ -94,9 +94,3 @@ impl TargetRegistry {
Ok(targets)
}
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn test_target_registry() {}
}
crates/notify/src/target/webhook.rs
+48 -100
View File
@@ -4,7 +4,6 @@ use crate::{
arn::TargetID, error::TargetError,
event::{Event, EventLog},
store::{Key, Store},
utils,
StoreError,
Target,
};
@@ -56,18 +55,14 @@ impl WebhookArgs {
if !self.queue_dir.is_empty() {
let path = std::path::Path::new(&self.queue_dir);
if !path.is_absolute() {
return Err(TargetError::Configuration(
"webhook queueDir path should be absolute".to_string(),
));
return Err(TargetError::Configuration("webhook queueDir path should be absolute".to_string()));
}
}
if !self.client_cert.is_empty() && self.client_key.is_empty()
|| self.client_cert.is_empty() && !self.client_key.is_empty()
{
return Err(TargetError::Configuration(
"cert and key must be specified as a pair".to_string(),
));
return Err(TargetError::Configuration("cert and key must be specified as a pair".to_string()));
}
Ok(())
@@ -79,7 +74,7 @@ pub struct WebhookTarget {
id: TargetID,
args: WebhookArgs,
http_client: Arc<Client>,
// 添加 Send + Sync 约束确保线程安全
// Add Send + Sync constraints to ensure thread safety
store: Option<Box<dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync>>,
initialized: AtomicBool,
addr: String,
@@ -103,36 +98,35 @@ impl WebhookTarget {
/// Creates a new WebhookTarget
#[instrument(skip(args), fields(target_id = %id))]
pub fn new(id: String, args: WebhookArgs) -> Result<Self, TargetError> {
// 首先验证参数
// First verify the parameters
args.validate()?;
// 创建 TargetID
// Create a TargetID
let target_id = TargetID::new(id, ChannelTargetType::Webhook.as_str().to_string());
// 构建 HTTP client
// Build HTTP client
let mut client_builder = Client::builder()
.timeout(Duration::from_secs(30))
.user_agent(utils::get_user_agent(utils::ServiceType::Basis));
.user_agent(rustfs_utils::sys::get_user_agent(rustfs_utils::sys::ServiceType::Basis));
// 补充证书处理逻辑
// Supplementary certificate processing logic
if !args.client_cert.is_empty() && !args.client_key.is_empty() {
// 添加客户端证书
let cert = std::fs::read(&args.client_cert).map_err(|e| {
TargetError::Configuration(format!("Failed to read client cert: {}", e))
})?;
let key = std::fs::read(&args.client_key).map_err(|e| {
TargetError::Configuration(format!("Failed to read client key: {}", e))
})?;
// Add client certificate
let cert = std::fs::read(&args.client_cert)
.map_err(|e| TargetError::Configuration(format!("Failed to read client cert: {}", e)))?;
let key = std::fs::read(&args.client_key)
.map_err(|e| TargetError::Configuration(format!("Failed to read client key: {}", e)))?;
let identity = reqwest::Identity::from_pem(&[cert, key].concat()).map_err(|e| {
TargetError::Configuration(format!("Failed to create identity: {}", e))
})?;
let identity = reqwest::Identity::from_pem(&[cert, key].concat())
.map_err(|e| TargetError::Configuration(format!("Failed to create identity: {}", e)))?;
client_builder = client_builder.identity(identity);
}
let http_client = Arc::new(client_builder.build().map_err(|e| {
TargetError::Configuration(format!("Failed to build HTTP client: {}", e))
})?);
let http_client = Arc::new(
client_builder
.build()
.map_err(|e| TargetError::Configuration(format!("Failed to build HTTP client: {}", e)))?,
);
// 构建存储
// Build storage
let queue_store = if !args.queue_dir.is_empty() {
let queue_dir = PathBuf::from(&args.queue_dir).join(format!(
"rustfs-{}-{}-{}",
@@ -140,43 +134,30 @@ impl WebhookTarget {
target_id.name,
target_id.id
));
let store = super::super::store::QueueStore::<Event>::new(
queue_dir,
args.queue_limit,
STORE_EXTENSION,
);
let store = super::super::store::QueueStore::<Event>::new(queue_dir, args.queue_limit, STORE_EXTENSION);
if let Err(e) = store.open() {
error!(
"Failed to open store for Webhook target {}: {}",
target_id.id, e
);
error!("Failed to open store for Webhook target {}: {}", target_id.id, e);
return Err(TargetError::Storage(format!("{}", e)));
}
// 确保 QueueStore 实现的 Store trait 匹配预期的错误类型
Some(Box::new(store)
as Box<
dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync,
>)
// Make sure that the Store trait implemented by QueueStore matches the expected error type
Some(Box::new(store) as Box<dyn Store<Event, Error = StoreError, Key = Key> + Send + Sync>)
} else {
None
};
// 解析地址
// resolved address
let addr = {
let host = args.endpoint.host_str().unwrap_or("localhost");
let port = args.endpoint.port().unwrap_or_else(|| {
if args.endpoint.scheme() == "https" {
443
} else {
80
}
});
let port = args
.endpoint
.port()
.unwrap_or_else(|| if args.endpoint.scheme() == "https" { 443 } else { 80 });
format!("{}:{}", host, port)
};
// 创建取消通道
// Create a cancel channel
let (cancel_sender, _) = mpsc::channel(1);
info!(target_id = %target_id.id, "Webhook target created");
Ok(WebhookTarget {
@@ -202,10 +183,7 @@ impl WebhookTarget {
return Err(TargetError::NotConnected);
}
Err(e) => {
error!(
"Failed to check if Webhook target {} is active: {}",
self.id, e
);
error!("Failed to check if Webhook target {} is active: {}", self.id, e);
return Err(e);
}
}
@@ -228,17 +206,13 @@ impl WebhookTarget {
records: vec![event.clone()],
};
let data = serde_json::to_vec(&log)
.map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?;
let data =
serde_json::to_vec(&log).map_err(|e| TargetError::Serialization(format!("Failed to serialize event: {}", e)))?;
// Vec<u8> 转换为 String
let data_string = String::from_utf8(data.clone()).map_err(|e| {
TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e))
})?;
debug!(
"Sending event to webhook target: {}, event log: {}",
self.id, data_string
);
let data_string = String::from_utf8(data.clone())
.map_err(|e| TargetError::Encoding(format!("Failed to convert event data to UTF-8: {}", e)))?;
debug!("Sending event to webhook target: {}, event log: {}", self.id, data_string);
// 构建请求
let mut req_builder = self
@@ -256,8 +230,7 @@ impl WebhookTarget {
}
1 => {
// 只有令牌，需要添加 "Bearer" 前缀
req_builder = req_builder
.header("Authorization", format!("Bearer {}", self.args.auth_token));
req_builder = req_builder.header("Authorization", format!("Bearer {}", self.args.auth_token));
}
_ => {
// 空字符串或其他情况，不添加认证头
@@ -305,16 +278,8 @@ impl Target for WebhookTarget {
.map_err(|e| TargetError::Network(format!("Failed to resolve host: {}", e)))?
.next()
.ok_or_else(|| TargetError::Network("No address found".to_string()))?;
debug!(
"is_active socket addr: {},target id:{}",
socket_addr, self.id.id
);
match tokio::time::timeout(
Duration::from_secs(5),
tokio::net::TcpStream::connect(socket_addr),
)
.await
{
debug!("is_active socket addr: {},target id:{}", socket_addr, self.id.id);
match tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(socket_addr)).await {
Ok(Ok(_)) => {
debug!("Connection to {} is active", self.addr);
Ok(true)
@@ -334,9 +299,9 @@ impl Target for WebhookTarget {
async fn save(&self, event: Event) -> Result<(), TargetError> {
if let Some(store) = &self.store {
// Call the store method directly, no longer need to acquire the lock
store.put(event).map_err(|e| {
TargetError::Storage(format!("Failed to save event to store: {}", e))
})?;
store
.put(event)
.map_err(|e| TargetError::Storage(format!("Failed to save event to store: {}", e)))?;
debug!("Event saved to store for target: {}", self.id);
Ok(())
} else {
@@ -373,10 +338,7 @@ impl Target for WebhookTarget {
Ok(event) => event,
Err(StoreError::NotFound) => return Ok(()),
Err(e) => {
return Err(TargetError::Storage(format!(
"Failed to get event from store: {}",
e
)));
return Err(TargetError::Storage(format!("Failed to get event from store: {}", e)));
}
};
@@ -388,23 +350,12 @@ impl Target for WebhookTarget {
}
// Use the immutable reference of the store to delete the event content corresponding to the key
debug!(
"Deleting event from store for target: {}, key:{}, start",
self.id,
key.to_string()
);
debug!("Deleting event from store for target: {}, key:{}, start", self.id, key.to_string());
match store.del(&key) {
Ok(_) => debug!(
"Event deleted from store for target: {}, key:{}, end",
self.id,
key.to_string()
),
Ok(_) => debug!("Event deleted from store for target: {}, key:{}, end", self.id, key.to_string()),
Err(e) => {
error!("Failed to delete event from store: {}", e);
return Err(TargetError::Storage(format!(
"Failed to delete event from store: {}",
e
)));
return Err(TargetError::Storage(format!("Failed to delete event from store: {}", e)));
}
}
@@ -433,10 +384,7 @@ impl Target for WebhookTarget {
async fn init(&self) -> Result<(), TargetError> {
// If the target is disabled, return to success directly
if !self.is_enabled() {
debug!(
"Webhook target {} is disabled, skipping initialization",
self.id
);
debug!("Webhook target {} is disabled, skipping initialization", self.id);
return Ok(());
}
ecstore/src/config/com.rs
+44 -41
View File
@@ -115,59 +115,62 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
Ok(cfg)
}
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
let data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if err == Error::ConfigNotFound {
warn!("config not found, start to init");
let cfg = new_and_save_server_config(api).await?;
warn!("config init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
};
}
};
fn get_config_file() -> String {
format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE)
}
read_server_config(api, data.as_slice()).await
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
warn!("Configuration not found ({}): Start initializing new configuration", context);
let cfg = new_and_save_server_config(api).await?;
warn!("Configuration initialization complete ({})", context);
Ok(cfg)
}
/// Handle configuration file read errors
fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
error!("Read configuration failed (path: '{}'): {:?}", file_path, err);
Err(err)
}
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
let config_file = get_config_file();
// Try to read the configuration file
match read_config(api.clone(), &config_file).await {
Ok(data) => read_server_config(api, &data).await,
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
Err(err) => handle_config_read_error(err, &config_file),
}
}
async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<Config> {
let cfg = {
if data.is_empty() {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
let cfg_data = match read_config(api.clone(), config_file.as_str()).await {
Ok(res) => res,
Err(err) => {
return if err == Error::ConfigNotFound {
warn!("config not found init start");
let cfg = new_and_save_server_config(api).await?;
warn!("config not found init done");
Ok(cfg)
} else {
error!("read config err {:?}", &err);
Err(err)
};
}
};
// TODO: decrypt
// If the provided data is empty, try to read from the file again
if data.is_empty() {
let config_file = get_config_file();
warn!("Received empty configuration data, try to reread from '{}'", config_file);
Config::unmarshal(cfg_data.as_slice())?
} else {
Config::unmarshal(data)?
// Try to read the configuration again
match read_config(api.clone(), &config_file).await {
Ok(cfg_data) => {
// TODO: decrypt
let cfg = Config::unmarshal(&cfg_data)?;
return Ok(cfg.merge());
}
Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
Err(err) => return handle_config_read_error(err, &config_file),
}
};
}
// Process non-empty configuration data
let cfg = Config::unmarshal(data)?;
Ok(cfg.merge())
}
async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
pub async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
let data = cfg.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
let config_file = get_config_file();
save_config(api, &config_file, data).await
}
ecstore/src/config/mod.rs
+2 -2
View File
@@ -107,8 +107,8 @@ impl Config {
cfg
}
pub fn get_value(&self, subsys: &str, key: &str) -> Option<KVS> {
if let Some(m) = self.0.get(subsys) {
pub fn get_value(&self, sub_sys: &str, key: &str) -> Option<KVS> {
if let Some(m) = self.0.get(sub_sys) {
m.get(key).cloned()
} else {
None
rustfs/src/event.rs
+26 -24
View File
@@ -1,32 +1,34 @@
// use rustfs_notify::EventNotifierConfig;
use tracing::{info, instrument};
use ecstore::config::GLOBAL_ServerConfig;
use tracing::{error, info, instrument};
#[instrument]
pub(crate) async fn init_event_notifier(notifier_config: Option<String>) {
pub(crate) async fn init_event_notifier() {
info!("Initializing event notifier...");
let notifier_config_present = notifier_config.is_some();
let config = if notifier_config_present {
info!("event_config is not empty, path: {:?}", notifier_config);
// EventNotifierConfig::event_load_config(notifier_config)
} else {
info!("event_config is empty");
// rustfs_notify::get_event_notifier_config().clone()
// EventNotifierConfig::default()
// 1. Get the global configuration loaded by ecstore
let server_config = match GLOBAL_ServerConfig.get() {
Some(config) => config.clone(), // Clone the config to pass ownership
None => {
error!("Event notifier initialization failed: Global server config not loaded.");
return;
}
};
info!("using event_config: {:?}", config);
// 2. Check if the notify subsystem exists in the configuration, and skip initialization if it doesn't
if server_config.get_value("notify", "_").is_none() {
info!("'notify' subsystem not configured, skipping event notifier initialization.");
return;
}
info!("Event notifier configuration found, proceeding with initialization.");
// 3. Initialize the notification system asynchronously with a global configuration
// Put it into a separate task to avoid blocking the main initialization process
tokio::spawn(async move {
// let result = rustfs_notify::initialize(&config).await;
// match result {
// Ok(_) => info!(
// "event notifier initialized successfully {}",
// if notifier_config_present {
// "by config file"
// } else {
// "by sys config"
// }
// ),
// Err(e) => error!("Failed to initialize event notifier: {}", e),
// }
if let Err(e) = rustfs_notify::initialize(server_config).await {
error!("Failed to initialize event notifier system: {}", e);
} else {
info!("Event notifier system initialized successfully.");
}
});
}
rustfs/src/storage/ecfs.rs
+29 -12
View File
@@ -16,8 +16,8 @@ use bytes::Bytes;
use chrono::DateTime;
use chrono::Utc;
use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use datafusion::arrow::json::writer::JsonArray;
use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder;
use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG;
use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG;
@@ -32,13 +32,13 @@ use ecstore::bucket::tagging::decode_tags;
use ecstore::bucket::tagging::encode_tags;
use ecstore::bucket::utils::serialize;
use ecstore::bucket::versioning_sys::BucketVersioningSys;
use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::cmd::bucket_replication::get_must_replicate_options;
use ecstore::cmd::bucket_replication::must_replicate;
use ecstore::cmd::bucket_replication::schedule_replication;
use ecstore::compress::MIN_COMPRESSIBLE_SIZE;
use ecstore::cmd::bucket_replication::ReplicationStatusType;
use ecstore::cmd::bucket_replication::ReplicationType;
use ecstore::compress::is_compressible;
use ecstore::compress::MIN_COMPRESSIBLE_SIZE;
use ecstore::error::StorageError;
use ecstore::new_object_layer_fn;
use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
@@ -52,40 +52,42 @@ use ecstore::store_api::ObjectIO;
use ecstore::store_api::ObjectOptions;
use ecstore::store_api::ObjectToDelete;
use ecstore::store_api::PutObjReader;
use ecstore::store_api::StorageAPI; // use ecstore::store_api::RESERVED_METADATA_PREFIX;
use ecstore::store_api::StorageAPI;
// use ecstore::store_api::RESERVED_METADATA_PREFIX;
use futures::StreamExt;
use http::HeaderMap;
use lazy_static::lazy_static;
use policy::auth;
use policy::policy::action::Action;
use policy::policy::action::S3Action;
use policy::policy::BucketPolicy;
use policy::policy::BucketPolicyArgs;
use policy::policy::Validator;
use policy::policy::action::Action;
use policy::policy::action::S3Action;
use query::instance::make_rustfsms;
use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_filemeta::headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING};
use rustfs_notify::EventName;
use rustfs_rio::CompressReader;
use rustfs_rio::HashReader;
use rustfs_rio::Reader;
use rustfs_rio::WarpReader;
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::path::path_join_buf;
use rustfs_utils::CompressionAlgorithm;
use rustfs_zip::CompressionFormat;
use s3s::S3;
use s3s::dto::*;
use s3s::s3_error;
use s3s::S3Error;
use s3s::S3ErrorCode;
use s3s::S3Result;
use s3s::dto::*;
use s3s::s3_error;
use s3s::S3;
use s3s::{S3Request, S3Response};
use std::collections::HashMap;
use std::fmt::Debug;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use time::OffsetDateTime;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use tokio_tar::Archive;
@@ -222,6 +224,21 @@ impl FS {
// e_tag,
// ..Default::default()
// };
// let event_args = rustfs_notify::event::EventArgs {
// event_name: EventName::ObjectCreatedPut, // 或者其他相应的事件类型
// bucket_name: bucket.clone(),
// object: _obj_info.clone(), // clone() 或传递所需字段
// req_params: crate::storage::global::extract_req_params(&req), // 假设有一个辅助函数来提取请求参数
// resp_elements: crate::storage::global::extract_resp_elements(&output), // 假设有一个辅助函数来提取响应元素
// host: crate::storage::global::get_request_host(&req.headers), // 假设的辅助函数
// user_agent: crate::storage::global::get_request_user_agent(&req.headers), // 假设的辅助函数
// };
//
// // 异步调用，不会阻塞当前请求的响应
// tokio::spawn(async move {
// rustfs_notify::notifier::GLOBAL_NOTIFIER.notify(event_args).await;
// });
}
}
rustfs/src/storage/event.rs
-1
View File
@@ -1 +0,0 @@
rustfs/src/storage/global.rs
+43
View File
@@ -0,0 +1,43 @@
use hyper::HeaderMap;
use s3s::{S3Request, S3Response};
use std::collections::HashMap;
/// Extract request parameters from S3Request, mainly header information.
pub fn extract_req_params<T>(req: &S3Request<T>) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in req.headers.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Extract response elements from S3Response, mainly header information.
pub fn extract_resp_elements<T>(resp: &S3Response<T>) -> HashMap<String, String> {
let mut params = HashMap::new();
for (key, value) in resp.headers.iter() {
if let Ok(val_str) = value.to_str() {
params.insert(key.as_str().to_string(), val_str.to_string());
}
}
params
}
/// Get host from header information.
pub fn get_request_host(headers: &HeaderMap) -> String {
headers
.get("host")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
/// Get user-agent from header information.
pub fn get_request_user_agent(headers: &HeaderMap) -> String {
headers
.get("user-agent")
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string()
}
rustfs/src/storage/mod.rs
+1 -1
View File
@@ -1,5 +1,5 @@
pub mod access;
pub mod ecfs;
// pub mod error;
mod event;
mod global;
pub mod options;