feat(event-notifier): improve environment variable handling

- Fix deserialization error when parsing config from environment variables
- Add proper array format support for adapters configuration
- Update environment variable examples with correct format
- Improve documentation for configuration loading
- Implement helper functions for environment variable validation

This change fixes the "invalid type: map, expected a sequence" error
by ensuring proper formatting of array-type fields in environment variables.
This commit is contained in:
houseme
2025-04-22 20:31:38 +08:00
parent 15b6a426fb
commit e4453adf82
18 changed files with 345 additions and 261 deletions
@@ -0,0 +1,28 @@
# ===== 全局配置 =====
NOTIFIER__STORE_PATH=/var/log/event-notification
NOTIFIER__CHANNEL_CAPACITY=5000
# ===== 适配器配置(数组格式) =====
# Webhook 适配器(索引 0
NOTIFIER__ADAPTERS_0__type=Webhook
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
NOTIFIER__ADAPTERS_0__max_retries=3
NOTIFIER__ADAPTERS_0__timeout=50
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
# Kafka 适配器(索引 1
NOTIFIER__ADAPTERS_1__type=Kafka
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
NOTIFIER__ADAPTERS_1__topic=notifications
NOTIFIER__ADAPTERS_1__max_retries=3
NOTIFIER__ADAPTERS_1__timeout=60
# MQTT 适配器(索引 2
NOTIFIER__ADAPTERS_2__type=Mqtt
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
NOTIFIER__ADAPTERS_2__port=1883
NOTIFIER__ADAPTERS_2__client_id=event-notifier
NOTIFIER__ADAPTERS_2__topic=events
NOTIFIER__ADAPTERS_2__max_retries=3
+25 -24
View File
@@ -1,27 +1,28 @@
# basic configuration
EVENT_NOTIF_STORE_PATH=/var/log/event-notification
EVENT_NOTIF_CHANNEL_CAPACITY=5000
# ===== global configuration =====
NOTIFIER__STORE_PATH=/var/log/event-notification
NOTIFIER__CHANNEL_CAPACITY=5000
# webhook adapter configuration
EVENT_NOTIF_ADAPTERS__0__TYPE=Webhook
EVENT_NOTIF_ADAPTERS__0__ENDPOINT=https://api.example.com/webhook
EVENT_NOTIF_ADAPTERS__0__AUTH_TOKEN=your-secret-token
EVENT_NOTIF_ADAPTERS__0__MAX_RETRIES=3
EVENT_NOTIF_ADAPTERS__0__TIMEOUT=5000
# ===== adapter configuration array format =====
# webhook adapter index 0
NOTIFIER__ADAPTERS_0__type=Webhook
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
NOTIFIER__ADAPTERS_0__max_retries=3
NOTIFIER__ADAPTERS_0__timeout=50
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
# kafka adapter configuration
EVENT_NOTIF_ADAPTERS__1__TYPE=Kafka
EVENT_NOTIF_ADAPTERS__1__BROKERS=localhost:9092
EVENT_NOTIF_ADAPTERS__1__TOPIC=notifications
EVENT_NOTIF_ADAPTERS__1__MAX_RETRIES=3
EVENT_NOTIF_ADAPTERS__1__TIMEOUT=5000
# kafka adapter index 1
NOTIFIER__ADAPTERS_1__type=Kafka
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
NOTIFIER__ADAPTERS_1__topic=notifications
NOTIFIER__ADAPTERS_1__max_retries=3
NOTIFIER__ADAPTERS_1__timeout=60
# mqtt adapter configuration
EVENT_NOTIF_ADAPTERS__2__TYPE=Mqtt
EVENT_NOTIF_ADAPTERS__2__BROKER=mqtt.example.com
EVENT_NOTIF_ADAPTERS__2__PORT=1883
EVENT_NOTIF_ADAPTERS__2__CLIENT_ID=event-notifier
EVENT_NOTIF_ADAPTERS__2__TOPIC=events
EVENT_NOTIF_ADAPTERS__2__MAX_RETRIES=3
EVENT_NOTIF_HTTP__PORT=8080
# mqtt adapter index 2
NOTIFIER__ADAPTERS_2__type=Mqtt
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
NOTIFIER__ADAPTERS_2__port=1883
NOTIFIER__ADAPTERS_2__client_id=event-notifier
NOTIFIER__ADAPTERS_2__topic=events
NOTIFIER__ADAPTERS_2__max_retries=3
+9 -8
View File
@@ -1,20 +1,24 @@
# config.toml
store_path = "/var/log/event-notification"
store_path = "/var/log/event-notifier"
channel_capacity = 5000
[[adapters]]
type = "Webhook"
endpoint = "https://api.example.com/webhook"
endpoint = "http://127.0.0.1:3000/webhook"
auth_token = "your-auth-token"
max_retries = 3
timeout = 5000
timeout = 50
[adapters.custom_headers]
custom_server = "value_server"
custom_client = "value_client"
[[adapters]]
type = "Kafka"
brokers = "localhost:9092"
topic = "notifications"
max_retries = 3
timeout = 5000
timeout = 60
[[adapters]]
type = "Mqtt"
@@ -22,7 +26,4 @@ broker = "mqtt.example.com"
port = 1883
client_id = "event-notifier"
topic = "events"
max_retries = 3
[http]
port = 8080
max_retries = 3
+5 -7
View File
@@ -1,6 +1,5 @@
use rustfs_event_notifier::{
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotificationConfig, Object, Source,
WebhookConfig,
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object, Source, WebhookConfig,
};
use std::collections::HashMap;
use tokio::signal;
@@ -8,7 +7,7 @@ use tracing::Level;
use tracing_subscriber::FmtSubscriber;
async fn setup_notification_system() -> Result<(), NotifierError> {
let config = NotificationConfig {
let config = NotifierConfig {
store_path: "./deploy/logs/event_store".into(),
channel_capacity: 100,
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
@@ -40,11 +39,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// tracing_subscriber::fmt::init();
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG) // set to debug or lower level
.with_target(false) // simplify output
.with_max_level(Level::DEBUG) // set to debug or lower level
.with_target(false) // simplify output
.finish();
tracing::subscriber::set_global_default(subscriber)
.expect("failed to set up log subscriber");
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
// set up notification system
if let Err(e) = setup_notification_system().await {
+25 -17
View File
@@ -1,21 +1,27 @@
use rustfs_event_notifier::create_adapters;
use rustfs_event_notifier::NotificationSystem;
use rustfs_event_notifier::{AdapterConfig, NotificationConfig, WebhookConfig};
use rustfs_event_notifier::NotifierSystem;
use rustfs_event_notifier::{AdapterConfig, NotifierConfig, WebhookConfig};
use rustfs_event_notifier::{Bucket, Event, Identity, Metadata, Name, Object, Source};
use std::collections::HashMap;
use std::error;
use std::sync::Arc;
use tokio::signal;
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
#[tokio::main]
async fn main() -> Result<(), Box<dyn error::Error>> {
tracing_subscriber::fmt::init();
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG) // set to debug or lower level
.with_target(false) // simplify output
.finish();
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
let config = NotificationConfig {
let config = NotifierConfig {
store_path: "./events".to_string(),
channel_capacity: 100,
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
endpoint: "http://localhost:8080/webhook".to_string(),
endpoint: "http://127.0.0.1:3000/webhook".to_string(),
auth_token: Some("secret-token".to_string()),
custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])),
max_retries: 3,
@@ -23,16 +29,12 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
})],
};
// loading configuration from specific env files
let _config = NotificationConfig::from_env_file(".env.example")?;
// load_config
// loading configuration from environment variables
let _config = NotifierConfig::load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
tracing::info!("load_config config: {:?} \n", _config);
// loading from a specific file
let _config = NotificationConfig::from_file("event.toml")?;
// Automatically load from multiple sources (Priority: Environment Variables > YAML > TOML)
let _config = NotificationConfig::load()?;
let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await?));
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
let adapters = create_adapters(&config.adapters)?;
// create an s3 metadata object
@@ -90,9 +92,15 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
signal::ctrl_c().await?;
tracing::info!("Received shutdown signal");
{
let system = system.lock().await;
system.shutdown();
let result = {
let mut system = system.lock().await;
system.shutdown().await
};
if let Err(e) = result {
tracing::error!("Failed to shut down the notification system: {}", e);
} else {
tracing::info!("Notification system shut down successfully");
}
system_handle.await??;
+66 -1
View File
@@ -12,6 +12,71 @@ async fn main() {
}
async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
println!("收到 webhook 请求 time: {},内容:{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), serde_json::to_string_pretty(&payload).unwrap());
let start = SystemTime::now();
let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards");
// get the number of seconds since the unix era
let seconds = since_the_epoch.as_secs();
// Manually calculate year, month, day, hour, minute, and second
let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds);
// output result
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
println!(
"received a webhook request time:{} content:\n {}",
seconds.to_string(),
serde_json::to_string_pretty(&payload).unwrap()
);
StatusCode::OK
}
fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
// assume that the time zone is utc
let seconds_per_minute = 60;
let seconds_per_hour = 3600;
let seconds_per_day = 86400;
// Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds
let mut total_seconds = seconds;
let mut year = 1970;
let mut month = 1;
let mut day = 1;
let mut hour = 0;
let mut minute = 0;
let mut second = 0;
// calculate year
while total_seconds >= 31536000 {
year += 1;
total_seconds -= 31536000; // simplified processing no leap year considered
}
// calculate month
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
for m in 0..12 {
if total_seconds >= days_in_month[m] * seconds_per_day {
month += 1;
total_seconds -= days_in_month[m] * seconds_per_day;
} else {
break;
}
}
// calculate the number of days
day += total_seconds / seconds_per_day;
total_seconds %= seconds_per_day;
// calculate hours
hour += total_seconds / seconds_per_hour;
total_seconds %= seconds_per_hour;
// calculate minutes
minute += total_seconds / seconds_per_minute;
total_seconds %= seconds_per_minute;
// calculate the number of seconds
second += total_seconds;
(year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32)
}