mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
feat(event-notifier): improve environment variable handling
- Fix deserialization error when parsing config from environment variables - Add proper array format support for adapters configuration - Update environment variable examples with correct format - Improve documentation for configuration loading - Implement helper functions for environment variable validation This change fixes the "invalid type: map, expected a sequence" error by ensuring proper formatting of array-type fields in environment variables.
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
# ===== 全局配置 =====
|
||||
NOTIFIER__STORE_PATH=/var/log/event-notification
|
||||
NOTIFIER__CHANNEL_CAPACITY=5000
|
||||
|
||||
# ===== 适配器配置(数组格式) =====
|
||||
# Webhook 适配器(索引 0)
|
||||
NOTIFIER__ADAPTERS_0__type=Webhook
|
||||
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook
|
||||
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
|
||||
NOTIFIER__ADAPTERS_0__max_retries=3
|
||||
NOTIFIER__ADAPTERS_0__timeout=50
|
||||
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
|
||||
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
|
||||
|
||||
# Kafka 适配器(索引 1)
|
||||
NOTIFIER__ADAPTERS_1__type=Kafka
|
||||
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
|
||||
NOTIFIER__ADAPTERS_1__topic=notifications
|
||||
NOTIFIER__ADAPTERS_1__max_retries=3
|
||||
NOTIFIER__ADAPTERS_1__timeout=60
|
||||
|
||||
# MQTT 适配器(索引 2)
|
||||
NOTIFIER__ADAPTERS_2__type=Mqtt
|
||||
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
|
||||
NOTIFIER__ADAPTERS_2__port=1883
|
||||
NOTIFIER__ADAPTERS_2__client_id=event-notifier
|
||||
NOTIFIER__ADAPTERS_2__topic=events
|
||||
NOTIFIER__ADAPTERS_2__max_retries=3
|
||||
@@ -1,27 +1,28 @@
|
||||
# basic configuration
|
||||
EVENT_NOTIF_STORE_PATH=/var/log/event-notification
|
||||
EVENT_NOTIF_CHANNEL_CAPACITY=5000
|
||||
# ===== global configuration =====
|
||||
NOTIFIER__STORE_PATH=/var/log/event-notification
|
||||
NOTIFIER__CHANNEL_CAPACITY=5000
|
||||
|
||||
# webhook adapter configuration
|
||||
EVENT_NOTIF_ADAPTERS__0__TYPE=Webhook
|
||||
EVENT_NOTIF_ADAPTERS__0__ENDPOINT=https://api.example.com/webhook
|
||||
EVENT_NOTIF_ADAPTERS__0__AUTH_TOKEN=your-secret-token
|
||||
EVENT_NOTIF_ADAPTERS__0__MAX_RETRIES=3
|
||||
EVENT_NOTIF_ADAPTERS__0__TIMEOUT=5000
|
||||
# ===== adapter configuration array format =====
|
||||
# webhook adapter index 0
|
||||
NOTIFIER__ADAPTERS_0__type=Webhook
|
||||
NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3000/webhook
|
||||
NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
|
||||
NOTIFIER__ADAPTERS_0__max_retries=3
|
||||
NOTIFIER__ADAPTERS_0__timeout=50
|
||||
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
|
||||
NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
|
||||
|
||||
# kafka adapter configuration
|
||||
EVENT_NOTIF_ADAPTERS__1__TYPE=Kafka
|
||||
EVENT_NOTIF_ADAPTERS__1__BROKERS=localhost:9092
|
||||
EVENT_NOTIF_ADAPTERS__1__TOPIC=notifications
|
||||
EVENT_NOTIF_ADAPTERS__1__MAX_RETRIES=3
|
||||
EVENT_NOTIF_ADAPTERS__1__TIMEOUT=5000
|
||||
# kafka adapter index 1
|
||||
NOTIFIER__ADAPTERS_1__type=Kafka
|
||||
NOTIFIER__ADAPTERS_1__brokers=localhost:9092
|
||||
NOTIFIER__ADAPTERS_1__topic=notifications
|
||||
NOTIFIER__ADAPTERS_1__max_retries=3
|
||||
NOTIFIER__ADAPTERS_1__timeout=60
|
||||
|
||||
# mqtt adapter configuration
|
||||
EVENT_NOTIF_ADAPTERS__2__TYPE=Mqtt
|
||||
EVENT_NOTIF_ADAPTERS__2__BROKER=mqtt.example.com
|
||||
EVENT_NOTIF_ADAPTERS__2__PORT=1883
|
||||
EVENT_NOTIF_ADAPTERS__2__CLIENT_ID=event-notifier
|
||||
EVENT_NOTIF_ADAPTERS__2__TOPIC=events
|
||||
EVENT_NOTIF_ADAPTERS__2__MAX_RETRIES=3
|
||||
|
||||
EVENT_NOTIF_HTTP__PORT=8080
|
||||
# mqtt adapter index 2
|
||||
NOTIFIER__ADAPTERS_2__type=Mqtt
|
||||
NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
|
||||
NOTIFIER__ADAPTERS_2__port=1883
|
||||
NOTIFIER__ADAPTERS_2__client_id=event-notifier
|
||||
NOTIFIER__ADAPTERS_2__topic=events
|
||||
NOTIFIER__ADAPTERS_2__max_retries=3
|
||||
@@ -1,20 +1,24 @@
|
||||
# config.toml
|
||||
store_path = "/var/log/event-notification"
|
||||
store_path = "/var/log/event-notifier"
|
||||
channel_capacity = 5000
|
||||
|
||||
[[adapters]]
|
||||
type = "Webhook"
|
||||
endpoint = "https://api.example.com/webhook"
|
||||
endpoint = "http://127.0.0.1:3000/webhook"
|
||||
auth_token = "your-auth-token"
|
||||
max_retries = 3
|
||||
timeout = 5000
|
||||
timeout = 50
|
||||
|
||||
[adapters.custom_headers]
|
||||
custom_server = "value_server"
|
||||
custom_client = "value_client"
|
||||
|
||||
[[adapters]]
|
||||
type = "Kafka"
|
||||
brokers = "localhost:9092"
|
||||
topic = "notifications"
|
||||
max_retries = 3
|
||||
timeout = 5000
|
||||
timeout = 60
|
||||
|
||||
[[adapters]]
|
||||
type = "Mqtt"
|
||||
@@ -22,7 +26,4 @@ broker = "mqtt.example.com"
|
||||
port = 1883
|
||||
client_id = "event-notifier"
|
||||
topic = "events"
|
||||
max_retries = 3
|
||||
|
||||
[http]
|
||||
port = 8080
|
||||
max_retries = 3
|
||||
@@ -1,6 +1,5 @@
|
||||
use rustfs_event_notifier::{
|
||||
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotificationConfig, Object, Source,
|
||||
WebhookConfig,
|
||||
AdapterConfig, Bucket, Error as NotifierError, Event, Identity, Metadata, Name, NotifierConfig, Object, Source, WebhookConfig,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use tokio::signal;
|
||||
@@ -8,7 +7,7 @@ use tracing::Level;
|
||||
use tracing_subscriber::FmtSubscriber;
|
||||
|
||||
async fn setup_notification_system() -> Result<(), NotifierError> {
|
||||
let config = NotificationConfig {
|
||||
let config = NotifierConfig {
|
||||
store_path: "./deploy/logs/event_store".into(),
|
||||
channel_capacity: 100,
|
||||
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
|
||||
@@ -40,11 +39,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// tracing_subscriber::fmt::init();
|
||||
|
||||
let subscriber = FmtSubscriber::builder()
|
||||
.with_max_level(Level::DEBUG) // set to debug or lower level
|
||||
.with_target(false) // simplify output
|
||||
.with_max_level(Level::DEBUG) // set to debug or lower level
|
||||
.with_target(false) // simplify output
|
||||
.finish();
|
||||
tracing::subscriber::set_global_default(subscriber)
|
||||
.expect("failed to set up log subscriber");
|
||||
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
|
||||
|
||||
// set up notification system
|
||||
if let Err(e) = setup_notification_system().await {
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
use rustfs_event_notifier::create_adapters;
|
||||
use rustfs_event_notifier::NotificationSystem;
|
||||
use rustfs_event_notifier::{AdapterConfig, NotificationConfig, WebhookConfig};
|
||||
use rustfs_event_notifier::NotifierSystem;
|
||||
use rustfs_event_notifier::{AdapterConfig, NotifierConfig, WebhookConfig};
|
||||
use rustfs_event_notifier::{Bucket, Event, Identity, Metadata, Name, Object, Source};
|
||||
use std::collections::HashMap;
|
||||
use std::error;
|
||||
use std::sync::Arc;
|
||||
use tokio::signal;
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::FmtSubscriber;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn error::Error>> {
|
||||
tracing_subscriber::fmt::init();
|
||||
let subscriber = FmtSubscriber::builder()
|
||||
.with_max_level(Level::DEBUG) // set to debug or lower level
|
||||
.with_target(false) // simplify output
|
||||
.finish();
|
||||
tracing::subscriber::set_global_default(subscriber).expect("failed to set up log subscriber");
|
||||
|
||||
let config = NotificationConfig {
|
||||
let config = NotifierConfig {
|
||||
store_path: "./events".to_string(),
|
||||
channel_capacity: 100,
|
||||
adapters: vec![AdapterConfig::Webhook(WebhookConfig {
|
||||
endpoint: "http://localhost:8080/webhook".to_string(),
|
||||
endpoint: "http://127.0.0.1:3000/webhook".to_string(),
|
||||
auth_token: Some("secret-token".to_string()),
|
||||
custom_headers: Some(HashMap::from([("X-Custom".to_string(), "value".to_string())])),
|
||||
max_retries: 3,
|
||||
@@ -23,16 +29,12 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
|
||||
})],
|
||||
};
|
||||
|
||||
// loading configuration from specific env files
|
||||
let _config = NotificationConfig::from_env_file(".env.example")?;
|
||||
// load_config
|
||||
// loading configuration from environment variables
|
||||
let _config = NotifierConfig::load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
|
||||
tracing::info!("load_config config: {:?} \n", _config);
|
||||
|
||||
// loading from a specific file
|
||||
let _config = NotificationConfig::from_file("event.toml")?;
|
||||
|
||||
// Automatically load from multiple sources (Priority: Environment Variables > YAML > TOML)
|
||||
let _config = NotificationConfig::load()?;
|
||||
|
||||
let system = Arc::new(tokio::sync::Mutex::new(NotificationSystem::new(config.clone()).await?));
|
||||
let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
|
||||
let adapters = create_adapters(&config.adapters)?;
|
||||
|
||||
// create an s3 metadata object
|
||||
@@ -90,9 +92,15 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
|
||||
|
||||
signal::ctrl_c().await?;
|
||||
tracing::info!("Received shutdown signal");
|
||||
{
|
||||
let system = system.lock().await;
|
||||
system.shutdown();
|
||||
let result = {
|
||||
let mut system = system.lock().await;
|
||||
system.shutdown().await
|
||||
};
|
||||
|
||||
if let Err(e) = result {
|
||||
tracing::error!("Failed to shut down the notification system: {}", e);
|
||||
} else {
|
||||
tracing::info!("Notification system shut down successfully");
|
||||
}
|
||||
|
||||
system_handle.await??;
|
||||
|
||||
@@ -12,6 +12,71 @@ async fn main() {
|
||||
}
|
||||
|
||||
async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
|
||||
println!("收到 webhook 请求 time: {},内容:{}", SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs().to_string(), serde_json::to_string_pretty(&payload).unwrap());
|
||||
let start = SystemTime::now();
|
||||
let since_the_epoch = start.duration_since(UNIX_EPOCH).expect("Time went backwards");
|
||||
|
||||
// get the number of seconds since the unix era
|
||||
let seconds = since_the_epoch.as_secs();
|
||||
|
||||
// Manually calculate year, month, day, hour, minute, and second
|
||||
let (year, month, day, hour, minute, second) = convert_seconds_to_date(seconds);
|
||||
|
||||
// output result
|
||||
println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
|
||||
println!(
|
||||
"received a webhook request time:{} content:\n {}",
|
||||
seconds.to_string(),
|
||||
serde_json::to_string_pretty(&payload).unwrap()
|
||||
);
|
||||
StatusCode::OK
|
||||
}
|
||||
|
||||
fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
|
||||
// assume that the time zone is utc
|
||||
let seconds_per_minute = 60;
|
||||
let seconds_per_hour = 3600;
|
||||
let seconds_per_day = 86400;
|
||||
|
||||
// Calculate the year, month, day, hour, minute, and second corresponding to the number of seconds
|
||||
let mut total_seconds = seconds;
|
||||
let mut year = 1970;
|
||||
let mut month = 1;
|
||||
let mut day = 1;
|
||||
let mut hour = 0;
|
||||
let mut minute = 0;
|
||||
let mut second = 0;
|
||||
|
||||
// calculate year
|
||||
while total_seconds >= 31536000 {
|
||||
year += 1;
|
||||
total_seconds -= 31536000; // simplified processing no leap year considered
|
||||
}
|
||||
|
||||
// calculate month
|
||||
let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||
for m in 0..12 {
|
||||
if total_seconds >= days_in_month[m] * seconds_per_day {
|
||||
month += 1;
|
||||
total_seconds -= days_in_month[m] * seconds_per_day;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// calculate the number of days
|
||||
day += total_seconds / seconds_per_day;
|
||||
total_seconds %= seconds_per_day;
|
||||
|
||||
// calculate hours
|
||||
hour += total_seconds / seconds_per_hour;
|
||||
total_seconds %= seconds_per_hour;
|
||||
|
||||
// calculate minutes
|
||||
minute += total_seconds / seconds_per_minute;
|
||||
total_seconds %= seconds_per_minute;
|
||||
|
||||
// calculate the number of seconds
|
||||
second += total_seconds;
|
||||
|
||||
(year as u32, month as u32, day as u32, hour as u32, minute as u32, second as u32)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user