Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/observability-metrics

# Conflicts:
#	Cargo.toml
#	crates/obs/examples/config.toml
#	crates/obs/src/telemetry.rs

crates/config/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "rustfs-config"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+config = { workspace = true }
+const-str = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+
+[lints]
+workspace = true

crates/config/src/config.rs

@@ -0,0 +1,23 @@
+use crate::event::config::NotifierConfig;
+use crate::ObservabilityConfig;
+
+/// RustFs configuration
+pub struct RustFsConfig {
+    pub observability: ObservabilityConfig,
+    pub event: NotifierConfig,
+}
+
+impl RustFsConfig {
+    pub fn new() -> Self {
+        Self {
+            observability: ObservabilityConfig::new(),
+            event: NotifierConfig::new(),
+        }
+    }
+}
+
+impl Default for RustFsConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/constants/app.rs

@@ -0,0 +1,91 @@
+use const_str::concat;
+
+/// Application name
+/// Default value: RustFs
+/// Environment variable: RUSTFS_APP_NAME
+pub const APP_NAME: &str = "RustFs";
+/// Application version
+/// Default value: 1.0.0
+/// Environment variable: RUSTFS_VERSION
+pub const VERSION: &str = "0.0.1";
+
+/// Default configuration logger level
+/// Default value: info
+/// Environment variable: RUSTFS_LOG_LEVEL
+pub const DEFAULT_LOG_LEVEL: &str = "info";
+
+/// Default configuration use stdout
+/// Default value: true
+pub const USE_STDOUT: bool = true;
+
+/// Default configuration sample ratio
+/// Default value: 1.0
+pub const SAMPLE_RATIO: f64 = 1.0;
+/// Default configuration meter interval
+/// Default value: 30
+pub const METER_INTERVAL: u64 = 30;
+
+/// Default configuration service version
+/// Default value: 0.0.1
+pub const SERVICE_VERSION: &str = "0.0.1";
+
+/// Default configuration environment
+/// Default value: production
+pub const ENVIRONMENT: &str = "production";
+
+/// maximum number of connections
+/// This is the maximum number of connections that the server will accept.
+/// This is used to limit the number of connections to the server.
+pub const MAX_CONNECTIONS: usize = 100;
+/// timeout for connections
+/// This is the timeout for connections to the server.
+/// This is used to limit the time that a connection can be open.
+pub const DEFAULT_TIMEOUT_MS: u64 = 3000;
+
+/// Default Access Key
+/// Default value: rustfsadmin
+/// Environment variable: RUSTFS_ACCESS_KEY
+/// Command line argument: --access-key
+/// Example: RUSTFS_ACCESS_KEY=rustfsadmin
+/// Example: --access-key rustfsadmin
+pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
+/// Default Secret Key
+/// Default value: rustfsadmin
+/// Environment variable: RUSTFS_SECRET_KEY
+/// Command line argument: --secret-key
+/// Example: RUSTFS_SECRET_KEY=rustfsadmin
+/// Example: --secret-key rustfsadmin
+pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin";
+/// Default configuration file for observability
+/// Default value: config/obs.toml
+/// Environment variable: RUSTFS_OBS_CONFIG
+/// Command line argument: --obs-config
+/// Example: RUSTFS_OBS_CONFIG=config/obs.toml
+/// Example: --obs-config config/obs.toml
+/// Example: --obs-config /etc/rustfs/obs.toml
+pub const DEFAULT_OBS_CONFIG: &str = "./deploy/config/obs.toml";
+
+/// Default TLS key for rustfs
+/// This is the default key for TLS.
+pub const RUSTFS_TLS_KEY: &str = "rustfs_key.pem";
+
+/// Default TLS cert for rustfs
+/// This is the default cert for TLS.
+pub const RUSTFS_TLS_CERT: &str = "rustfs_cert.pem";
+
+/// Default port for rustfs
+/// This is the default port for rustfs.
+/// This is used to bind the server to a specific port.
+pub const DEFAULT_PORT: u16 = 9000;
+
+/// Default address for rustfs
+/// This is the default address for rustfs.
+pub const DEFAULT_ADDRESS: &str = concat!(":", DEFAULT_PORT);
+
+/// Default port for rustfs console
+/// This is the default port for rustfs console.
+pub const DEFAULT_CONSOLE_PORT: u16 = 9002;
+
+/// Default address for rustfs console
+/// This is the default address for rustfs console.
+pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT);

crates/config/src/constants/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod app;

crates/config/src/event/adapters.rs

@@ -0,0 +1,27 @@
+use crate::event::kafka::KafkaAdapter;
+use crate::event::mqtt::MqttAdapter;
+use crate::event::webhook::WebhookAdapter;
+use serde::{Deserialize, Serialize};
+
+/// Configuration for the notification system.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum AdapterConfig {
+    Webhook(WebhookAdapter),
+    Kafka(KafkaAdapter),
+    Mqtt(MqttAdapter),
+}
+
+impl AdapterConfig {
+    /// create a new configuration with default values
+    pub fn new() -> Self {
+        Self::Webhook(WebhookAdapter::new())
+    }
+}
+
+impl Default for AdapterConfig {
+    /// create a new configuration with default values
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/event/config.rs

@@ -0,0 +1,43 @@
+use crate::event::adapters::AdapterConfig;
+use serde::{Deserialize, Serialize};
+use std::env;
+
+#[allow(dead_code)]
+const DEFAULT_CONFIG_FILE: &str = "event";
+
+/// Configuration for the notification system.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct NotifierConfig {
+    #[serde(default = "default_store_path")]
+    pub store_path: String,
+    #[serde(default = "default_channel_capacity")]
+    pub channel_capacity: usize,
+    pub adapters: Vec<AdapterConfig>,
+}
+
+impl Default for NotifierConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl NotifierConfig {
+    /// create a new configuration with default values
+    pub fn new() -> Self {
+        Self {
+            store_path: default_store_path(),
+            channel_capacity: default_channel_capacity(),
+            adapters: vec![AdapterConfig::new()],
+        }
+    }
+}
+
+/// Provide temporary directories as default storage paths
+fn default_store_path() -> String {
+    env::temp_dir().join("event-notification").to_string_lossy().to_string()
+}
+
+/// Provides the recommended default channel capacity for high concurrency systems
+fn default_channel_capacity() -> usize {
+    10000 // Reasonable default values for high concurrency systems
+}

crates/config/src/event/kafka.rs

@@ -0,0 +1,29 @@
+use serde::{Deserialize, Serialize};
+
+/// Configuration for the Kafka adapter.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KafkaAdapter {
+    pub brokers: String,
+    pub topic: String,
+    pub max_retries: u32,
+    pub timeout: u64,
+}
+
+impl KafkaAdapter {
+    /// create a new configuration with default values
+    pub fn new() -> Self {
+        Self {
+            brokers: "localhost:9092".to_string(),
+            topic: "kafka_topic".to_string(),
+            max_retries: 3,
+            timeout: 1000,
+        }
+    }
+}
+
+impl Default for KafkaAdapter {
+    /// create a new configuration with default values
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/event/mod.rs

@@ -0,0 +1,5 @@
+pub(crate) mod adapters;
+pub(crate) mod config;
+pub(crate) mod kafka;
+pub(crate) mod mqtt;
+pub(crate) mod webhook;

crates/config/src/event/mqtt.rs

@@ -0,0 +1,31 @@
+use serde::{Deserialize, Serialize};
+
+/// Configuration for the MQTT adapter.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MqttAdapter {
+    pub broker: String,
+    pub port: u16,
+    pub client_id: String,
+    pub topic: String,
+    pub max_retries: u32,
+}
+
+impl MqttAdapter {
+    /// create a new configuration with default values
+    pub fn new() -> Self {
+        Self {
+            broker: "localhost".to_string(),
+            port: 1883,
+            client_id: "mqtt_client".to_string(),
+            topic: "mqtt_topic".to_string(),
+            max_retries: 3,
+        }
+    }
+}
+
+impl Default for MqttAdapter {
+    /// create a new configuration with default values
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/event/webhook.rs

@@ -0,0 +1,51 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Configuration for the notification system.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WebhookAdapter {
+    pub endpoint: String,
+    pub auth_token: Option<String>,
+    pub custom_headers: Option<HashMap<String, String>>,
+    pub max_retries: u32,
+    pub timeout: u64,
+}
+
+impl WebhookAdapter {
+    /// verify that the configuration is valid
+    pub fn validate(&self) -> Result<(), String> {
+        // verify that endpoint cannot be empty
+        if self.endpoint.trim().is_empty() {
+            return Err("Webhook endpoint cannot be empty".to_string());
+        }
+
+        // verification timeout must be reasonable
+        if self.timeout == 0 {
+            return Err("Webhook timeout must be greater than 0".to_string());
+        }
+
+        // Verify that the maximum number of retry is reasonable
+        if self.max_retries > 10 {
+            return Err("Maximum retry count cannot exceed 10".to_string());
+        }
+
+        Ok(())
+    }
+
+    /// Get the default configuration
+    pub fn new() -> Self {
+        Self {
+            endpoint: "".to_string(),
+            auth_token: None,
+            custom_headers: Some(HashMap::new()),
+            max_retries: 3,
+            timeout: 1000,
+        }
+    }
+}
+
+impl Default for WebhookAdapter {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/lib.rs

@@ -0,0 +1,11 @@
+use crate::observability::config::ObservabilityConfig;
+
+mod config;
+mod constants;
+mod event;
+mod observability;
+
+pub use config::RustFsConfig;
+pub use constants::app::*;
+
+pub use event::config::NotifierConfig;

crates/config/src/observability/config.rs

@@ -0,0 +1,28 @@
+use crate::observability::logger::LoggerConfig;
+use crate::observability::otel::OtelConfig;
+use crate::observability::sink::SinkConfig;
+use serde::{Deserialize, Serialize};
+
+/// Observability configuration
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct ObservabilityConfig {
+    pub otel: OtelConfig,
+    pub sinks: Vec<SinkConfig>,
+    pub logger: Option<LoggerConfig>,
+}
+
+impl ObservabilityConfig {
+    pub fn new() -> Self {
+        Self {
+            otel: OtelConfig::new(),
+            sinks: vec![SinkConfig::new()],
+            logger: Some(LoggerConfig::new()),
+        }
+    }
+}
+
+impl Default for ObservabilityConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/observability/file.rs

@@ -0,0 +1,59 @@
+use serde::{Deserialize, Serialize};
+use std::env;
+
+/// File sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FileSink {
+    pub path: String,
+    #[serde(default = "default_buffer_size")]
+    pub buffer_size: Option<usize>,
+    #[serde(default = "default_flush_interval_ms")]
+    pub flush_interval_ms: Option<u64>,
+    #[serde(default = "default_flush_threshold")]
+    pub flush_threshold: Option<usize>,
+}
+
+impl FileSink {
+    pub fn new() -> Self {
+        Self {
+            path: env::var("RUSTFS_SINKS_FILE_PATH")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(default_path),
+            buffer_size: default_buffer_size(),
+            flush_interval_ms: default_flush_interval_ms(),
+            flush_threshold: default_flush_threshold(),
+        }
+    }
+}
+
+impl Default for FileSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_buffer_size() -> Option<usize> {
+    Some(8192)
+}
+fn default_flush_interval_ms() -> Option<u64> {
+    Some(1000)
+}
+fn default_flush_threshold() -> Option<usize> {
+    Some(100)
+}
+
+fn default_path() -> String {
+    let temp_dir = env::temp_dir().join("rustfs");
+
+    if let Err(e) = std::fs::create_dir_all(&temp_dir) {
+        eprintln!("Failed to create log directory: {}", e);
+        return "rustfs/rustfs.log".to_string();
+    }
+
+    temp_dir
+        .join("rustfs.log")
+        .to_str()
+        .unwrap_or("rustfs/rustfs.log")
+        .to_string()
+}

crates/config/src/observability/kafka.rs

@@ -0,0 +1,36 @@
+use serde::{Deserialize, Serialize};
+
+/// Kafka sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KafkaSink {
+    pub brokers: String,
+    pub topic: String,
+    #[serde(default = "default_batch_size")]
+    pub batch_size: Option<usize>,
+    #[serde(default = "default_batch_timeout_ms")]
+    pub batch_timeout_ms: Option<u64>,
+}
+
+impl KafkaSink {
+    pub fn new() -> Self {
+        Self {
+            brokers: "localhost:9092".to_string(),
+            topic: "rustfs".to_string(),
+            batch_size: default_batch_size(),
+            batch_timeout_ms: default_batch_timeout_ms(),
+        }
+    }
+}
+
+impl Default for KafkaSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_batch_size() -> Option<usize> {
+    Some(100)
+}
+fn default_batch_timeout_ms() -> Option<u64> {
+    Some(1000)
+}

crates/config/src/observability/logger.rs

@@ -0,0 +1,21 @@
+use serde::{Deserialize, Serialize};
+
+/// Logger configuration
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct LoggerConfig {
+    pub queue_capacity: Option<usize>,
+}
+
+impl LoggerConfig {
+    pub fn new() -> Self {
+        Self {
+            queue_capacity: Some(10000),
+        }
+    }
+}
+
+impl Default for LoggerConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/observability/mod.rs

@@ -0,0 +1,7 @@
+pub(crate) mod config;
+pub(crate) mod file;
+pub(crate) mod kafka;
+pub(crate) mod logger;
+pub(crate) mod otel;
+pub(crate) mod sink;
+pub(crate) mod webhook;

crates/config/src/observability/otel.rs

@@ -0,0 +1,69 @@
+use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
+use crate::{APP_NAME, DEFAULT_LOG_LEVEL};
+use serde::{Deserialize, Serialize};
+use std::env;
+
+/// OpenTelemetry configuration
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct OtelConfig {
+    pub endpoint: String,                    // Endpoint for metric collection
+    pub use_stdout: Option<bool>,            // Output to stdout
+    pub sample_ratio: Option<f64>,           // Trace sampling ratio
+    pub meter_interval: Option<u64>,         // Metric collection interval
+    pub service_name: Option<String>,        // Service name
+    pub service_version: Option<String>,     // Service version
+    pub environment: Option<String>,         // Environment
+    pub logger_level: Option<String>,        // Logger level
+    pub local_logging_enabled: Option<bool>, // Local logging enabled
+}
+
+impl OtelConfig {
+    pub fn new() -> Self {
+        extract_otel_config_from_env()
+    }
+}
+
+impl Default for OtelConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+// Helper function: Extract observable configuration from environment variables
+fn extract_otel_config_from_env() -> OtelConfig {
+    OtelConfig {
+        endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
+        use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(USE_STDOUT)),
+        sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(SAMPLE_RATIO)),
+        meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(METER_INTERVAL)),
+        service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(APP_NAME.to_string())),
+        service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(SERVICE_VERSION.to_string())),
+        environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(ENVIRONMENT.to_string())),
+        logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(DEFAULT_LOG_LEVEL.to_string())),
+        local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(false)),
+    }
+}

crates/config/src/observability/sink.rs

@@ -0,0 +1,25 @@
+use crate::observability::file::FileSink;
+use crate::observability::kafka::KafkaSink;
+use crate::observability::webhook::WebhookSink;
+use serde::{Deserialize, Serialize};
+
+/// Sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum SinkConfig {
+    Kafka(KafkaSink),
+    Webhook(WebhookSink),
+    File(FileSink),
+}
+
+impl SinkConfig {
+    pub fn new() -> Self {
+        Self::File(FileSink::new())
+    }
+}
+
+impl Default for SinkConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/config/src/observability/webhook.rs

@@ -0,0 +1,39 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Webhook sink configuration
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct WebhookSink {
+    pub endpoint: String,
+    pub auth_token: String,
+    pub headers: Option<HashMap<String, String>>,
+    #[serde(default = "default_max_retries")]
+    pub max_retries: Option<usize>,
+    #[serde(default = "default_retry_delay_ms")]
+    pub retry_delay_ms: Option<u64>,
+}
+
+impl WebhookSink {
+    pub fn new() -> Self {
+        Self {
+            endpoint: "".to_string(),
+            auth_token: "".to_string(),
+            headers: Some(HashMap::new()),
+            max_retries: default_max_retries(),
+            retry_delay_ms: default_retry_delay_ms(),
+        }
+    }
+}
+
+impl Default for WebhookSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_max_retries() -> Option<usize> {
+    Some(3)
+}
+fn default_retry_delay_ms() -> Option<u64> {
+    Some(100)
+}

crates/event-notifier/Cargo.toml

@@ -9,13 +9,12 @@ version.workspace = true
 [features]
 default = ["webhook"]
 webhook = ["dep:reqwest"]
-kafka = ["rdkafka"]
 mqtt = ["rumqttc"]
+kafka = ["dep:rdkafka"]
 
 [dependencies]
 async-trait = { workspace = true }
 config = { workspace = true }
-rdkafka = { workspace = true, features = ["tokio"], optional = true }
 reqwest = { workspace = true, optional = true }
 rumqttc = { workspace = true, optional = true }
 serde = { workspace = true }
@@ -29,12 +28,16 @@ tokio = { workspace = true, features = ["sync", "net", "macros", "signal", "rt-m
 tokio-util = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 
+# Only enable kafka features and related dependencies on Linux
+[target.'cfg(target_os = "linux")'.dependencies]
+rdkafka = { workspace = true, features = ["tokio"], optional = true }
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["test-util"] }
 tracing-subscriber = { workspace = true }
 http = { workspace = true }
 axum = { workspace = true }
+dotenvy = "0.15.7"
 
 [lints]
 workspace = true

crates/event-notifier/examples/.env-zh.example

@@ -1,28 +0,0 @@
-# ===== 全局配置 =====
-NOTIFIER__STORE_PATH=/var/log/event-notification
-NOTIFIER__CHANNEL_CAPACITY=5000
-
-# ===== 适配器配置（数组格式） =====
-# Webhook 适配器（索引 0）
-NOTIFIER__ADAPTERS_0__type=Webhook
-NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
-NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
-NOTIFIER__ADAPTERS_0__max_retries=3
-NOTIFIER__ADAPTERS_0__timeout=50
-NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
-NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
-
-# Kafka 适配器（索引 1）
-NOTIFIER__ADAPTERS_1__type=Kafka
-NOTIFIER__ADAPTERS_1__brokers=localhost:9092
-NOTIFIER__ADAPTERS_1__topic=notifications
-NOTIFIER__ADAPTERS_1__max_retries=3
-NOTIFIER__ADAPTERS_1__timeout=60
-
-# MQTT 适配器（索引 2）
-NOTIFIER__ADAPTERS_2__type=Mqtt
-NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
-NOTIFIER__ADAPTERS_2__port=1883
-NOTIFIER__ADAPTERS_2__client_id=event-notifier
-NOTIFIER__ADAPTERS_2__topic=events
-NOTIFIER__ADAPTERS_2__max_retries=3

crates/event-notifier/examples/.env.example

@@ -1,28 +1,28 @@
-# ===== global configuration =====
-NOTIFIER__STORE_PATH=/var/log/event-notification
-NOTIFIER__CHANNEL_CAPACITY=5000
-
-# ===== adapter configuration array format =====
-# webhook adapter index 0
-NOTIFIER__ADAPTERS_0__type=Webhook
-NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
-NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
-NOTIFIER__ADAPTERS_0__max_retries=3
-NOTIFIER__ADAPTERS_0__timeout=50
-NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
-NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
-
-# kafka adapter index 1
-NOTIFIER__ADAPTERS_1__type=Kafka
-NOTIFIER__ADAPTERS_1__brokers=localhost:9092
-NOTIFIER__ADAPTERS_1__topic=notifications
-NOTIFIER__ADAPTERS_1__max_retries=3
-NOTIFIER__ADAPTERS_1__timeout=60
-
-# mqtt adapter index 2
-NOTIFIER__ADAPTERS_2__type=Mqtt
-NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
-NOTIFIER__ADAPTERS_2__port=1883
-NOTIFIER__ADAPTERS_2__client_id=event-notifier
-NOTIFIER__ADAPTERS_2__topic=events
-NOTIFIER__ADAPTERS_2__max_retries=3
+## ===== global configuration =====
+#NOTIFIER__STORE_PATH=/var/log/event-notification
+#NOTIFIER__CHANNEL_CAPACITY=5000
+#
+## ===== adapter configuration array format =====
+## webhook adapter index 0
+#NOTIFIER__ADAPTERS_0__type=Webhook
+#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
+#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
+#NOTIFIER__ADAPTERS_0__max_retries=3
+#NOTIFIER__ADAPTERS_0__timeout=50
+#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=server-value
+#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=client-value
+#
+## kafka adapter index 1
+#NOTIFIER__ADAPTERS_1__type=Kafka
+#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
+#NOTIFIER__ADAPTERS_1__topic=notifications
+#NOTIFIER__ADAPTERS_1__max_retries=3
+#NOTIFIER__ADAPTERS_1__timeout=60
+#
+## mqtt adapter index 2
+#NOTIFIER__ADAPTERS_2__type=Mqtt
+#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
+#NOTIFIER__ADAPTERS_2__port=1883
+#NOTIFIER__ADAPTERS_2__client_id=event-notifier
+#NOTIFIER__ADAPTERS_2__topic=events
+#NOTIFIER__ADAPTERS_2__max_retries=3

crates/event-notifier/examples/.env.zh.example

@@ -0,0 +1,28 @@
+## ===== 全局配置 =====
+#NOTIFIER__STORE_PATH=/var/log/event-notification
+#NOTIFIER__CHANNEL_CAPACITY=5000
+#
+## ===== 适配器配置（数组格式） =====
+## Webhook 适配器（索引 0）
+#NOTIFIER__ADAPTERS_0__type=Webhook
+#NOTIFIER__ADAPTERS_0__endpoint=http://127.0.0.1:3020/webhook
+#NOTIFIER__ADAPTERS_0__auth_token=your-auth-token
+#NOTIFIER__ADAPTERS_0__max_retries=3
+#NOTIFIER__ADAPTERS_0__timeout=50
+#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_server=value
+#NOTIFIER__ADAPTERS_0__custom_headers__x_custom_client=value
+#
+## Kafka 适配器（索引 1）
+#NOTIFIER__ADAPTERS_1__type=Kafka
+#NOTIFIER__ADAPTERS_1__brokers=localhost:9092
+#NOTIFIER__ADAPTERS_1__topic=notifications
+#NOTIFIER__ADAPTERS_1__max_retries=3
+#NOTIFIER__ADAPTERS_1__timeout=60
+#
+## MQTT 适配器（索引 2）
+#NOTIFIER__ADAPTERS_2__type=Mqtt
+#NOTIFIER__ADAPTERS_2__broker=mqtt.example.com
+#NOTIFIER__ADAPTERS_2__port=1883
+#NOTIFIER__ADAPTERS_2__client_id=event-notifier
+#NOTIFIER__ADAPTERS_2__topic=events
+#NOTIFIER__ADAPTERS_2__max_retries=3

crates/event-notifier/examples/simple.rs

@@ -33,7 +33,9 @@ async fn main() -> Result<(), Box<dyn error::Error>> {
     // loading configuration from environment variables
     let _config = NotifierConfig::event_load_config(Some("./crates/event-notifier/examples/event.toml".to_string()));
     tracing::info!("event_load_config config: {:?} \n", _config);
-
+    dotenvy::dotenv()?;
+    let _config = NotifierConfig::event_load_config(None);
+    tracing::info!("event_load_config config: {:?} \n", _config);
     let system = Arc::new(tokio::sync::Mutex::new(NotifierSystem::new(config.clone()).await?));
     let adapters = create_adapters(&config.adapters)?;
 

crates/event-notifier/examples/webhook.rs

@@ -37,7 +37,7 @@ async fn receive_webhook(Json(payload): Json<Value>) -> StatusCode {
     println!("current time:{:04}-{:02}-{:02} {:02}:{:02}:{:02}", year, month, day, hour, minute, second);
     println!(
         "received a webhook request time:{} content:\n {}",
-        seconds.to_string(),
+        seconds,
         serde_json::to_string_pretty(&payload).unwrap()
     );
     StatusCode::OK
@@ -66,10 +66,10 @@ fn convert_seconds_to_date(seconds: u64) -> (u32, u32, u32, u32, u32, u32) {
 
     // calculate month
     let days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
-    for m in 0..12 {
-        if total_seconds >= days_in_month[m] * seconds_per_day {
+    for m in &days_in_month {
+        if total_seconds >= m * seconds_per_day {
             month += 1;
-            total_seconds -= days_in_month[m] * seconds_per_day;
+            total_seconds -= m * seconds_per_day;
         } else {
             break;
         }

crates/event-notifier/src/adapter/mod.rs

@@ -4,7 +4,7 @@ use crate::Event;
 use async_trait::async_trait;
 use std::sync::Arc;
 
-#[cfg(feature = "kafka")]
+#[cfg(all(feature = "kafka", target_os = "linux"))]
 pub(crate) mod kafka;
 #[cfg(feature = "mqtt")]
 pub(crate) mod mqtt;
@@ -31,7 +31,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result<Vec<Arc<dyn ChannelA
                 webhook_config.validate().map_err(Error::ConfigError)?;
                 adapters.push(Arc::new(webhook::WebhookAdapter::new(webhook_config.clone())));
             }
-            #[cfg(feature = "kafka")]
+            #[cfg(all(feature = "kafka", target_os = "linux"))]
             AdapterConfig::Kafka(kafka_config) => {
                 adapters.push(Arc::new(kafka::KafkaAdapter::new(kafka_config)?));
             }
@@ -43,7 +43,7 @@ pub fn create_adapters(configs: &[AdapterConfig]) -> Result<Vec<Arc<dyn ChannelA
             }
             #[cfg(not(feature = "webhook"))]
             AdapterConfig::Webhook(_) => return Err(Error::FeatureDisabled("webhook")),
-            #[cfg(not(feature = "kafka"))]
+            #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
             AdapterConfig::Kafka(_) => return Err(Error::FeatureDisabled("kafka")),
             #[cfg(not(feature = "mqtt"))]
             AdapterConfig::Mqtt(_) => return Err(Error::FeatureDisabled("mqtt")),

crates/event-notifier/src/bus.rs

@@ -7,11 +7,13 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::sync::mpsc;
 use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
+use tracing::instrument;
 
 /// Handles incoming events from the producer.
 ///
 /// This function is responsible for receiving events from the producer and sending them to the appropriate adapters.
 /// It also handles the shutdown process and saves any pending logs to the event store.
+#[instrument(skip_all)]
 pub async fn event_bus(
     mut rx: mpsc::Receiver<Event>,
     adapters: Vec<Arc<dyn ChannelAdapter>>,

crates/event-notifier/src/config.rs

@@ -1,4 +1,4 @@
-use config::{Config, Environment, File, FileFormat};
+use config::{Config, File, FileFormat};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::env;
@@ -138,15 +138,6 @@ impl NotifierConfig {
         let app_config = Config::builder()
             .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false))
             .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false))
-            .add_source(
-                Environment::default()
-                    .prefix("NOTIFIER")
-                    .prefix_separator("__")
-                    .separator("__")
-                    .list_separator("_")
-                    .with_list_parse_key("adapters")
-                    .try_parsing(true),
-            )
             .build()
             .unwrap_or_default();
         match app_config.try_deserialize::<NotifierConfig>() {
@@ -162,7 +153,7 @@ impl NotifierConfig {
     }
 }
 
-const DEFAULT_CONFIG_FILE: &str = "obs";
+const DEFAULT_CONFIG_FILE: &str = "event";
 
 /// Provide temporary directories as default storage paths
 fn default_store_path() -> String {

crates/event-notifier/src/error.rs

@@ -15,7 +15,7 @@ pub enum Error {
     Serde(#[from] serde_json::Error),
     #[error("HTTP error: {0}")]
     Http(#[from] reqwest::Error),
-    #[cfg(feature = "kafka")]
+    #[cfg(all(feature = "kafka", target_os = "linux"))]
     #[error("Kafka error: {0}")]
     Kafka(#[from] rdkafka::error::KafkaError),
     #[cfg(feature = "mqtt")]

crates/event-notifier/src/event.rs

@@ -15,6 +15,18 @@ pub struct Identity {
     pub principal_id: String,
 }
 
+impl Identity {
+    /// Create a new Identity instance
+    pub fn new(principal_id: String) -> Self {
+        Self { principal_id }
+    }
+
+    /// Set the principal ID
+    pub fn set_principal_id(&mut self, principal_id: String) {
+        self.principal_id = principal_id;
+    }
+}
+
 /// A struct representing the bucket information
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Bucket {
@@ -24,6 +36,32 @@ pub struct Bucket {
     pub arn: String,
 }
 
+impl Bucket {
+    /// Create a new Bucket instance
+    pub fn new(name: String, owner_identity: Identity, arn: String) -> Self {
+        Self {
+            name,
+            owner_identity,
+            arn,
+        }
+    }
+
+    /// Set the name of the bucket
+    pub fn set_name(&mut self, name: String) {
+        self.name = name;
+    }
+
+    /// Set the ARN of the bucket
+    pub fn set_arn(&mut self, arn: String) {
+        self.arn = arn;
+    }
+
+    /// Set the owner identity of the bucket
+    pub fn set_owner_identity(&mut self, owner_identity: Identity) {
+        self.owner_identity = owner_identity;
+    }
+}
+
 /// A struct representing the object information
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Object {
@@ -41,6 +79,64 @@ pub struct Object {
     pub sequencer: String,
 }
 
+impl Object {
+    /// Create a new Object instance
+    pub fn new(
+        key: String,
+        size: Option<i64>,
+        etag: Option<String>,
+        content_type: Option<String>,
+        user_metadata: Option<HashMap<String, String>>,
+        version_id: Option<String>,
+        sequencer: String,
+    ) -> Self {
+        Self {
+            key,
+            size,
+            etag,
+            content_type,
+            user_metadata,
+            version_id,
+            sequencer,
+        }
+    }
+
+    /// Set the key
+    pub fn set_key(&mut self, key: String) {
+        self.key = key;
+    }
+
+    /// Set the size
+    pub fn set_size(&mut self, size: Option<i64>) {
+        self.size = size;
+    }
+
+    /// Set the etag
+    pub fn set_etag(&mut self, etag: Option<String>) {
+        self.etag = etag;
+    }
+
+    /// Set the content type
+    pub fn set_content_type(&mut self, content_type: Option<String>) {
+        self.content_type = content_type;
+    }
+
+    /// Set the user metadata
+    pub fn set_user_metadata(&mut self, user_metadata: Option<HashMap<String, String>>) {
+        self.user_metadata = user_metadata;
+    }
+
+    /// Set the version ID
+    pub fn set_version_id(&mut self, version_id: Option<String>) {
+        self.version_id = version_id;
+    }
+
+    /// Set the sequencer
+    pub fn set_sequencer(&mut self, sequencer: String) {
+        self.sequencer = sequencer;
+    }
+}
+
 /// A struct representing the metadata of the event
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Metadata {
@@ -52,6 +148,57 @@ pub struct Metadata {
     pub object: Object,
 }
 
+impl Default for Metadata {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+impl Metadata {
+    /// Create a new Metadata instance with default values
+    pub fn new() -> Self {
+        Self {
+            schema_version: "1.0".to_string(),
+            configuration_id: "default".to_string(),
+            bucket: Bucket::new(
+                "default".to_string(),
+                Identity::new("default".to_string()),
+                "arn:aws:s3:::default".to_string(),
+            ),
+            object: Object::new("default".to_string(), None, None, None, None, None, "default".to_string()),
+        }
+    }
+
+    /// Create a new Metadata instance
+    pub fn create(schema_version: String, configuration_id: String, bucket: Bucket, object: Object) -> Self {
+        Self {
+            schema_version,
+            configuration_id,
+            bucket,
+            object,
+        }
+    }
+
+    /// Set the schema version
+    pub fn set_schema_version(&mut self, schema_version: String) {
+        self.schema_version = schema_version;
+    }
+
+    /// Set the configuration ID
+    pub fn set_configuration_id(&mut self, configuration_id: String) {
+        self.configuration_id = configuration_id;
+    }
+
+    /// Set the bucket
+    pub fn set_bucket(&mut self, bucket: Bucket) {
+        self.bucket = bucket;
+    }
+
+    /// Set the object
+    pub fn set_object(&mut self, object: Object) {
+        self.object = object;
+    }
+}
+
 /// A struct representing the source of the event
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Source {
@@ -61,6 +208,28 @@ pub struct Source {
     pub user_agent: String,
 }
 
+impl Source {
+    /// Create a new Source instance
+    pub fn new(host: String, port: String, user_agent: String) -> Self {
+        Self { host, port, user_agent }
+    }
+
+    /// Set the host
+    pub fn set_host(&mut self, host: String) {
+        self.host = host;
+    }
+
+    /// Set the port
+    pub fn set_port(&mut self, port: String) {
+        self.port = port;
+    }
+
+    /// Set the user agent
+    pub fn set_user_agent(&mut self, user_agent: String) {
+        self.user_agent = user_agent;
+    }
+}
+
 /// Builder for creating an Event.
 ///
 /// This struct is used to build an Event object with various parameters.

crates/event-notifier/src/global.rs

@@ -1,6 +1,7 @@
 use crate::{create_adapters, Error, Event, NotifierConfig, NotifierSystem};
 use std::sync::{atomic, Arc};
 use tokio::sync::{Mutex, OnceCell};
+use tracing::instrument;
 
 static GLOBAL_SYSTEM: OnceCell<Arc<Mutex<NotifierSystem>>> = OnceCell::const_new();
 static INITIALIZED: atomic::AtomicBool = atomic::AtomicBool::new(false);
@@ -113,6 +114,7 @@ pub fn is_ready() -> bool {
 /// - The system is not initialized.
 /// - The system is not ready.
 /// - Sending the event fails.
+#[instrument(fields(event))]
 pub async fn send_event(event: Event) -> Result<(), Error> {
     if !READY.load(atomic::Ordering::SeqCst) {
         return Err(Error::custom("Notification system not ready, please wait for initialization to complete"));
@@ -124,6 +126,7 @@ pub async fn send_event(event: Event) -> Result<(), Error> {
 }
 
 /// Shuts down the notification system.
+#[instrument]
 pub async fn shutdown() -> Result<(), Error> {
     if let Some(system) = GLOBAL_SYSTEM.get() {
         tracing::info!("Shutting down notification system start");
@@ -189,7 +192,7 @@ mod tests {
         let config = NotifierConfig::default();
         let _ = initialize(config.clone()).await; // first initialization
         let result = initialize(config).await; // second initialization
-        assert!(!result.is_ok(), "Initialization should succeed");
+        assert!(result.is_err(), "Initialization should succeed");
         assert!(result.is_err(), "Re-initialization should fail");
     }
 
@@ -211,7 +214,7 @@ mod tests {
             ..Default::default()
         };
         let result = initialize(config).await;
-        assert!(!result.is_err(), "Initialization with invalid config should fail");
+        assert!(result.is_ok(), "Initialization with invalid config should fail");
         assert!(is_initialized(), "System should not be marked as initialized after failure");
         assert!(is_ready(), "System should not be marked as ready after failure");
     }

crates/event-notifier/src/lib.rs

@@ -8,7 +8,7 @@ mod notifier;
 mod store;
 
 pub use adapter::create_adapters;
-#[cfg(feature = "kafka")]
+#[cfg(all(feature = "kafka", target_os = "linux"))]
 pub use adapter::kafka::KafkaAdapter;
 #[cfg(feature = "mqtt")]
 pub use adapter::mqtt::MqttAdapter;
@@ -16,7 +16,7 @@ pub use adapter::mqtt::MqttAdapter;
 pub use adapter::webhook::WebhookAdapter;
 pub use adapter::ChannelAdapter;
 pub use bus::event_bus;
-#[cfg(feature = "kafka")]
+#[cfg(all(feature = "kafka", target_os = "linux"))]
 pub use config::KafkaConfig;
 #[cfg(feature = "mqtt")]
 pub use config::MqttConfig;

crates/event-notifier/src/notifier.rs

@@ -2,6 +2,7 @@ use crate::{event_bus, ChannelAdapter, Error, Event, EventStore, NotifierConfig}
 use std::sync::Arc;
 use tokio::sync::mpsc;
 use tokio_util::sync::CancellationToken;
+use tracing::instrument;
 
 /// The `NotificationSystem` struct represents the notification system.
 /// It manages the event bus and the adapters.
@@ -18,6 +19,7 @@ pub struct NotifierSystem {
 
 impl NotifierSystem {
     /// Creates a new `NotificationSystem` instance.
+    #[instrument(skip(config))]
     pub async fn new(config: NotifierConfig) -> Result<Self, Error> {
         let (tx, rx) = mpsc::channel::<Event>(config.channel_capacity);
         let store = Arc::new(EventStore::new(&config.store_path).await?);
@@ -44,6 +46,7 @@ impl NotifierSystem {
 
     /// Starts the notification system.
     /// It initializes the event bus and the producer.
+    #[instrument(skip_all)]
     pub async fn start(&mut self, adapters: Vec<Arc<dyn ChannelAdapter>>) -> Result<(), Error> {
         if self.shutdown.is_cancelled() {
             let error = Error::custom("System is shutting down");
@@ -67,6 +70,7 @@ impl NotifierSystem {
 
     /// Sends an event to the notification system.
     /// This method is used to send events to the event bus.
+    #[instrument(skip(self))]
     pub async fn send_event(&self, event: Event) -> Result<(), Error> {
         self.log(tracing::Level::DEBUG, "send_event", &format!("Sending event: {:?}", event));
         if self.shutdown.is_cancelled() {
@@ -85,6 +89,7 @@ impl NotifierSystem {
 
     /// Shuts down the notification system.
     /// This method is used to cancel the event bus and producer tasks.
+    #[instrument(skip(self))]
     pub async fn shutdown(&mut self) -> Result<(), Error> {
         tracing::info!("Shutting down the notification system");
         self.shutdown.cancel();
@@ -112,10 +117,13 @@ impl NotifierSystem {
         self.shutdown.is_cancelled()
     }
 
-    fn handle_error(&self, context: &str, error: &Error) {
+    #[instrument(skip(self))]
+    pub fn handle_error(&self, context: &str, error: &Error) {
         self.log(tracing::Level::ERROR, context, &format!("{:?}", error));
         // TODO Can be extended to record to files or send to monitoring systems
     }
+
+    #[instrument(skip(self))]
     fn log(&self, level: tracing::Level, context: &str, message: &str) {
         match level {
             tracing::Level::ERROR => tracing::error!("[{}] {}", context, message),

crates/event-notifier/src/store.rs

@@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
 use tokio::fs::{create_dir_all, File, OpenOptions};
 use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter};
 use tokio::sync::RwLock;
+use tracing::instrument;
 
 /// `EventStore` is a struct that manages the storage of event logs.
 pub struct EventStore {
@@ -21,6 +22,7 @@ impl EventStore {
         })
     }
 
+    #[instrument(skip(self))]
     pub async fn save_logs(&self, logs: &[Log]) -> Result<(), Error> {
         let _guard = self.lock.write().await;
         let file_path = format!(

crates/obs/Cargo.toml

@@ -13,11 +13,11 @@ workspace = true
 default = ["file"]
 file = []
 gpu = ["dep:nvml-wrapper"]
-kafka = ["dep:rdkafka"]
 webhook = ["dep:reqwest"]
-full = ["file", "gpu", "kafka", "webhook"]
+kafka = ["dep:rdkafka"]
 
 [dependencies]
+rustfs-config = { workspace = true }
 async-trait = { workspace = true }
 chrono = { workspace = true }
 config = { workspace = true }
@@ -29,6 +29,7 @@ opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 opentelemetry-stdout = { workspace = true }
 opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
 opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
+rustfs-utils = { workspace = true, features = ["ip"] }
 serde = { workspace = true }
 smallvec = { workspace = true, features = ["serde"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
@@ -37,12 +38,14 @@ tracing-error = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
-rdkafka = { workspace = true, features = ["tokio"], optional = true }
 reqwest = { workspace = true, optional = true, default-features = false }
 serde_json = { workspace = true }
 sysinfo = { workspace = true }
 thiserror = { workspace = true }
 
+# Only enable kafka features and related dependencies on Linux
+[target.'cfg(target_os = "linux")'.dependencies]
+rdkafka = { workspace = true, features = ["tokio"], optional = true }
 
 
 [dev-dependencies]

crates/obs/examples/config.toml

@@ -9,26 +9,27 @@ environments = "develop"
 logger_level = "debug"
 local_logging_enabled = true # Default is false if not specified
 
-[sinks]
-[sinks.kafka]
-enabled = false
-bootstrap_servers = "localhost:9092"
-topic = "logs"
-batch_size = 100 # Default is 100 if not specified
-batch_timeout_ms = 1000 # Default is 1000ms if not specified
 
-[sinks.webhook]
-enabled = false
-endpoint = "http://localhost:8080/webhook"
-auth_token = ""
-batch_size = 100 # Default is 3 if not specified
-batch_timeout_ms = 1000 # Default is 100ms if not specified
+#[[sinks]]
+#type = "Kafka"
+#bootstrap_servers = "localhost:9092"
+#topic = "logs"
+#batch_size = 100 # Default is 100 if not specified
+#batch_timeout_ms = 100 # Default is 1000ms if not specified
+#
+#[[sinks]]
+#type = "Webhook"
+#endpoint = "http://localhost:8080/webhook"
+#auth_token = ""
+#batch_size = 100 # Default is 3 if not specified
+#batch_timeout_ms = 100 # Default is 100ms if not specified
 
-[sinks.file]
-enabled = true
-path = "deploy/logs/app.log"
-batch_size = 100
-batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
+[[sinks]]
+type = "File"
+path = "deploy/logs/rustfs.log"
+buffer_size = 102 # Default is 8192 bytes if not specified
+flush_interval_ms = 1000
+flush_threshold = 100
 
 [logger]
 queue_capacity = 10000

crates/obs/src/config.rs

@@ -1,6 +1,6 @@
-use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
-use config::{Config, Environment, File, FileFormat};
-use serde::Deserialize;
+use config::{Config, File, FileFormat};
+use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
+use serde::{Deserialize, Serialize};
 use std::env;
 
 /// OpenTelemetry Configuration
@@ -11,7 +11,7 @@ use std::env;
 /// Add use_stdout for output to stdout
 /// Add logger level for log level
 /// Add local_logging_enabled for local logging enabled
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct OtelConfig {
     pub endpoint: String,                    // Endpoint for metric collection
     pub use_stdout: Option<bool>,            // Output to stdout
@@ -24,7 +24,7 @@ pub struct OtelConfig {
     pub local_logging_enabled: Option<bool>, // Local logging enabled
 }
 
-// Helper function: Extract observable configuration from environment variables
+/// Helper function: Extract observable configuration from environment variables
 fn extract_otel_config_from_env() -> OtelConfig {
     OtelConfig {
         endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
@@ -43,7 +43,7 @@ fn extract_otel_config_from_env() -> OtelConfig {
         service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
             .ok()
             .and_then(|v| v.parse().ok())
-            .or(Some(SERVICE_NAME.to_string())),
+            .or(Some(APP_NAME.to_string())),
         service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
             .ok()
             .and_then(|v| v.parse().ok())
@@ -55,7 +55,7 @@ fn extract_otel_config_from_env() -> OtelConfig {
         logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
             .ok()
             .and_then(|v| v.parse().ok())
-            .or(Some(LOGGER_LEVEL.to_string())),
+            .or(Some(DEFAULT_LOG_LEVEL.to_string())),
         local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
             .ok()
             .and_then(|v| v.parse().ok())
@@ -63,36 +63,89 @@ fn extract_otel_config_from_env() -> OtelConfig {
     }
 }
 
-impl Default for OtelConfig {
-    fn default() -> Self {
+impl OtelConfig {
+    /// Create a new instance of OtelConfig with default values
+    ///
+    /// # Returns
+    /// A new instance of OtelConfig
+    pub fn new() -> Self {
         extract_otel_config_from_env()
     }
 }
 
+impl Default for OtelConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// Kafka Sink Configuration - Add batch parameters
-#[derive(Debug, Deserialize, Clone, Default)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct KafkaSinkConfig {
-    pub enabled: bool,
-    pub bootstrap_servers: String,
+    pub brokers: String,
     pub topic: String,
     pub batch_size: Option<usize>,     // Batch size, default 100
     pub batch_timeout_ms: Option<u64>, // Batch timeout time, default 1000ms
 }
 
+impl KafkaSinkConfig {
+    pub fn new() -> Self {
+        Self {
+            brokers: env::var("RUSTFS_SINKS_KAFKA_BROKERS")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(|| "localhost:9092".to_string()),
+            topic: env::var("RUSTFS_SINKS_KAFKA_TOPIC")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(|| "default_topic".to_string()),
+            batch_size: Some(100),
+            batch_timeout_ms: Some(1000),
+        }
+    }
+}
+
+impl Default for KafkaSinkConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// Webhook Sink Configuration - Add Retry Parameters
-#[derive(Debug, Deserialize, Clone, Default)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct WebhookSinkConfig {
-    pub enabled: bool,
     pub endpoint: String,
     pub auth_token: String,
     pub max_retries: Option<usize>,  // Maximum number of retry times, default 3
     pub retry_delay_ms: Option<u64>, // Retry the delay cardinality, default 100ms
 }
 
+impl WebhookSinkConfig {
+    pub fn new() -> Self {
+        Self {
+            endpoint: env::var("RUSTFS_SINKS_WEBHOOK_ENDPOINT")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(|| "http://localhost:8080".to_string()),
+            auth_token: env::var("RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(|| "default_token".to_string()),
+            max_retries: Some(3),
+            retry_delay_ms: Some(100),
+        }
+    }
+}
+
+impl Default for WebhookSinkConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// File Sink Configuration - Add buffering parameters
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct FileSinkConfig {
-    pub enabled: bool,
     pub path: String,
     pub buffer_size: Option<usize>,     // Write buffer size, default 8192
     pub flush_interval_ms: Option<u64>, // Refresh interval time, default 1000ms
@@ -107,19 +160,15 @@ impl FileSinkConfig {
             eprintln!("Failed to create log directory: {}", e);
             return "rustfs/rustfs.log".to_string();
         }
-
+        println!("Using log directory: {:?}", temp_dir);
         temp_dir
             .join("rustfs.log")
             .to_str()
             .unwrap_or("rustfs/rustfs.log")
             .to_string()
     }
-}
-
-impl Default for FileSinkConfig {
-    fn default() -> Self {
-        FileSinkConfig {
-            enabled: true,
+    pub fn new() -> Self {
+        Self {
             path: env::var("RUSTFS_SINKS_FILE_PATH")
                 .ok()
                 .filter(|s| !s.trim().is_empty())
@@ -131,38 +180,53 @@ impl Default for FileSinkConfig {
     }
 }
 
+impl Default for FileSinkConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// Sink configuration collection
-#[derive(Debug, Deserialize, Clone)]
-pub struct SinkConfig {
-    pub kafka: Option<KafkaSinkConfig>,
-    pub webhook: Option<WebhookSinkConfig>,
-    pub file: Option<FileSinkConfig>,
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum SinkConfig {
+    File(FileSinkConfig),
+    Kafka(KafkaSinkConfig),
+    Webhook(WebhookSinkConfig),
+}
+
+impl SinkConfig {
+    pub fn new() -> Self {
+        Self::File(FileSinkConfig::new())
+    }
 }
 
 impl Default for SinkConfig {
     fn default() -> Self {
-        SinkConfig {
-            kafka: None,
-            webhook: None,
-            file: Some(FileSinkConfig::default()),
-        }
+        Self::new()
     }
 }
 
 ///Logger Configuration
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct LoggerConfig {
     pub queue_capacity: Option<usize>,
 }
 
-impl Default for LoggerConfig {
-    fn default() -> Self {
-        LoggerConfig {
+impl LoggerConfig {
+    pub fn new() -> Self {
+        Self {
             queue_capacity: Some(10000),
         }
     }
 }
 
+impl Default for LoggerConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// Overall application configuration
 /// Add observability, sinks, and logger configuration
 ///
@@ -180,7 +244,7 @@ impl Default for LoggerConfig {
 #[derive(Debug, Deserialize, Clone)]
 pub struct AppConfig {
     pub observability: OtelConfig,
-    pub sinks: SinkConfig,
+    pub sinks: Vec<SinkConfig>,
     pub logger: Option<LoggerConfig>,
 }
 
@@ -192,7 +256,7 @@ impl AppConfig {
     pub fn new() -> Self {
         Self {
             observability: OtelConfig::default(),
-            sinks: SinkConfig::default(),
+            sinks: vec![SinkConfig::default()],
             logger: Some(LoggerConfig::default()),
         }
     }
@@ -258,14 +322,6 @@ pub fn load_config(config_dir: Option<String>) -> AppConfig {
     let app_config = Config::builder()
         .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false))
         .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false))
-        .add_source(
-            Environment::default()
-                .prefix("RUSTFS")
-                .prefix_separator("__")
-                .separator("__")
-                .with_list_parse_key("volumes")
-                .try_parsing(true),
-        )
         .build()
         .unwrap_or_default();
 

crates/obs/src/global.rs

@@ -3,14 +3,6 @@ use std::sync::{Arc, Mutex};
 use tokio::sync::{OnceCell, SetError};
 use tracing::{error, info};
 
-pub(crate) const USE_STDOUT: bool = true;
-pub(crate) const SERVICE_NAME: &str = "RustFS";
-pub(crate) const SAMPLE_RATIO: f64 = 1.0;
-pub(crate) const METER_INTERVAL: u64 = 60;
-pub(crate) const SERVICE_VERSION: &str = "0.1.0";
-pub(crate) const ENVIRONMENT: &str = "production";
-pub(crate) const LOGGER_LEVEL: &str = "info";
-
 /// Global guard for OpenTelemetry tracing
 static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 

crates/obs/src/lib.rs

@@ -32,20 +32,13 @@ mod config;
 mod entry;
 mod global;
 mod logger;
-mod sink;
+mod sinks;
 mod system;
 mod telemetry;
-mod utils;
 mod worker;
 
 use crate::logger::InitLogStatus;
 pub use config::load_config;
-#[cfg(feature = "file")]
-pub use config::FileSinkConfig;
-#[cfg(feature = "kafka")]
-pub use config::KafkaSinkConfig;
-#[cfg(feature = "webhook")]
-pub use config::WebhookSinkConfig;
 pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
 pub use entry::args::Args;
 pub use entry::audit::{ApiDetails, AuditLogEntry};
@@ -79,7 +72,7 @@ use tracing::{error, info};
 /// ```
 pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::OtelGuard) {
     let guard = init_telemetry(&config.observability);
-    let sinks = sink::create_sinks(&config).await;
+    let sinks = sinks::create_sinks(&config).await;
     let logger = init_global_logger(&config, sinks).await;
     let obs_config = config.observability.clone();
     tokio::spawn(async move {

crates/obs/src/logger.rs

@@ -1,6 +1,6 @@
-use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
-use crate::sink::Sink;
+use crate::sinks::Sink;
 use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
+use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
 use std::sync::Arc;
 use std::time::SystemTime;
 use tokio::sync::mpsc::{self, Receiver, Sender};
@@ -428,7 +428,7 @@ impl Default for InitLogStatus {
     fn default() -> Self {
         Self {
             timestamp: SystemTime::now(),
-            service_name: String::from(SERVICE_NAME),
+            service_name: String::from(APP_NAME),
             version: SERVICE_VERSION.to_string(),
             environment: ENVIRONMENT.to_string(),
         }
@@ -442,7 +442,7 @@ impl InitLogStatus {
         let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string());
         Self {
             timestamp: SystemTime::now(),
-            service_name: String::from(SERVICE_NAME),
+            service_name: String::from(APP_NAME),
             version,
             environment,
         }

crates/obs/src/sink.rs

@@ -1,497 +0,0 @@
-use crate::{AppConfig, LogRecord, UnifiedLogEntry};
-use async_trait::async_trait;
-use std::sync::Arc;
-use tokio::fs::OpenOptions;
-use tokio::io;
-use tokio::io::AsyncWriteExt;
-
-/// Sink Trait definition, asynchronously write logs
-#[async_trait]
-pub trait Sink: Send + Sync {
-    async fn write(&self, entry: &UnifiedLogEntry);
-}
-
-#[cfg(feature = "kafka")]
-/// Kafka Sink Implementation
-pub struct KafkaSink {
-    producer: rdkafka::producer::FutureProducer,
-    topic: String,
-    batch_size: usize,
-    batch_timeout_ms: u64,
-    entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
-    last_flush: Arc<std::sync::atomic::AtomicU64>,
-}
-
-#[cfg(feature = "kafka")]
-impl KafkaSink {
-    /// Create a new KafkaSink instance
-    pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
-        // Create Arc-wrapped values first
-        let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
-        let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
-            std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as u64,
-        ));
-        let sink = KafkaSink {
-            producer: producer.clone(),
-            topic: topic.clone(),
-            batch_size,
-            batch_timeout_ms,
-            entries: entries.clone(),
-            last_flush: last_flush.clone(),
-        };
-
-        // Start background flusher
-        tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
-
-        sink
-    }
-
-    /// Add a getter method to read the batch_timeout_ms field
-    #[allow(dead_code)]
-    pub fn batch_timeout(&self) -> u64 {
-        self.batch_timeout_ms
-    }
-
-    /// Add a method to dynamically adjust the timeout if needed
-    #[allow(dead_code)]
-    pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
-        self.batch_timeout_ms = new_timeout_ms;
-    }
-
-    async fn periodic_flush(
-        producer: rdkafka::producer::FutureProducer,
-        topic: String,
-        entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
-        last_flush: Arc<std::sync::atomic::AtomicU64>,
-        timeout_ms: u64,
-    ) {
-        loop {
-            tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
-
-            let now = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as u64;
-
-            let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
-
-            if now - last >= timeout_ms {
-                let mut batch = entries.lock().await;
-                if !batch.is_empty() {
-                    Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
-                    last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
-                }
-            }
-        }
-    }
-
-    async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
-        for entry in entries {
-            let payload = match serde_json::to_string(&entry) {
-                Ok(p) => p,
-                Err(e) => {
-                    eprintln!("Failed to serialize log entry: {}", e);
-                    continue;
-                }
-            };
-
-            let span_id = entry.get_timestamp().to_rfc3339();
-
-            let _ = producer
-                .send(
-                    rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
-                    std::time::Duration::from_secs(5),
-                )
-                .await;
-        }
-    }
-}
-
-#[cfg(feature = "kafka")]
-#[async_trait]
-impl Sink for KafkaSink {
-    async fn write(&self, entry: &UnifiedLogEntry) {
-        let mut batch = self.entries.lock().await;
-        batch.push(entry.clone());
-
-        let should_flush_by_size = batch.len() >= self.batch_size;
-        let should_flush_by_time = {
-            let now = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as u64;
-            let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
-            now - last >= self.batch_timeout_ms
-        };
-
-        if should_flush_by_size || should_flush_by_time {
-            // Existing flush logic
-            let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
-            let producer = self.producer.clone();
-            let topic = self.topic.clone();
-
-            self.last_flush.store(
-                std::time::SystemTime::now()
-                    .duration_since(std::time::UNIX_EPOCH)
-                    .unwrap()
-                    .as_millis() as u64,
-                std::sync::atomic::Ordering::Relaxed,
-            );
-
-            tokio::spawn(async move {
-                KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
-            });
-        }
-    }
-}
-
-#[cfg(feature = "kafka")]
-impl Drop for KafkaSink {
-    fn drop(&mut self) {
-        // Perform any necessary cleanup here
-        // For example, you might want to flush any remaining entries
-        let producer = self.producer.clone();
-        let topic = self.topic.clone();
-        let entries = self.entries.clone();
-        let last_flush = self.last_flush.clone();
-
-        tokio::spawn(async move {
-            let mut batch = entries.lock().await;
-            if !batch.is_empty() {
-                KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
-                last_flush.store(
-                    std::time::SystemTime::now()
-                        .duration_since(std::time::UNIX_EPOCH)
-                        .unwrap()
-                        .as_millis() as u64,
-                    std::sync::atomic::Ordering::Relaxed,
-                );
-            }
-        });
-
-        eprintln!("Dropping KafkaSink with topic: {}", self.topic);
-    }
-}
-
-#[cfg(feature = "webhook")]
-/// Webhook Sink Implementation
-pub struct WebhookSink {
-    endpoint: String,
-    auth_token: String,
-    client: reqwest::Client,
-    max_retries: usize,
-    retry_delay_ms: u64,
-}
-
-#[cfg(feature = "webhook")]
-impl WebhookSink {
-    pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
-        WebhookSink {
-            endpoint,
-            auth_token,
-            client: reqwest::Client::builder()
-                .timeout(std::time::Duration::from_secs(10))
-                .build()
-                .unwrap_or_else(|_| reqwest::Client::new()),
-            max_retries,
-            retry_delay_ms,
-        }
-    }
-}
-
-#[cfg(feature = "webhook")]
-#[async_trait]
-impl Sink for WebhookSink {
-    async fn write(&self, entry: &UnifiedLogEntry) {
-        let mut retries = 0;
-        let url = self.endpoint.clone();
-        let entry_clone = entry.clone();
-        let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
-        while retries < self.max_retries {
-            match self
-                .client
-                .post(&url)
-                .header(reqwest::header::AUTHORIZATION, auth_value.clone())
-                .json(&entry_clone)
-                .send()
-                .await
-            {
-                Ok(response) if response.status().is_success() => {
-                    return;
-                }
-                _ => {
-                    retries += 1;
-                    if retries < self.max_retries {
-                        tokio::time::sleep(tokio::time::Duration::from_millis(
-                            self.retry_delay_ms * (1 << retries), // Exponential backoff
-                        ))
-                        .await;
-                    }
-                }
-            }
-        }
-
-        eprintln!("Failed to send log to webhook after {} retries", self.max_retries);
-    }
-}
-
-#[cfg(feature = "webhook")]
-impl Drop for WebhookSink {
-    fn drop(&mut self) {
-        // Perform any necessary cleanup here
-        // For example, you might want to log that the sink is being dropped
-        eprintln!("Dropping WebhookSink with URL: {}", self.endpoint);
-    }
-}
-
-#[cfg(feature = "file")]
-/// File Sink Implementation
-pub struct FileSink {
-    path: String,
-    buffer_size: usize,
-    writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
-    entry_count: std::sync::atomic::AtomicUsize,
-    last_flush: std::sync::atomic::AtomicU64,
-    flush_interval_ms: u64, // Time between flushes
-    flush_threshold: usize, // Number of entries before flush
-}
-
-#[cfg(feature = "file")]
-impl FileSink {
-    /// Create a new FileSink instance
-    pub async fn new(
-        path: String,
-        buffer_size: usize,
-        flush_interval_ms: u64,
-        flush_threshold: usize,
-    ) -> Result<Self, io::Error> {
-        // check if the file exists
-        let file_exists = tokio::fs::metadata(&path).await.is_ok();
-        // if the file not exists, create it
-        if !file_exists {
-            tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
-            tracing::debug!("File does not exist, creating it. Path: {:?}", path)
-        }
-        let file = if file_exists {
-            // If the file exists, open it in append mode
-            tracing::debug!("FileSink: File exists, opening in append mode.");
-            OpenOptions::new().append(true).create(true).open(&path).await?
-        } else {
-            // If the file does not exist, create it
-            tracing::debug!("FileSink: File does not exist, creating a new file.");
-            // Create the file and write a header or initial content if needed
-            OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
-        };
-        let writer = io::BufWriter::with_capacity(buffer_size, file);
-        let now = std::time::SystemTime::now()
-            .duration_since(std::time::UNIX_EPOCH)
-            .unwrap()
-            .as_millis() as u64;
-        Ok(FileSink {
-            path,
-            buffer_size,
-            writer: Arc::new(tokio::sync::Mutex::new(writer)),
-            entry_count: std::sync::atomic::AtomicUsize::new(0),
-            last_flush: std::sync::atomic::AtomicU64::new(now),
-            flush_interval_ms,
-            flush_threshold,
-        })
-    }
-
-    #[allow(dead_code)]
-    async fn initialize_writer(&mut self) -> io::Result<()> {
-        let file = tokio::fs::File::create(&self.path).await?;
-
-        // Use buffer_size to create a buffer writer with a specified capacity
-        let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
-
-        // Replace the original writer with the new Mutex
-        self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
-        Ok(())
-    }
-
-    // Get the current buffer size
-    #[allow(dead_code)]
-    pub fn buffer_size(&self) -> usize {
-        self.buffer_size
-    }
-
-    // How to dynamically adjust the buffer size
-    #[allow(dead_code)]
-    pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
-        if self.buffer_size != new_size {
-            self.buffer_size = new_size;
-            // Reinitialize the writer directly, without checking is_some()
-            self.initialize_writer().await?;
-        }
-        Ok(())
-    }
-
-    // Check if flushing is needed based on count or time
-    fn should_flush(&self) -> bool {
-        // Check entry count threshold
-        if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
-            return true;
-        }
-
-        // Check time threshold
-        let now = std::time::SystemTime::now()
-            .duration_since(std::time::UNIX_EPOCH)
-            .unwrap()
-            .as_millis() as u64;
-
-        let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
-        now - last >= self.flush_interval_ms
-    }
-}
-
-#[cfg(feature = "file")]
-#[async_trait]
-impl Sink for FileSink {
-    async fn write(&self, entry: &UnifiedLogEntry) {
-        let line = format!("{:?}\n", entry);
-        let mut writer = self.writer.lock().await;
-
-        if let Err(e) = writer.write_all(line.as_bytes()).await {
-            eprintln!(
-                "Failed to write log to file {}: {},entry timestamp:{:?}",
-                self.path,
-                e,
-                entry.get_timestamp()
-            );
-            return;
-        }
-
-        // Only flush periodically to improve performance
-        // Logic to determine when to flush could be added here
-        // Increment the entry count
-        self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        // Check if we should flush
-        if self.should_flush() {
-            if let Err(e) = writer.flush().await {
-                eprintln!("Failed to flush log file {}: {}", self.path, e);
-                return;
-            }
-
-            // Reset counters
-            self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
-
-            let now = std::time::SystemTime::now()
-                .duration_since(std::time::UNIX_EPOCH)
-                .unwrap()
-                .as_millis() as u64;
-
-            self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
-        }
-    }
-}
-
-#[cfg(feature = "file")]
-impl Drop for FileSink {
-    fn drop(&mut self) {
-        let writer = self.writer.clone();
-        let path = self.path.clone();
-
-        tokio::task::spawn_blocking(move || {
-            let rt = tokio::runtime::Runtime::new().unwrap();
-            rt.block_on(async {
-                let mut writer = writer.lock().await;
-                if let Err(e) = writer.flush().await {
-                    eprintln!("Failed to flush log file {}: {}", path, e);
-                }
-            });
-        });
-    }
-}
-
-/// Create a list of Sink instances
-pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
-    let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
-
-    #[cfg(feature = "kafka")]
-    {
-        match &config.sinks.kafka {
-            Some(sink_kafka) => {
-                if sink_kafka.enabled {
-                    match rdkafka::config::ClientConfig::new()
-                        .set("bootstrap.servers", &sink_kafka.bootstrap_servers)
-                        .set("message.timeout.ms", "5000")
-                        .create()
-                    {
-                        Ok(producer) => {
-                            sinks.push(Arc::new(KafkaSink::new(
-                                producer,
-                                sink_kafka.topic.clone(),
-                                sink_kafka.batch_size.unwrap_or(100),
-                                sink_kafka.batch_timeout_ms.unwrap_or(1000),
-                            )));
-                        }
-                        Err(e) => {
-                            tracing::error!("Failed to create Kafka producer: {}", e);
-                        }
-                    }
-                } else {
-                    tracing::info!("Kafka sink is disabled in the configuration");
-                }
-            }
-            _ => {
-                tracing::info!("Kafka sink is not configured or disabled");
-            }
-        }
-    }
-    #[cfg(feature = "webhook")]
-    {
-        match &config.sinks.webhook {
-            Some(sink_webhook) => {
-                if sink_webhook.enabled {
-                    sinks.push(Arc::new(WebhookSink::new(
-                        sink_webhook.endpoint.clone(),
-                        sink_webhook.auth_token.clone(),
-                        sink_webhook.max_retries.unwrap_or(3),
-                        sink_webhook.retry_delay_ms.unwrap_or(100),
-                    )));
-                } else {
-                    tracing::info!("Webhook sink is disabled in the configuration");
-                }
-            }
-            _ => {
-                tracing::info!("Webhook sink is not configured or disabled");
-            }
-        }
-    }
-
-    #[cfg(feature = "file")]
-    {
-        // let config = config.clone();
-        match &config.sinks.file {
-            Some(sink_file) => {
-                tracing::info!("File sink is enabled in the configuration");
-                let path = if sink_file.enabled {
-                    sink_file.path.clone()
-                } else {
-                    "rustfs.log".to_string()
-                };
-                tracing::debug!("FileSink: Using path: {}", path);
-                sinks.push(Arc::new(
-                    FileSink::new(
-                        path.clone(),
-                        sink_file.buffer_size.unwrap_or(8192),
-                        sink_file.flush_interval_ms.unwrap_or(1000),
-                        sink_file.flush_threshold.unwrap_or(100),
-                    )
-                    .await
-                    .unwrap(),
-                ));
-            }
-            _ => {
-                tracing::info!("File sink is not configured or disabled");
-            }
-        }
-    }
-
-    sinks
-}

crates/obs/src/sinks/file.rs

@@ -0,0 +1,164 @@
+use crate::sinks::Sink;
+use crate::{LogRecord, UnifiedLogEntry};
+use async_trait::async_trait;
+use std::sync::Arc;
+use tokio::fs::OpenOptions;
+use tokio::io;
+use tokio::io::AsyncWriteExt;
+
+/// File Sink Implementation
+pub struct FileSink {
+    path: String,
+    buffer_size: usize,
+    writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
+    entry_count: std::sync::atomic::AtomicUsize,
+    last_flush: std::sync::atomic::AtomicU64,
+    flush_interval_ms: u64, // Time between flushes
+    flush_threshold: usize, // Number of entries before flush
+}
+
+impl FileSink {
+    /// Create a new FileSink instance
+    pub async fn new(
+        path: String,
+        buffer_size: usize,
+        flush_interval_ms: u64,
+        flush_threshold: usize,
+    ) -> Result<Self, io::Error> {
+        // check if the file exists
+        let file_exists = tokio::fs::metadata(&path).await.is_ok();
+        // if the file not exists, create it
+        if !file_exists {
+            tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
+            tracing::debug!("File does not exist, creating it. Path: {:?}", path)
+        }
+        let file = if file_exists {
+            // If the file exists, open it in append mode
+            tracing::debug!("FileSink: File exists, opening in append mode.");
+            OpenOptions::new().append(true).create(true).open(&path).await?
+        } else {
+            // If the file does not exist, create it
+            tracing::debug!("FileSink: File does not exist, creating a new file.");
+            // Create the file and write a header or initial content if needed
+            OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
+        };
+        let writer = io::BufWriter::with_capacity(buffer_size, file);
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as u64;
+        Ok(FileSink {
+            path,
+            buffer_size,
+            writer: Arc::new(tokio::sync::Mutex::new(writer)),
+            entry_count: std::sync::atomic::AtomicUsize::new(0),
+            last_flush: std::sync::atomic::AtomicU64::new(now),
+            flush_interval_ms,
+            flush_threshold,
+        })
+    }
+
+    #[allow(dead_code)]
+    async fn initialize_writer(&mut self) -> io::Result<()> {
+        let file = tokio::fs::File::create(&self.path).await?;
+
+        // Use buffer_size to create a buffer writer with a specified capacity
+        let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
+
+        // Replace the original writer with the new Mutex
+        self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
+        Ok(())
+    }
+
+    // Get the current buffer size
+    #[allow(dead_code)]
+    pub fn buffer_size(&self) -> usize {
+        self.buffer_size
+    }
+
+    // How to dynamically adjust the buffer size
+    #[allow(dead_code)]
+    pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
+        if self.buffer_size != new_size {
+            self.buffer_size = new_size;
+            // Reinitialize the writer directly, without checking is_some()
+            self.initialize_writer().await?;
+        }
+        Ok(())
+    }
+
+    // Check if flushing is needed based on count or time
+    fn should_flush(&self) -> bool {
+        // Check entry count threshold
+        if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
+            return true;
+        }
+
+        // Check time threshold
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as u64;
+
+        let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
+        now - last >= self.flush_interval_ms
+    }
+}
+
+#[async_trait]
+impl Sink for FileSink {
+    async fn write(&self, entry: &UnifiedLogEntry) {
+        let line = format!("{:?}\n", entry);
+        let mut writer = self.writer.lock().await;
+
+        if let Err(e) = writer.write_all(line.as_bytes()).await {
+            eprintln!(
+                "Failed to write log to file {}: {},entry timestamp:{:?}",
+                self.path,
+                e,
+                entry.get_timestamp()
+            );
+            return;
+        }
+
+        // Only flush periodically to improve performance
+        // Logic to determine when to flush could be added here
+        // Increment the entry count
+        self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+
+        // Check if we should flush
+        if self.should_flush() {
+            if let Err(e) = writer.flush().await {
+                eprintln!("Failed to flush log file {}: {}", self.path, e);
+                return;
+            }
+
+            // Reset counters
+            self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
+
+            let now = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as u64;
+
+            self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
+        }
+    }
+}
+
+impl Drop for FileSink {
+    fn drop(&mut self) {
+        let writer = self.writer.clone();
+        let path = self.path.clone();
+
+        tokio::task::spawn_blocking(move || {
+            let rt = tokio::runtime::Runtime::new().unwrap();
+            rt.block_on(async {
+                let mut writer = writer.lock().await;
+                if let Err(e) = writer.flush().await {
+                    eprintln!("Failed to flush log file {}: {}", path, e);
+                }
+            });
+        });
+    }
+}

crates/obs/src/sinks/kafka.rs

@@ -0,0 +1,165 @@
+use crate::sinks::Sink;
+use crate::{LogRecord, UnifiedLogEntry};
+use async_trait::async_trait;
+use std::sync::Arc;
+
+/// Kafka Sink Implementation
+pub struct KafkaSink {
+    producer: rdkafka::producer::FutureProducer,
+    topic: String,
+    batch_size: usize,
+    batch_timeout_ms: u64,
+    entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
+    last_flush: Arc<std::sync::atomic::AtomicU64>,
+}
+
+impl KafkaSink {
+    /// Create a new KafkaSink instance
+    pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
+        // Create Arc-wrapped values first
+        let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
+        let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as u64,
+        ));
+        let sink = KafkaSink {
+            producer: producer.clone(),
+            topic: topic.clone(),
+            batch_size,
+            batch_timeout_ms,
+            entries: entries.clone(),
+            last_flush: last_flush.clone(),
+        };
+
+        // Start background flusher
+        tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
+
+        sink
+    }
+
+    /// Add a getter method to read the batch_timeout_ms field
+    #[allow(dead_code)]
+    pub fn batch_timeout(&self) -> u64 {
+        self.batch_timeout_ms
+    }
+
+    /// Add a method to dynamically adjust the timeout if needed
+    #[allow(dead_code)]
+    pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
+        self.batch_timeout_ms = new_timeout_ms;
+    }
+
+    async fn periodic_flush(
+        producer: rdkafka::producer::FutureProducer,
+        topic: String,
+        entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
+        last_flush: Arc<std::sync::atomic::AtomicU64>,
+        timeout_ms: u64,
+    ) {
+        loop {
+            tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
+
+            let now = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as u64;
+
+            let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
+
+            if now - last >= timeout_ms {
+                let mut batch = entries.lock().await;
+                if !batch.is_empty() {
+                    Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
+                    last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
+                }
+            }
+        }
+    }
+
+    async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
+        for entry in entries {
+            let payload = match serde_json::to_string(&entry) {
+                Ok(p) => p,
+                Err(e) => {
+                    eprintln!("Failed to serialize log entry: {}", e);
+                    continue;
+                }
+            };
+
+            let span_id = entry.get_timestamp().to_rfc3339();
+
+            let _ = producer
+                .send(
+                    rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
+                    std::time::Duration::from_secs(5),
+                )
+                .await;
+        }
+    }
+}
+
+#[async_trait]
+impl Sink for KafkaSink {
+    async fn write(&self, entry: &UnifiedLogEntry) {
+        let mut batch = self.entries.lock().await;
+        batch.push(entry.clone());
+
+        let should_flush_by_size = batch.len() >= self.batch_size;
+        let should_flush_by_time = {
+            let now = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as u64;
+            let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
+            now - last >= self.batch_timeout_ms
+        };
+
+        if should_flush_by_size || should_flush_by_time {
+            // Existing flush logic
+            let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
+            let producer = self.producer.clone();
+            let topic = self.topic.clone();
+
+            self.last_flush.store(
+                std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as u64,
+                std::sync::atomic::Ordering::Relaxed,
+            );
+
+            tokio::spawn(async move {
+                KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
+            });
+        }
+    }
+}
+
+impl Drop for KafkaSink {
+    fn drop(&mut self) {
+        // Perform any necessary cleanup here
+        // For example, you might want to flush any remaining entries
+        let producer = self.producer.clone();
+        let topic = self.topic.clone();
+        let entries = self.entries.clone();
+        let last_flush = self.last_flush.clone();
+
+        tokio::spawn(async move {
+            let mut batch = entries.lock().await;
+            if !batch.is_empty() {
+                KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
+                last_flush.store(
+                    std::time::SystemTime::now()
+                        .duration_since(std::time::UNIX_EPOCH)
+                        .unwrap()
+                        .as_millis() as u64,
+                    std::sync::atomic::Ordering::Relaxed,
+                );
+            }
+        });
+
+        eprintln!("Dropping KafkaSink with topic: {}", self.topic);
+    }
+}

crates/obs/src/sinks/mod.rs

@@ -0,0 +1,92 @@
+use crate::{AppConfig, SinkConfig, UnifiedLogEntry};
+use async_trait::async_trait;
+use std::sync::Arc;
+
+#[cfg(feature = "file")]
+mod file;
+#[cfg(all(feature = "kafka", target_os = "linux"))]
+mod kafka;
+#[cfg(feature = "webhook")]
+mod webhook;
+
+/// Sink Trait definition, asynchronously write logs
+#[async_trait]
+pub trait Sink: Send + Sync {
+    async fn write(&self, entry: &UnifiedLogEntry);
+}
+
+/// Create a list of Sink instances
+pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
+    let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
+
+    for sink_config in &config.sinks {
+        match sink_config {
+            #[cfg(all(feature = "kafka", target_os = "linux"))]
+            SinkConfig::Kafka(kafka_config) => {
+                match rdkafka::config::ClientConfig::new()
+                    .set("bootstrap.servers", &kafka_config.brokers)
+                    .set("message.timeout.ms", "5000")
+                    .create()
+                {
+                    Ok(producer) => {
+                        sinks.push(Arc::new(kafka::KafkaSink::new(
+                            producer,
+                            kafka_config.topic.clone(),
+                            kafka_config.batch_size.unwrap_or(100),
+                            kafka_config.batch_timeout_ms.unwrap_or(1000),
+                        )));
+                        tracing::info!("Kafka sink created for topic: {}", kafka_config.topic);
+                    }
+                    Err(e) => {
+                        tracing::error!("Failed to create Kafka producer: {}", e);
+                    }
+                }
+            }
+            #[cfg(feature = "webhook")]
+            SinkConfig::Webhook(webhook_config) => {
+                sinks.push(Arc::new(webhook::WebhookSink::new(
+                    webhook_config.endpoint.clone(),
+                    webhook_config.auth_token.clone(),
+                    webhook_config.max_retries.unwrap_or(3),
+                    webhook_config.retry_delay_ms.unwrap_or(100),
+                )));
+                tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint);
+            }
+
+            #[cfg(feature = "file")]
+            SinkConfig::File(file_config) => {
+                tracing::debug!("FileSink: Using path: {}", file_config.path);
+                match file::FileSink::new(
+                    file_config.path.clone(),
+                    file_config.buffer_size.unwrap_or(8192),
+                    file_config.flush_interval_ms.unwrap_or(1000),
+                    file_config.flush_threshold.unwrap_or(100),
+                )
+                .await
+                {
+                    Ok(sink) => {
+                        sinks.push(Arc::new(sink));
+                        tracing::info!("File sink created for path: {}", file_config.path);
+                    }
+                    Err(e) => {
+                        tracing::error!("Failed to create File sink: {}", e);
+                    }
+                }
+            }
+            #[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
+            SinkConfig::Kafka(_) => {
+                tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled");
+            }
+            #[cfg(not(feature = "webhook"))]
+            SinkConfig::Webhook(_) => {
+                tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled");
+            }
+            #[cfg(not(feature = "file"))]
+            SinkConfig::File(_) => {
+                tracing::warn!("File sink is configured but the 'file' feature is not enabled");
+            }
+        }
+    }
+
+    sinks
+}

crates/obs/src/sinks/webhook.rs

@@ -0,0 +1,70 @@
+use crate::sinks::Sink;
+use crate::UnifiedLogEntry;
+use async_trait::async_trait;
+
+/// Webhook Sink Implementation
+pub struct WebhookSink {
+    endpoint: String,
+    auth_token: String,
+    client: reqwest::Client,
+    max_retries: usize,
+    retry_delay_ms: u64,
+}
+
+impl WebhookSink {
+    pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
+        WebhookSink {
+            endpoint,
+            auth_token,
+            client: reqwest::Client::builder()
+                .timeout(std::time::Duration::from_secs(10))
+                .build()
+                .unwrap_or_else(|_| reqwest::Client::new()),
+            max_retries,
+            retry_delay_ms,
+        }
+    }
+}
+
+#[async_trait]
+impl Sink for WebhookSink {
+    async fn write(&self, entry: &UnifiedLogEntry) {
+        let mut retries = 0;
+        let url = self.endpoint.clone();
+        let entry_clone = entry.clone();
+        let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
+        while retries < self.max_retries {
+            match self
+                .client
+                .post(&url)
+                .header(reqwest::header::AUTHORIZATION, auth_value.clone())
+                .json(&entry_clone)
+                .send()
+                .await
+            {
+                Ok(response) if response.status().is_success() => {
+                    return;
+                }
+                _ => {
+                    retries += 1;
+                    if retries < self.max_retries {
+                        tokio::time::sleep(tokio::time::Duration::from_millis(
+                            self.retry_delay_ms * (1 << retries), // Exponential backoff
+                        ))
+                        .await;
+                    }
+                }
+            }
+        }
+
+        eprintln!("Failed to send log to webhook after {} retries", self.max_retries);
+    }
+}
+
+impl Drop for WebhookSink {
+    fn drop(&mut self) {
+        // Perform any necessary cleanup here
+        // For example, you might want to log that the sink is being dropped
+        eprintln!("Dropping WebhookSink with URL: {}", self.endpoint);
+    }
+}

crates/obs/src/telemetry.rs

@@ -1,5 +1,3 @@
-use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
-use crate::utils::get_local_ip_with_default;
 use crate::OtelConfig;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{global, KeyValue};
@@ -15,6 +13,8 @@ use opentelemetry_semantic_conventions::{
     attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
     SCHEMA_URL,
 };
+use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
+use rustfs_utils::get_local_ip_with_default;
 use smallvec::SmallVec;
 use std::borrow::Cow;
 use std::io::IsTerminal;
@@ -70,7 +70,7 @@ impl Drop for OtelGuard {
 /// create OpenTelemetry Resource
 fn resource(config: &OtelConfig) -> Resource {
     Resource::builder()
-        .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
+        .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string())
         .with_schema_url(
             [
                 KeyValue::new(
@@ -101,8 +101,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
     let endpoint = &config.endpoint;
     let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
     let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
-    let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
-    let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
+    let logger_level = config.logger_level.as_deref().unwrap_or(DEFAULT_LOG_LEVEL);
+    let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
 
     // Pre-create resource objects to avoid repeated construction
     let res = resource(config);
@@ -210,7 +210,8 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
             .with_thread_names(true)
             .with_thread_ids(true)
             .with_file(true)
-            .with_line_number(true);
+            .with_line_number(true)
+            .with_filter(build_env_filter(logger_level, None));
 
         let filter = build_env_filter(logger_level, None);
         let otel_filter = build_env_filter(logger_level, None);
@@ -218,7 +219,7 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
         let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
 
         // Configure registry to avoid repeated calls to filter methods
-        let _registry = tracing_subscriber::registry()
+        tracing_subscriber::registry()
             .with(filter)
             .with(ErrorLayer::default())
             .with(if config.local_logging_enabled.unwrap_or(false) {
@@ -230,10 +231,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
             .with(otel_layer)
             .with(MetricsLayer::new(meter_provider.clone()))
             .init();
+
         if !endpoint.is_empty() {
             info!(
-                "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
-                endpoint, logger_level
+                "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}",
+                endpoint,
+                logger_level,
+                std::env::var("RUST_LOG").unwrap_or_else(|_| "未设置".to_string())
             );
         }
     }
@@ -248,7 +252,6 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter {
     let level = default_level.unwrap_or(logger_level);
     let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
-
     if !matches!(logger_level, "trace" | "debug") {
         let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
         for directive in directives {

crates/obs/src/worker.rs

@@ -1,9 +1,9 @@
-use crate::{sink::Sink, UnifiedLogEntry};
+use crate::{sinks::Sink, UnifiedLogEntry};
 use std::sync::Arc;
 use tokio::sync::mpsc::Receiver;
 
 /// Start the log processing worker thread
-pub async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
+pub(crate) async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
     let mut receiver = receiver;
     while let Some(entry) = receiver.recv().await {
         for sink in &sinks {

crates/utils/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "rustfs-utils"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+local-ip-address = { workspace = true, optional = true }
+rustfs-config = { workspace = true }
+rustls = { workspace = true, optional = true }
+rustls-pemfile = { workspace = true, optional = true }
+rustls-pki-types = { workspace = true, optional = true }
+tracing = { workspace = true }
+
+[lints]
+workspace = true
+
+[features]
+default = ["ip"]  # features that are enabled by default
+ip = ["dep:local-ip-address"]  # ip characteristics and their dependencies
+tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"]  # tls characteristics and their dependencies
+net = ["ip"]  # empty network features
+full = ["ip", "tls", "net"]  # all features

crates/utils/src/certs.rs

@@ -0,0 +1,186 @@
+use rustfs_config::{RUSTFS_TLS_CERT, RUSTFS_TLS_KEY};
+use rustls::server::{ClientHello, ResolvesServerCert, ResolvesServerCertUsingSni};
+use rustls::sign::CertifiedKey;
+use rustls_pemfile::{certs, private_key};
+use rustls_pki_types::{CertificateDer, PrivateKeyDer};
+use std::collections::HashMap;
+use std::io::Error;
+use std::path::Path;
+use std::sync::Arc;
+use std::{fs, io};
+use tracing::{debug, warn};
+
+/// Load public certificate from file.
+/// This function loads a public certificate from the specified file.
+pub fn load_certs(filename: &str) -> io::Result<Vec<CertificateDer<'static>>> {
+    // Open certificate file.
+    let cert_file = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?;
+    let mut reader = io::BufReader::new(cert_file);
+
+    // Load and return certificate.
+    let certs = certs(&mut reader)
+        .collect::<Result<Vec<_>, _>>()
+        .map_err(|e| certs_error(format!("certificate file {} format error:{:?}", filename, e)))?;
+    if certs.is_empty() {
+        return Err(certs_error(format!(
+            "No valid certificate was found in the certificate file {}",
+            filename
+        )));
+    }
+    Ok(certs)
+}
+
+/// Load private key from file.
+/// This function loads a private key from the specified file.
+pub fn load_private_key(filename: &str) -> io::Result<PrivateKeyDer<'static>> {
+    // Open keyfile.
+    let keyfile = fs::File::open(filename).map_err(|e| certs_error(format!("failed to open {}: {}", filename, e)))?;
+    let mut reader = io::BufReader::new(keyfile);
+
+    // Load and return a single private key.
+    private_key(&mut reader)?.ok_or_else(|| certs_error(format!("no private key found in {}", filename)))
+}
+
+/// error function
+pub fn certs_error(err: String) -> Error {
+    Error::new(io::ErrorKind::Other, err)
+}
+
+/// Load all certificates and private keys in the directory
+/// This function loads all certificate and private key pairs from the specified directory.
+/// It looks for files named `rustfs_cert.pem` and `rustfs_key.pem` in each subdirectory.
+/// The root directory can also contain a default certificate/private key pair.
+pub fn load_all_certs_from_directory(
+    dir_path: &str,
+) -> io::Result<HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>> {
+    let mut cert_key_pairs = HashMap::new();
+    let dir = Path::new(dir_path);
+
+    if !dir.exists() || !dir.is_dir() {
+        return Err(certs_error(format!(
+            "The certificate directory does not exist or is not a directory: {}",
+            dir_path
+        )));
+    }
+
+    // 1. First check whether there is a certificate/private key pair in the root directory
+    let root_cert_path = dir.join(RUSTFS_TLS_CERT);
+    let root_key_path = dir.join(RUSTFS_TLS_KEY);
+
+    if root_cert_path.exists() && root_key_path.exists() {
+        debug!("find the root directory certificate: {:?}", root_cert_path);
+        let root_cert_str = root_cert_path
+            .to_str()
+            .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root certificate path: {:?}", root_cert_path)))?;
+        let root_key_str = root_key_path
+            .to_str()
+            .ok_or_else(|| certs_error(format!("Invalid UTF-8 in root key path: {:?}", root_key_path)))?;
+        match load_cert_key_pair(root_cert_str, root_key_str) {
+            Ok((certs, key)) => {
+                // The root directory certificate is used as the default certificate and is stored using special keys.
+                cert_key_pairs.insert("default".to_string(), (certs, key));
+            }
+            Err(e) => {
+                warn!("unable to load root directory certificate: {}", e);
+            }
+        }
+    }
+
+    // 2.iterate through all folders in the directory
+    for entry in fs::read_dir(dir)? {
+        let entry = entry?;
+        let path = entry.path();
+
+        if path.is_dir() {
+            let domain_name = path
+                .file_name()
+                .and_then(|name| name.to_str())
+                .ok_or_else(|| certs_error(format!("invalid domain name directory:{:?}", path)))?;
+
+            // find certificate and private key files
+            let cert_path = path.join(RUSTFS_TLS_CERT); // e.g., rustfs_cert.pem
+            let key_path = path.join(RUSTFS_TLS_KEY); // e.g., rustfs_key.pem
+
+            if cert_path.exists() && key_path.exists() {
+                debug!("find the domain name certificate: {} in {:?}", domain_name, cert_path);
+                match load_cert_key_pair(cert_path.to_str().unwrap(), key_path.to_str().unwrap()) {
+                    Ok((certs, key)) => {
+                        cert_key_pairs.insert(domain_name.to_string(), (certs, key));
+                    }
+                    Err(e) => {
+                        warn!("unable to load the certificate for {} domain name: {}", domain_name, e);
+                    }
+                }
+            }
+        }
+    }
+
+    if cert_key_pairs.is_empty() {
+        return Err(certs_error(format!(
+            "No valid certificate/private key pair found in directory {}",
+            dir_path
+        )));
+    }
+
+    Ok(cert_key_pairs)
+}
+
+/// loading a single certificate private key pair
+/// This function loads a certificate and private key from the specified paths.
+/// It returns a tuple containing the certificate and private key.
+fn load_cert_key_pair(cert_path: &str, key_path: &str) -> io::Result<(Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)> {
+    let certs = load_certs(cert_path)?;
+    let key = load_private_key(key_path)?;
+    Ok((certs, key))
+}
+
+/// Create a multi-cert resolver
+/// This function loads all certificates and private keys from the specified directory.
+/// It uses the first certificate/private key pair found in the root directory as the default certificate.
+/// The rest of the certificates/private keys are used for SNI resolution.
+///
+pub fn create_multi_cert_resolver(
+    cert_key_pairs: HashMap<String, (Vec<CertificateDer<'static>>, PrivateKeyDer<'static>)>,
+) -> io::Result<impl ResolvesServerCert> {
+    #[derive(Debug)]
+    struct MultiCertResolver {
+        cert_resolver: ResolvesServerCertUsingSni,
+        default_cert: Option<Arc<CertifiedKey>>,
+    }
+    impl ResolvesServerCert for MultiCertResolver {
+        fn resolve(&self, client_hello: ClientHello) -> Option<Arc<CertifiedKey>> {
+            // try matching certificates with sni
+            if let Some(cert) = self.cert_resolver.resolve(client_hello) {
+                return Some(cert);
+            }
+
+            // If there is no matching SNI certificate, use the default certificate
+            self.default_cert.clone()
+        }
+    }
+
+    let mut resolver = ResolvesServerCertUsingSni::new();
+    let mut default_cert = None;
+
+    for (domain, (certs, key)) in cert_key_pairs {
+        // create a signature
+        let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
+            .map_err(|e| certs_error(format!("unsupported private key types:{}, err:{:?}", domain, e)))?;
+
+        // create a CertifiedKey
+        let certified_key = CertifiedKey::new(certs, signing_key);
+        if domain == "default" {
+            default_cert = Some(Arc::new(certified_key.clone()));
+        } else {
+            // add certificate to resolver
+            resolver
+                .add(&domain, certified_key)
+                .map_err(|e| certs_error(format!("failed to add a domain name certificate:{},err: {:?}", domain, e)))?;
+        }
+    }
+
+    Ok(MultiCertResolver {
+        cert_resolver: resolver,
+        default_cert,
+    })
+}

crates/utils/src/ip.rs

@@ -1,4 +1,3 @@
-use local_ip_address::{local_ip, local_ipv6};
 use std::net::{IpAddr, Ipv4Addr};
 
 /// Get the IP address of the machine
@@ -11,7 +10,9 @@ use std::net::{IpAddr, Ipv4Addr};
 /// * `Some(IpAddr)` - Native IP address (IPv4 or IPv6)
 /// * `None` - Unable to obtain any native IP address
 pub fn get_local_ip() -> Option<IpAddr> {
-    local_ip().ok().or_else(|| local_ipv6().ok())
+    local_ip_address::local_ip()
+        .ok()
+        .or_else(|| local_ip_address::local_ipv6().ok())
 }
 
 /// Get the IP address of the machine as a string

crates/utils/src/lib.rs

@@ -0,0 +1,8 @@
+mod certs;
+mod ip;
+mod net;
+
+#[cfg(feature = "ip")]
+pub use certs::*;
+#[cfg(feature = "ip")]
+pub use ip::*;

crates/utils/src/net.rs

@@ -0,0 +1 @@
+

crates/zip/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "zip"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+
+[dependencies]
+async-compression = { version = "0.4.0", features = [
+    "tokio",
+    "bzip2",
+    "gzip",
+    "zlib",
+    "zstd",
+    "xz",
+] }
+# async_zip = { version = "0.0.17", features = ["tokio"] }
+# rc-zip-tokio = "4.2.6"
+tokio = { version = "1.45.0", features = ["full"] }
+tokio-stream = "0.1.17"
+tokio-tar = { workspace = true }
+xz2 = { version = "0.1", optional = true, features = ["static"] }
+
+
+[lints]
+workspace = true

crates/zip/src/lib.rs

@@ -0,0 +1,124 @@
+use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZlibDecoder, ZstdDecoder};
+use tokio::io::{self, AsyncRead, BufReader};
+use tokio_stream::StreamExt;
+use tokio_tar::Archive;
+
+#[derive(Debug, PartialEq)]
+pub enum CompressionFormat {
+    Gzip,  //.gz
+    Bzip2, //.bz2
+    // Lz4,   //.lz4
+    Zip,
+    Xz,   //.xz
+    Zlib, //.z
+    Zstd, //.zst
+    Unknown,
+}
+
+impl CompressionFormat {
+    pub fn from_extension(ext: &str) -> Self {
+        match ext {
+            "gz" => CompressionFormat::Gzip,
+            "bz2" => CompressionFormat::Bzip2,
+            // "lz4" => CompressionFormat::Lz4,
+            "zip" => CompressionFormat::Zip,
+            "xz" => CompressionFormat::Xz,
+            "zlib" => CompressionFormat::Zlib,
+            "zst" => CompressionFormat::Zstd,
+            _ => CompressionFormat::Unknown,
+        }
+    }
+
+    pub fn get_decoder<R>(&self, input: R) -> io::Result<Box<dyn AsyncRead + Send + Unpin>>
+    where
+        R: AsyncRead + Send + Unpin + 'static,
+    {
+        let reader = BufReader::new(input);
+
+        let decoder: Box<dyn AsyncRead + Send + Unpin + 'static> = match self {
+            CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)),
+            CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)),
+            // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)),
+            CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)),
+            CompressionFormat::Xz => Box::new(XzDecoder::new(reader)),
+            CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)),
+            _ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format")),
+        };
+
+        Ok(decoder)
+    }
+}
+
+pub async fn decompress<R, F>(input: R, format: CompressionFormat, mut callback: F) -> io::Result<()>
+where
+    R: AsyncRead + Send + Unpin + 'static,
+    F: AsyncFnMut(tokio_tar::Entry<Archive<Box<dyn AsyncRead + Send + Unpin + 'static>>>) -> std::io::Result<()> + Send + 'static,
+{
+    // 打开输入文件
+    // println!("format {:?}", format);
+
+    let decoder = format.get_decoder(input)?;
+
+    // let reader: BufReader<R> = BufReader::new(input);
+
+    // // 根据文件扩展名选择解压器
+    // let decoder: Box<dyn AsyncRead + Send + Unpin> = match format {
+    //     CompressionFormat::Gzip => Box::new(GzipDecoder::new(reader)),
+    //     CompressionFormat::Bzip2 => Box::new(BzDecoder::new(reader)),
+    //     // CompressionFormat::Lz4 => Box::new(Lz4Decoder::new(reader)),
+    //     CompressionFormat::Zlib => Box::new(ZlibDecoder::new(reader)),
+    //     CompressionFormat::Xz => Box::new(XzDecoder::new(reader)),
+    //     CompressionFormat::Zstd => Box::new(ZstdDecoder::new(reader)),
+    //     // CompressionFormat::Zip => Box::new(DeflateDecoder::new(reader)),
+    //     _ => {
+    //         return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format"));
+    //     }
+    // };
+
+    let mut ar = Archive::new(decoder);
+    let mut entries = ar.entries().unwrap();
+    while let Some(entry) = entries.next().await {
+        let f = match entry {
+            Ok(f) => f,
+            Err(e) => {
+                println!("Error reading entry: {}", e);
+                return Err(e);
+            }
+        };
+        // println!("{}", f.path().unwrap().display());
+        callback(f).await?;
+    }
+
+    Ok(())
+}
+
+// #[tokio::test]
+// async fn test_decompress() -> io::Result<()> {
+//     use std::path::Path;
+//     use tokio::fs::File;
+
+//     let input_path = "/Users/weisd/Downloads/wsd.tar.gz"; // 替换为你的压缩文件路径
+
+//     let f = File::open(input_path).await?;
+
+//     let Some(ext) = Path::new(input_path).extension().and_then(|s| s.to_str()) else {
+//         return Err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported file format"));
+//     };
+
+//     match decompress(
+//         f,
+//         CompressionFormat::from_extension(ext),
+//         |entry: tokio_tar::Entry<Archive<Box<dyn AsyncRead + Send + Unpin>>>| async move {
+//             let path = entry.path().unwrap();
+//             println!("Extracted: {}", path.display());
+//             Ok(())
+//         },
+//     )
+//     .await
+//     {
+//         Ok(_) => println!("解压成功！"),
+//         Err(e) => println!("解压失败: {}", e),
+//     }
+
+//     Ok(())
+// }