Feature/rustfs config (#396)

* init rustfs config

* improve code for rustfs-config crate

* add

* improve code for comment

* fix: modify rustfs-config crate name

* add default fn

* improve error logger

* fix: modify docker config yaml

* improve code for config

* feat: restrict kafka feature to Linux only

- Add target-specific feature configuration in Cargo.toml for obs and event-notifier crates
- Implement conditional compilation for kafka feature only on Linux systems
- Add appropriate error handling for non-Linux platforms
- Ensure backward compatibility with existing code

* refactor(ci): optimize build workflow for better efficiency

- Integrate GUI build steps into main build-rustfs job
- Add conditional GUI build execution based on tag releases
- Simplify workflow by removing redundant build-rustfs-gui job
- Copy binary directly to embedded-rustfs directory without downloading artifacts
- Update merge job dependency to only rely on build-rustfs
- Improve cross-platform compatibility for Windows binary naming (.exe)
- Streamline artifact uploading and OSS publishing process
- Maintain consistent conditional logic for release operations

* refactor(ci): optimize build workflow for better efficiency

- Integrate GUI build steps into main build-rustfs job
- Add conditional GUI build execution based on tag releases
- Simplify workflow by removing redundant build-rustfs-gui job
- Copy binary directly to embedded-rustfs directory without downloading artifacts
- Update merge job dependency to only rely on build-rustfs
- Improve cross-platform compatibility for Windows binary naming (.exe)
- Streamline artifact uploading and OSS publishing process
- Maintain consistent conditional logic for release operations

* fix(ci): add repo-token to setup-protoc action for authentication

- Add GITHUB_TOKEN parameter to arduino/setup-protoc@v3 action
- Ensure proper authentication for Protoc installation in CI workflow
- Maintain consistent setup across different CI environments

* modify config

* improve readme.md

* remove env config relation

* add allow(dead_code)

crates/config/src/observability/config.rs

@@ -1,13 +1,13 @@
 use crate::observability::logger::LoggerConfig;
 use crate::observability::otel::OtelConfig;
 use crate::observability::sink::SinkConfig;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 /// Observability configuration
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct ObservabilityConfig {
     pub otel: OtelConfig,
-    pub sinks: SinkConfig,
+    pub sinks: Vec<SinkConfig>,
     pub logger: Option<LoggerConfig>,
 }
 
@@ -15,7 +15,7 @@ impl ObservabilityConfig {
     pub fn new() -> Self {
         Self {
             otel: OtelConfig::new(),
-            sinks: SinkConfig::new(),
+            sinks: vec![SinkConfig::new()],
             logger: Some(LoggerConfig::new()),
         }
     }

crates/config/src/observability/file.rs

@@ -0,0 +1,59 @@
+use serde::{Deserialize, Serialize};
+use std::env;
+
+/// File sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FileSink {
+    pub path: String,
+    #[serde(default = "default_buffer_size")]
+    pub buffer_size: Option<usize>,
+    #[serde(default = "default_flush_interval_ms")]
+    pub flush_interval_ms: Option<u64>,
+    #[serde(default = "default_flush_threshold")]
+    pub flush_threshold: Option<usize>,
+}
+
+impl FileSink {
+    pub fn new() -> Self {
+        Self {
+            path: env::var("RUSTFS_SINKS_FILE_PATH")
+                .ok()
+                .filter(|s| !s.trim().is_empty())
+                .unwrap_or_else(default_path),
+            buffer_size: default_buffer_size(),
+            flush_interval_ms: default_flush_interval_ms(),
+            flush_threshold: default_flush_threshold(),
+        }
+    }
+}
+
+impl Default for FileSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_buffer_size() -> Option<usize> {
+    Some(8192)
+}
+fn default_flush_interval_ms() -> Option<u64> {
+    Some(1000)
+}
+fn default_flush_threshold() -> Option<usize> {
+    Some(100)
+}
+
+fn default_path() -> String {
+    let temp_dir = env::temp_dir().join("rustfs");
+
+    if let Err(e) = std::fs::create_dir_all(&temp_dir) {
+        eprintln!("Failed to create log directory: {}", e);
+        return "rustfs/rustfs.log".to_string();
+    }
+
+    temp_dir
+        .join("rustfs.log")
+        .to_str()
+        .unwrap_or("rustfs/rustfs.log")
+        .to_string()
+}

crates/config/src/observability/file_sink.rs

@@ -1,25 +0,0 @@
-use serde::Deserialize;
-
-/// File sink configuration
-#[derive(Debug, Deserialize, Clone)]
-pub struct FileSinkConfig {
-    pub path: String,
-    pub max_size: u64,
-    pub max_backups: u64,
-}
-
-impl FileSinkConfig {
-    pub fn new() -> Self {
-        Self {
-            path: "".to_string(),
-            max_size: 0,
-            max_backups: 0,
-        }
-    }
-}
-
-impl Default for FileSinkConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}

crates/config/src/observability/kafka.rs

@@ -0,0 +1,36 @@
+use serde::{Deserialize, Serialize};
+
+/// Kafka sink configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct KafkaSink {
+    pub brokers: String,
+    pub topic: String,
+    #[serde(default = "default_batch_size")]
+    pub batch_size: Option<usize>,
+    #[serde(default = "default_batch_timeout_ms")]
+    pub batch_timeout_ms: Option<u64>,
+}
+
+impl KafkaSink {
+    pub fn new() -> Self {
+        Self {
+            brokers: "localhost:9092".to_string(),
+            topic: "rustfs".to_string(),
+            batch_size: default_batch_size(),
+            batch_timeout_ms: default_batch_timeout_ms(),
+        }
+    }
+}
+
+impl Default for KafkaSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_batch_size() -> Option<usize> {
+    Some(100)
+}
+fn default_batch_timeout_ms() -> Option<u64> {
+    Some(1000)
+}

crates/config/src/observability/kafka_sink.rs

@@ -1,23 +0,0 @@
-use serde::Deserialize;
-
-/// Kafka sink configuration
-#[derive(Debug, Deserialize, Clone)]
-pub struct KafkaSinkConfig {
-    pub brokers: Vec<String>,
-    pub topic: String,
-}
-
-impl KafkaSinkConfig {
-    pub fn new() -> Self {
-        Self {
-            brokers: vec!["localhost:9092".to_string()],
-            topic: "rustfs".to_string(),
-        }
-    }
-}
-
-impl Default for KafkaSinkConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}

crates/config/src/observability/logger.rs

@@ -1,7 +1,7 @@
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 /// Logger configuration
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct LoggerConfig {
     pub queue_capacity: Option<usize>,
 }

crates/config/src/observability/mod.rs

@@ -1,8 +1,7 @@
 pub(crate) mod config;
-pub(crate) mod file_sink;
-pub(crate) mod kafka_sink;
+pub(crate) mod file;
+pub(crate) mod kafka;
 pub(crate) mod logger;
-pub(crate) mod observability;
 pub(crate) mod otel;
 pub(crate) mod sink;
-pub(crate) mod webhook_sink;
+pub(crate) mod webhook;

crates/config/src/observability/observability.rs

@@ -1,22 +0,0 @@
-use crate::observability::logger::LoggerConfig;
-use crate::observability::otel::OtelConfig;
-use crate::observability::sink::SinkConfig;
-use serde::Deserialize;
-
-/// Observability configuration
-#[derive(Debug, Deserialize, Clone)]
-pub struct ObservabilityConfig {
-    pub otel: OtelConfig,
-    pub sinks: SinkConfig,
-    pub logger: Option<LoggerConfig>,
-}
-
-impl ObservabilityConfig {
-    pub fn new() -> Self {
-        Self {
-            otel: OtelConfig::new(),
-            sinks: SinkConfig::new(),
-            logger: Some(LoggerConfig::new()),
-        }
-    }
-}

crates/config/src/observability/otel.rs

@@ -1,22 +1,25 @@
-use serde::Deserialize;
+use crate::constants::app::{ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT};
+use crate::{APP_NAME, DEFAULT_LOG_LEVEL};
+use serde::{Deserialize, Serialize};
+use std::env;
 
 /// OpenTelemetry configuration
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct OtelConfig {
-    pub endpoint: String,
-    pub service_name: String,
-    pub service_version: String,
-    pub resource_attributes: Vec<String>,
+    pub endpoint: String,                    // Endpoint for metric collection
+    pub use_stdout: Option<bool>,            // Output to stdout
+    pub sample_ratio: Option<f64>,           // Trace sampling ratio
+    pub meter_interval: Option<u64>,         // Metric collection interval
+    pub service_name: Option<String>,        // Service name
+    pub service_version: Option<String>,     // Service version
+    pub environment: Option<String>,         // Environment
+    pub logger_level: Option<String>,        // Logger level
+    pub local_logging_enabled: Option<bool>, // Local logging enabled
 }
 
 impl OtelConfig {
     pub fn new() -> Self {
-        Self {
-            endpoint: "http://localhost:4317".to_string(),
-            service_name: "rustfs".to_string(),
-            service_version: "0.1.0".to_string(),
-            resource_attributes: vec![],
-        }
+        extract_otel_config_from_env()
     }
 }
 
@@ -25,3 +28,42 @@ impl Default for OtelConfig {
         Self::new()
     }
 }
+
+// Helper function: Extract observable configuration from environment variables
+fn extract_otel_config_from_env() -> OtelConfig {
+    OtelConfig {
+        endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()),
+        use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(USE_STDOUT)),
+        sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(SAMPLE_RATIO)),
+        meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(METER_INTERVAL)),
+        service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(APP_NAME.to_string())),
+        service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(SERVICE_VERSION.to_string())),
+        environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(ENVIRONMENT.to_string())),
+        logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(DEFAULT_LOG_LEVEL.to_string())),
+        local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .or(Some(false)),
+    }
+}

crates/config/src/observability/sink.rs

@@ -1,23 +1,20 @@
-use crate::observability::file_sink::FileSinkConfig;
-use crate::observability::kafka_sink::KafkaSinkConfig;
-use crate::observability::webhook_sink::WebhookSinkConfig;
-use serde::Deserialize;
+use crate::observability::file::FileSink;
+use crate::observability::kafka::KafkaSink;
+use crate::observability::webhook::WebhookSink;
+use serde::{Deserialize, Serialize};
 
 /// Sink configuration
-#[derive(Debug, Deserialize, Clone)]
-pub struct SinkConfig {
-    pub kafka: Option<KafkaSinkConfig>,
-    pub webhook: Option<WebhookSinkConfig>,
-    pub file: Option<FileSinkConfig>,
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum SinkConfig {
+    Kafka(KafkaSink),
+    Webhook(WebhookSink),
+    File(FileSink),
 }
 
 impl SinkConfig {
     pub fn new() -> Self {
-        Self {
-            kafka: None,
-            webhook: None,
-            file: Some(FileSinkConfig::new()),
-        }
+        Self::File(FileSink::new())
     }
 }
 

crates/config/src/observability/webhook.rs

@@ -0,0 +1,39 @@
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Webhook sink configuration
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct WebhookSink {
+    pub endpoint: String,
+    pub auth_token: String,
+    pub headers: Option<HashMap<String, String>>,
+    #[serde(default = "default_max_retries")]
+    pub max_retries: Option<usize>,
+    #[serde(default = "default_retry_delay_ms")]
+    pub retry_delay_ms: Option<u64>,
+}
+
+impl WebhookSink {
+    pub fn new() -> Self {
+        Self {
+            endpoint: "".to_string(),
+            auth_token: "".to_string(),
+            headers: Some(HashMap::new()),
+            max_retries: default_max_retries(),
+            retry_delay_ms: default_retry_delay_ms(),
+        }
+    }
+}
+
+impl Default for WebhookSink {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn default_max_retries() -> Option<usize> {
+    Some(3)
+}
+fn default_retry_delay_ms() -> Option<u64> {
+    Some(100)
+}

crates/config/src/observability/webhook_sink.rs

@@ -1,25 +0,0 @@
-use serde::Deserialize;
-
-/// Webhook sink configuration
-#[derive(Debug, Deserialize, Clone)]
-pub struct WebhookSinkConfig {
-    pub url: String,
-    pub method: String,
-    pub headers: Vec<(String, String)>,
-}
-
-impl WebhookSinkConfig {
-    pub fn new() -> Self {
-        Self {
-            url: "http://localhost:8080/webhook".to_string(),
-            method: "POST".to_string(),
-            headers: vec![],
-        }
-    }
-}
-
-impl Default for WebhookSinkConfig {
-    fn default() -> Self {
-        Self::new()
-    }
-}