modify logger level from info to error (#744)

* modify logger level from `info` to `error`

* fix test

* improve tokio runtime config

* add rustfs helm chart files (#747)

* add rustfs helm chart files

* update readme file with helm chart

* delete helm chart license file

* fix typo in readme file

* fix: restore localized samples in tests (#749)

* fix: restore required localized examples

* style: fix formatting issues

* improve code for Observability

* upgrade crates version

* fix

* up

* fix

---------

Co-authored-by: majinghe <42570491+majinghe@users.noreply.github.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/obs/src/config.rs

@@ -18,21 +18,39 @@ use rustfs_config::observability::{
     ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT,
 };
 use rustfs_config::{
-    APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, DEFAULT_LOG_ROTATION_TIME,
-    DEFAULT_OBS_LOG_FILENAME, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
+    APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_LOCAL_LOGGING_ENABLED, DEFAULT_LOG_ROTATION_SIZE_MB,
+    DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
 };
 use rustfs_utils::dirs::get_log_directory_to_string;
 use serde::{Deserialize, Serialize};
 use std::env;
 
-/// OpenTelemetry Configuration
-/// Add service name, service version, environment
-/// Add interval time for metric collection
-/// Add sample ratio for trace sampling
-/// Add endpoint for metric collection
-/// Add use_stdout for output to stdout
-/// Add logger level for log level
-/// Add local_logging_enabled for local logging enabled
+/// Observability: OpenTelemetry configuration
+/// # Fields
+/// * `endpoint`: Endpoint for metric collection
+/// * `use_stdout`: Output to stdout
+/// * `sample_ratio`: Trace sampling ratio
+/// * `meter_interval`: Metric collection interval
+/// * `service_name`: Service name
+/// * `service_version`: Service version
+/// * `environment`: Environment
+/// * `logger_level`: Logger level
+/// * `local_logging_enabled`: Local logging enabled
+/// # Added flexi_logger related configurations
+/// * `log_directory`: Log file directory
+/// * `log_filename`: The name of the log file
+/// * `log_rotation_size_mb`: Log file size cut threshold (MB)
+/// * `log_rotation_time`: Logs are cut by time (Hour,Day,Minute,Second)
+/// * `log_keep_files`: Number of log files to be retained
+/// # Returns
+/// A new instance of OtelConfig
+///
+/// # Example
+/// ```no_run
+/// use rustfs_obs::OtelConfig;
+///
+/// let config = OtelConfig::new();
+/// ```
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct OtelConfig {
     pub endpoint: String,                    // Endpoint for metric collection
@@ -102,7 +120,7 @@ impl OtelConfig {
             local_logging_enabled: env::var(ENV_OBS_LOCAL_LOGGING_ENABLED)
                 .ok()
                 .and_then(|v| v.parse().ok())
-                .or(Some(false)),
+                .or(Some(DEFAULT_LOG_LOCAL_LOGGING_ENABLED)),
             log_directory: Some(get_log_directory_to_string(ENV_OBS_LOG_DIRECTORY)),
             log_filename: env::var(ENV_OBS_LOG_FILENAME)
                 .ok()
@@ -127,11 +145,28 @@ impl OtelConfig {
     ///
     /// # Returns
     /// A new instance of OtelConfig
+    ///
+    /// # Example
+    /// ```no_run
+    /// use rustfs_obs::OtelConfig;
+    ///
+    /// let config = OtelConfig::new();
+    /// ```
     pub fn new() -> Self {
         Self::extract_otel_config_from_env(None)
     }
 }
 
+/// Implement Default trait for OtelConfig
+/// This allows creating a default instance of OtelConfig using OtelConfig::default()
+/// which internally calls OtelConfig::new()
+///
+/// # Example
+/// ```no_run
+/// use rustfs_obs::OtelConfig;
+///
+/// let config = OtelConfig::default();
+/// ```
 impl Default for OtelConfig {
     fn default() -> Self {
         Self::new()
@@ -165,6 +200,20 @@ impl AppConfig {
         }
     }
 
+    /// Create a new instance of AppConfig with specified endpoint
+    ///
+    /// # Arguments
+    /// * `endpoint` - An optional string representing the endpoint for metric collection
+    ///
+    /// # Returns
+    /// A new instance of AppConfig
+    ///
+    /// # Example
+    /// ```no_run
+    /// use rustfs_obs::AppConfig;
+    ///
+    /// let config = AppConfig::new_with_endpoint(Some("http://localhost:4317".to_string()));
+    /// ```
     pub fn new_with_endpoint(endpoint: Option<String>) -> Self {
         Self {
             observability: OtelConfig::extract_otel_config_from_env(endpoint),
@@ -172,7 +221,16 @@ impl AppConfig {
     }
 }
 
-// implement default for AppConfig
+/// Implement Default trait for AppConfig
+/// This allows creating a default instance of AppConfig using AppConfig::default()
+/// which internally calls AppConfig::new()
+///
+/// # Example
+/// ```no_run
+/// use rustfs_obs::AppConfig;
+///
+/// let config = AppConfig::default();
+/// ```
 impl Default for AppConfig {
     fn default() -> Self {
         Self::new()

crates/obs/src/global.rs

@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::AppConfig;
 use crate::telemetry::{OtelGuard, init_telemetry};
-use opentelemetry::metrics::Meter;
-use rustfs_config::APP_NAME;
+use crate::{AppConfig, SystemObserver};
 use std::sync::{Arc, Mutex};
 use tokio::sync::{OnceCell, SetError};
 use tracing::{error, info};
@@ -26,20 +24,11 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 /// Flag indicating if observability is enabled
 pub(crate) static IS_OBSERVABILITY_ENABLED: OnceCell<bool> = OnceCell::const_new();
 
-/// Name of the observability meter
-pub(crate) static OBSERVABILITY_METER_NAME: OnceCell<String> = OnceCell::const_new();
-
 /// Check whether Observability is enabled
 pub fn is_observability_enabled() -> bool {
     IS_OBSERVABILITY_ENABLED.get().copied().unwrap_or(false)
 }
 
-/// Get the global meter for observability
-pub fn global_meter() -> Meter {
-    let meter_name = OBSERVABILITY_METER_NAME.get().map(|s| s.as_str()).unwrap_or(APP_NAME);
-    opentelemetry::global::meter(meter_name)
-}
-
 /// Error type for global guard operations
 #[derive(Debug, thiserror::Error)]
 pub enum GlobalError {
@@ -75,18 +64,33 @@ pub enum GlobalError {
 ///
 /// # Example
 /// ```no_run
-/// use rustfs_obs::init_obs;
+/// # use rustfs_obs::init_obs;
 ///
 /// # #[tokio::main]
 /// # async fn main() {
-/// let guard = init_obs(None).await;
+/// #    let guard = init_obs(None).await;
 /// # }
 /// ```
 pub async fn init_obs(endpoint: Option<String>) -> OtelGuard {
     // Load the configuration file
     let config = AppConfig::new_with_endpoint(endpoint);
 
-    init_telemetry(&config.observability)
+    let otel_guard = init_telemetry(&config.observability);
+    // Server will be created per connection - this ensures isolation
+    tokio::spawn(async move {
+        // Record the PID-related metrics of the current process
+        let obs_result = SystemObserver::init_process_observer().await;
+        match obs_result {
+            Ok(_) => {
+                info!(target: "rustfs::obs::system::metrics","Process observer initialized successfully");
+            }
+            Err(e) => {
+                error!(target: "rustfs::obs::system::metrics","Failed to initialize process observer: {}", e);
+            }
+        }
+    });
+
+    otel_guard
 }
 
 /// Set the global guard for OpenTelemetry
@@ -99,14 +103,14 @@ pub async fn init_obs(endpoint: Option<String>) -> OtelGuard {
 /// * `Err(GuardError)` if setting fails
 ///
 /// # Example
-/// ```rust
-/// use rustfs_obs::{ init_obs, set_global_guard};
+/// ```no_run
+/// # use rustfs_obs::{ init_obs, set_global_guard};
 ///
-/// async fn init() -> Result<(), Box<dyn std::error::Error>> {
-///     let guard = init_obs(None).await;
-///     set_global_guard(guard)?;
-///     Ok(())
-/// }
+/// # async fn init() -> Result<(), Box<dyn std::error::Error>> {
+/// #    let guard = init_obs(None).await;
+/// #    set_global_guard(guard)?;
+/// #    Ok(())
+/// # }
 /// ```
 pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
     info!("Initializing global OpenTelemetry guard");
@@ -120,29 +124,20 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
 /// * `Err(GuardError)` if guard not initialized
 ///
 /// # Example
-/// ```rust
-/// use rustfs_obs::get_global_guard;
+/// ```no_run
+/// # use rustfs_obs::get_global_guard;
 ///
-/// async fn trace_operation() -> Result<(), Box<dyn std::error::Error>> {
-///     let guard = get_global_guard()?;
-///     let _lock = guard.lock().unwrap();
-///     // Perform traced operation
-///     Ok(())
-/// }
+/// # async fn trace_operation() -> Result<(), Box<dyn std::error::Error>> {
+/// #    let guard = get_global_guard()?;
+/// #    let _lock = guard.lock().unwrap();
+/// #    // Perform traced operation
+/// #    Ok(())
+/// # }
 /// ```
 pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
     GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
 }
 
-/// Try to get the global guard for OpenTelemetry
-///
-/// # Returns
-/// * `Some(Arc<Mutex<OtelGuard>>)` if guard exists
-/// * `None` if guard not initialized
-pub fn try_get_global_guard() -> Option<Arc<Mutex<OtelGuard>>> {
-    GLOBAL_GUARD.get().cloned()
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/obs/src/lib.rs

@@ -17,7 +17,7 @@
 //! provides tools for system and service monitoring
 //!
 //! ## feature mark
-//!
+//! - `default`: default monitoring function
 //! - `gpu`: gpu monitoring function
 //! - `full`: includes all functions
 //!
@@ -47,6 +47,6 @@ mod metrics;
 mod system;
 mod telemetry;
 
-pub use config::AppConfig;
+pub use config::{AppConfig, OtelConfig};
 pub use global::*;
 pub use system::SystemObserver;

crates/obs/src/system/attributes.rs

@@ -16,10 +16,10 @@ use crate::GlobalError;
 use opentelemetry::KeyValue;
 use sysinfo::{Pid, System};
 
-pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
-pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
-pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
-pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
+pub(crate) const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
+pub(crate) const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
+pub(crate) const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
+pub(crate) const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
 
 /// Struct to hold process attributes
 pub struct ProcessAttributes {

crates/obs/src/system/collector.rs

@@ -14,7 +14,6 @@
 
 use crate::GlobalError;
 use crate::system::attributes::ProcessAttributes;
-use crate::system::gpu::GpuCollector;
 use crate::system::metrics::{DIRECTION, INTERFACE, Metrics, STATUS};
 use opentelemetry::KeyValue;
 use std::time::SystemTime;
@@ -27,7 +26,8 @@ use tokio::time::{Duration, sleep};
 pub struct Collector {
     metrics: Metrics,
     attributes: ProcessAttributes,
-    gpu_collector: GpuCollector,
+    #[cfg(feature = "gpu")]
+    gpu_collector: crate::system::gpu::GpuCollector,
     pid: Pid,
     system: System,
     networks: Networks,
@@ -41,12 +41,14 @@ impl Collector {
         let attributes = ProcessAttributes::new(pid, &mut system)?;
         let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
         let metrics = Metrics::new(&meter);
-        let gpu_collector = GpuCollector::new(pid)?;
+        #[cfg(feature = "gpu")]
+        let gpu_collector = crate::system::gpu::GpuCollector::new(pid)?;
         let networks = Networks::new_with_refreshed_list();
 
         Ok(Collector {
             metrics,
             attributes,
+            #[cfg(feature = "gpu")]
             gpu_collector,
             pid,
             system,
@@ -163,6 +165,7 @@ impl Collector {
         );
 
         // GPU Metrics (Optional) Non-MacOS
+        #[cfg(feature = "gpu")]
         self.gpu_collector.collect(&self.metrics, &self.attributes)?;
 
         Ok(())

crates/obs/src/system/gpu.rs

@@ -12,29 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "gpu")]
 use crate::GlobalError;
-#[cfg(feature = "gpu")]
 use crate::system::attributes::ProcessAttributes;
-#[cfg(feature = "gpu")]
 use crate::system::metrics::Metrics;
-#[cfg(feature = "gpu")]
 use nvml_wrapper::Nvml;
-#[cfg(feature = "gpu")]
 use nvml_wrapper::enums::device::UsedGpuMemory;
-#[cfg(feature = "gpu")]
 use sysinfo::Pid;
-#[cfg(feature = "gpu")]
 use tracing::warn;
 
 /// `GpuCollector` is responsible for collecting GPU memory usage metrics.
-#[cfg(feature = "gpu")]
 pub struct GpuCollector {
     nvml: Nvml,
     pid: Pid,
 }
 
-#[cfg(feature = "gpu")]
 impl GpuCollector {
     pub fn new(pid: Pid) -> Result<Self, GlobalError> {
         let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
@@ -64,21 +55,3 @@ impl GpuCollector {
         Ok(())
     }
 }
-
-#[cfg(not(feature = "gpu"))]
-pub struct GpuCollector;
-
-#[cfg(not(feature = "gpu"))]
-impl GpuCollector {
-    pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
-        Ok(GpuCollector)
-    }
-
-    pub fn collect(
-        &self,
-        _metrics: &crate::system::metrics::Metrics,
-        _attributes: &crate::system::attributes::ProcessAttributes,
-    ) -> Result<(), crate::GlobalError> {
-        Ok(())
-    }
-}

crates/obs/src/system/metrics.rs

@@ -12,19 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
-pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
-pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
-pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
-pub const PROCESS_DISK_IO: &str = "process.disk.io";
-pub const PROCESS_NETWORK_IO: &str = "process.network.io";
-pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
-pub const PROCESS_STATUS: &str = "process.status";
+use opentelemetry::metrics::{Gauge, Meter};
+
+pub(crate) const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
+pub(crate) const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
+pub(crate) const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
+pub(crate) const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
+pub(crate) const PROCESS_DISK_IO: &str = "process.disk.io";
+pub(crate) const PROCESS_NETWORK_IO: &str = "process.network.io";
+pub(crate) const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
+pub(crate) const PROCESS_STATUS: &str = "process.status";
 #[cfg(feature = "gpu")]
 pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
-pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
-pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
-pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
+pub(crate) const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
+pub(crate) const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
+pub(crate) const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
 
 /// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
 /// It contains various metrics such as CPU usage, memory usage,
@@ -36,20 +38,20 @@ pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("i
 /// The `new` method initializes the metrics using the provided
 /// `opentelemetry::metrics::Meter`.
 pub struct Metrics {
-    pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
-    pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
-    pub memory_usage: opentelemetry::metrics::Gauge<i64>,
-    pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
-    pub disk_io: opentelemetry::metrics::Gauge<i64>,
-    pub network_io: opentelemetry::metrics::Gauge<i64>,
-    pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
-    pub process_status: opentelemetry::metrics::Gauge<i64>,
+    pub cpu_usage: Gauge<f64>,
+    pub cpu_utilization: Gauge<f64>,
+    pub memory_usage: Gauge<i64>,
+    pub memory_virtual: Gauge<i64>,
+    pub disk_io: Gauge<i64>,
+    pub network_io: Gauge<i64>,
+    pub network_io_per_interface: Gauge<i64>,
+    pub process_status: Gauge<i64>,
     #[cfg(feature = "gpu")]
-    pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
+    pub gpu_memory_usage: Gauge<u64>,
 }
 
 impl Metrics {
-    pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
+    pub fn new(meter: &Meter) -> Self {
         let cpu_usage = meter
             .f64_gauge(PROCESS_CPU_USAGE)
             .with_description("The percentage of CPU in use.")

crates/obs/src/system/mod.rs

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::GlobalError;
+use crate::{GlobalError, is_observability_enabled};
+use opentelemetry::global::meter;
 
-pub(crate) mod attributes;
+mod attributes;
 mod collector;
-pub(crate) mod gpu;
-pub(crate) mod metrics;
+#[cfg(feature = "gpu")]
+mod gpu;
+mod metrics;
 
 pub struct SystemObserver {}
 
@@ -25,10 +27,12 @@ impl SystemObserver {
     /// Initialize the indicator collector for the current process
     /// This function will create a new `Collector` instance and start collecting metrics.
     /// It will run indefinitely until the process is terminated.
-    pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
-        let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
-        let mut collector = collector::Collector::new(pid, meter, 30000)?;
-        collector.run().await
+    pub async fn init_process_observer() -> Result<(), GlobalError> {
+        if is_observability_enabled() {
+            let meter = meter("system");
+            return SystemObserver::init_process_observer_for_pid(meter, 30000).await;
+        }
+        Ok(())
     }
 
     /// Initialize the metric collector for the specified PID process

crates/obs/src/telemetry.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::config::OtelConfig;
-use crate::global::{IS_OBSERVABILITY_ENABLED, OBSERVABILITY_METER_NAME};
+use crate::global::IS_OBSERVABILITY_ENABLED;
 use flexi_logger::{
     Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode,
     WriteMode::{AsyncWith, BufferAndFlush},
@@ -36,7 +36,8 @@ use opentelemetry_semantic_conventions::{
     attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
 };
 use rustfs_config::{
-    APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
+    APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_LOCAL_LOGGING_ENABLED, ENVIRONMENT, METER_INTERVAL,
+    SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
     observability::{
         DEFAULT_OBS_ENVIRONMENT_PRODUCTION, DEFAULT_OBS_LOG_FLUSH_MS, DEFAULT_OBS_LOG_MESSAGE_CAPA, DEFAULT_OBS_LOG_POOL_CAPA,
         ENV_OBS_LOG_DIRECTORY,
@@ -294,7 +295,7 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
             tracing_subscriber::registry()
                 .with(filter)
                 .with(ErrorLayer::default())
-                .with(if config.local_logging_enabled.unwrap_or(false) {
+                .with(if config.local_logging_enabled.unwrap_or(DEFAULT_LOG_LOCAL_LOGGING_ENABLED) {
                     Some(fmt_layer)
                 } else {
                     None
@@ -312,7 +313,6 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
                     env::var("RUST_LOG").unwrap_or_else(|_| "Not set".to_string())
                 );
                 IS_OBSERVABILITY_ENABLED.set(true).ok();
-                OBSERVABILITY_METER_NAME.set(service_name.to_string()).ok();
             }
         }
         counter!("rustfs.start.total").increment(1);