feat: add TraceLayer for HTTP service and improve metrics (#361)

* improve code for opentelemetry and add system metrics

* feat: add TraceLayer for HTTP service and improve metrics

- Add TraceLayer to HTTP server for request tracing
- Implement system metrics for process monitoring
- Optimize init_telemetry method for better resource management
- Add graceful shutdown handling for telemetry components
- Fix GracefulShutdown ownership issues with Arc wrapper

* improve code for init_process_observer

* remove tomlfmt.toml

* Translation comment

* improve code for console CompressionLayer params

crates/obs/src/config.rs

@@ -178,7 +178,6 @@ pub struct AppConfig {
     pub logger: Option<LoggerConfig>,
 }
 
-// 为 AppConfig 实现 Default
 impl AppConfig {
     pub fn new() -> Self {
         Self {
@@ -189,6 +188,7 @@ impl AppConfig {
     }
 }
 
+// implement default for AppConfig
 impl Default for AppConfig {
     fn default() -> Self {
         Self::new()

crates/obs/src/global.rs

@@ -16,11 +16,27 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 
 /// Error type for global guard operations
 #[derive(Debug, thiserror::Error)]
-pub enum GuardError {
+pub enum GlobalError {
     #[error("Failed to set global guard: {0}")]
     SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
     #[error("Global guard not initialized")]
     NotInitialized,
+    #[error("Global system metrics err: {0}")]
+    MetricsError(String),
+    #[error("Failed to get current PID: {0}")]
+    PidError(String),
+    #[error("Process with PID {0} not found")]
+    ProcessNotFound(u32),
+    #[error("Failed to get physical core count")]
+    CoreCountError,
+    #[error("GPU initialization failed: {0}")]
+    GpuInitError(String),
+    #[error("GPU device not found: {0}")]
+    GpuDeviceError(String),
+    #[error("Failed to send log: {0}")]
+    SendFailed(&'static str),
+    #[error("Operation timed out: {0}")]
+    Timeout(&'static str),
 }
 
 /// Set the global guard for OpenTelemetry
@@ -43,9 +59,9 @@ pub enum GuardError {
 ///     Ok(())
 /// }
 /// ```
-pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
+pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
     info!("Initializing global OpenTelemetry guard");
-    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GuardError::SetError)
+    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
 }
 
 /// Get the global guard for OpenTelemetry
@@ -65,8 +81,8 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
 ///     Ok(())
 /// }
 /// ```
-pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GuardError> {
-    GLOBAL_GUARD.get().cloned().ok_or(GuardError::NotInitialized)
+pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
+    GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
 }
 
 /// Try to get the global guard for OpenTelemetry
@@ -84,6 +100,6 @@ mod tests {
     #[tokio::test]
     async fn test_get_uninitialized_guard() {
         let result = get_global_guard();
-        assert!(matches!(result, Err(GuardError::NotInitialized)));
+        assert!(matches!(result, Err(GlobalError::NotInitialized)));
     }
 }

crates/obs/src/lib.rs

@@ -1,23 +1,24 @@
-/// # obs
-///
-/// `obs` is a logging and observability library for Rust.
-/// It provides a simple and easy-to-use interface for logging and observability.
-/// It is built on top of the `log` crate and `opentelemetry` crate.
-///
-/// ## Features
-/// - Structured logging
-/// - Distributed tracing
-/// - Metrics collection
-/// - Log processing worker
-/// - Multiple sinks
-/// - Configuration-based setup
-/// - Telemetry guard
-/// - Global logger
-/// - Log levels
-/// - Log entry types
-/// - Log record
-/// - Object version
-/// - Local IP address
+//! # RustFS Observability
+//!
+//! provides tools for system and service monitoring
+//!
+//! ## feature mark
+//!
+//! - `file`: enable file logging enabled by default
+//! - `gpu`: gpu monitoring function
+//! - `kafka`: enable kafka metric output
+//! - `webhook`: enable webhook notifications
+//! - `full`: includes all functions
+//!
+//! to enable gpu monitoring add in cargo toml
+//!
+//! ```toml
+//! # using gpu monitoring
+//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
+//!
+//! # use all functions
+//! rustfs-obs = { version = "0.1.0", features = ["full"] }
+//! ```
 ///
 /// ## Usage
 ///
@@ -32,28 +33,34 @@ mod entry;
 mod global;
 mod logger;
 mod sink;
+mod system;
 mod telemetry;
 mod utils;
 mod worker;
+
+use crate::logger::InitLogStatus;
 pub use config::load_config;
-pub use config::{AppConfig, OtelConfig};
+#[cfg(feature = "file")]
+pub use config::FileSinkConfig;
+#[cfg(feature = "kafka")]
+pub use config::KafkaSinkConfig;
+#[cfg(feature = "webhook")]
+pub use config::WebhookSinkConfig;
+pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
 pub use entry::args::Args;
 pub use entry::audit::{ApiDetails, AuditLogEntry};
 pub use entry::base::BaseLogEntry;
 pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
 pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
-pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GuardError};
-pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
-pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
-pub use logger::{log_init_state, InitLogStatus};
-pub use logger::{LogError, Logger};
-pub use sink::Sink;
+pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
+pub use logger::Logger;
+pub use logger::{get_global_logger, init_global_logger, start_logger};
+pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
 use std::sync::Arc;
+pub use system::{init_process_observer, init_process_observer_for_pid};
 pub use telemetry::init_telemetry;
-pub use telemetry::{get_global_registry, metrics};
 use tokio::sync::Mutex;
-pub use utils::{get_local_ip, get_local_ip_with_default};
-pub use worker::start_worker;
+use tracing::{error, info};
 
 /// Initialize the observability module
 ///
@@ -74,6 +81,19 @@ pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::Otel
     let guard = init_telemetry(&config.observability);
     let sinks = sink::create_sinks(&config).await;
     let logger = init_global_logger(&config, sinks).await;
+    let obs_config = config.observability.clone();
+    tokio::spawn(async move {
+        let result = InitLogStatus::init_start_log(&obs_config).await;
+        match result {
+            Ok(_) => {
+                info!("Logger initialized successfully");
+            }
+            Err(e) => {
+                error!("Failed to initialize logger: {}", e);
+            }
+        }
+    });
+
     (logger, guard)
 }
 

crates/obs/src/logger.rs

@@ -1,5 +1,6 @@
 use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
-use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, OtelConfig, ServerLogEntry, Sink, UnifiedLogEntry};
+use crate::sink::Sink;
+use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
 use std::sync::Arc;
 use std::time::SystemTime;
 use tokio::sync::mpsc::{self, Receiver, Sender};
@@ -43,26 +44,26 @@ impl Logger {
 
     /// Log a server entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
-    pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), LogError> {
+    pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Server(entry)).await
     }
 
     /// Log an audit entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
-    pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), LogError> {
+    pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
     }
 
     /// Log a console entry
     #[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
-    pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), LogError> {
+    pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
         self.log_entry(UnifiedLogEntry::Console(entry)).await
     }
 
     /// Asynchronous logging of unified log entries
     #[tracing::instrument(skip(self), fields(log_source = "logger"))]
     #[tracing::instrument(level = "error", skip_all)]
-    pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), LogError> {
+    pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
         // Extract information for tracing based on entry type
         match &entry {
             UnifiedLogEntry::Server(server) => {
@@ -123,11 +124,11 @@ impl Logger {
                 tracing::warn!("Log queue full, applying backpressure");
                 match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
                     Ok(Ok(_)) => Ok(()),
-                    Ok(Err(_)) => Err(LogError::SendFailed("Channel closed")),
-                    Err(_) => Err(LogError::Timeout("Queue backpressure timeout")),
+                    Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
+                    Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
                 }
             }
-            Err(mpsc::error::TrySendError::Closed(_)) => Err(LogError::SendFailed("Logger channel closed")),
+            Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
         }
     }
 
@@ -160,7 +161,7 @@ impl Logger {
         request_id: Option<String>,
         user_id: Option<String>,
         fields: Vec<(String, String)>,
-    ) -> Result<(), LogError> {
+    ) -> Result<(), GlobalError> {
         let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
 
         let server_entry = ServerLogEntry::new(level, source.to_string())
@@ -190,7 +191,7 @@ impl Logger {
     ///   let _ = logger.write("This is an information message", "example", Level::INFO).await;
     /// }
     /// ```
-    pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), LogError> {
+    pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
         self.write_with_context(message, source, level, None, None, Vec::new()).await
     }
 
@@ -208,31 +209,12 @@ impl Logger {
     ///  let _ = logger.shutdown().await;
     /// }
     /// ```
-    pub async fn shutdown(self) -> Result<(), LogError> {
+    pub async fn shutdown(self) -> Result<(), GlobalError> {
         drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
         Ok(())
     }
 }
 
-/// Log error type
-/// This enum defines the error types that can occur when logging.
-/// It is used to provide more detailed error information.
-/// # Example
-/// ```
-/// use rustfs_obs::LogError;
-/// use thiserror::Error;
-///
-/// LogError::SendFailed("Failed to send log");
-/// LogError::Timeout("Operation timed out");
-/// ```
-#[derive(Debug, thiserror::Error)]
-pub enum LogError {
-    #[error("Failed to send log: {0}")]
-    SendFailed(&'static str),
-    #[error("Operation timed out: {0}")]
-    Timeout(&'static str),
-}
-
 /// Start the log module
 /// This function starts the log module.
 /// It initializes the logger and starts the worker to process logs.
@@ -297,48 +279,6 @@ pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
     GLOBAL_LOGGER.get().expect("Logger not initialized")
 }
 
-/// Get the global logger instance with a lock
-/// This function returns a reference to the global logger instance with a lock.
-/// It is used to ensure that the logger is thread-safe.
-///
-/// # Returns
-/// A reference to the global logger instance with a lock
-///
-/// # Example
-/// ```
-/// use rustfs_obs::locked_logger;
-///
-/// async fn example() {
-///     let logger = locked_logger().await;
-/// }
-/// ```
-pub async fn locked_logger() -> tokio::sync::MutexGuard<'static, Logger> {
-    get_global_logger().lock().await
-}
-
-/// Initialize with default empty logger if needed (optional)
-/// This function initializes the logger with a default empty logger if needed.
-/// It is used to ensure that the logger is initialized before logging.
-///
-/// # Returns
-/// A reference to the global logger instance
-///
-/// # Example
-/// ```
-/// use rustfs_obs::ensure_logger_initialized;
-///
-/// let logger = ensure_logger_initialized();
-/// ```
-pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
-    if GLOBAL_LOGGER.get().is_none() {
-        let config = AppConfig::default();
-        let sinks = vec![];
-        let logger = Arc::new(Mutex::new(start_logger(&config, sinks)));
-        let _ = GLOBAL_LOGGER.set(logger);
-    }
-    GLOBAL_LOGGER.get().unwrap()
-}
-
 /// Log information
 /// This function logs information messages.
 ///
@@ -357,7 +297,7 @@ pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
 ///    let _ = log_info("This is an information message", "example").await;
 /// }
 /// ```
-pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::INFO).await
 }
 
@@ -375,7 +315,7 @@ pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
 /// async fn example() {
 ///     let _ = log_error("This is an error message", "example").await;
 /// }
-pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::ERROR).await
 }
 
@@ -395,7 +335,7 @@ pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
 ///     let _ = log_warn("This is a warning message", "example").await;
 /// }
 /// ```
-pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::WARN).await
 }
 
@@ -415,7 +355,7 @@ pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
 ///     let _ = log_debug("This is a debug message", "example").await;
 /// }
 /// ```
-pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::DEBUG).await
 }
 
@@ -436,7 +376,7 @@ pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
 ///    let _ = log_trace("This is a trace message", "example").await;
 /// }
 /// ```
-pub async fn log_trace(message: &str, source: &str) -> Result<(), LogError> {
+pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
     get_global_logger().lock().await.write(message, source, Level::TRACE).await
 }
 
@@ -467,7 +407,7 @@ pub async fn log_with_context(
     request_id: Option<String>,
     user_id: Option<String>,
     fields: Vec<(String, String)>,
-) -> Result<(), LogError> {
+) -> Result<(), GlobalError> {
     get_global_logger()
         .lock()
         .await
@@ -477,7 +417,7 @@ pub async fn log_with_context(
 
 /// Log initialization status
 #[derive(Debug)]
-pub struct InitLogStatus {
+pub(crate) struct InitLogStatus {
     pub timestamp: SystemTime,
     pub service_name: String,
     pub version: String,
@@ -508,14 +448,14 @@ impl InitLogStatus {
         }
     }
 
-    pub async fn init_start_log(config: &OtelConfig) -> Result<(), LogError> {
+    pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
         let status = Self::new_config(config);
         log_init_state(Some(status)).await
     }
 }
 
 /// Log initialization details during system startup
-pub async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), LogError> {
+async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
     let status = status.unwrap_or_default();
 
     let base_entry = BaseLogEntry::new()

crates/obs/src/system/attributes.rs

@@ -0,0 +1,44 @@
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use sysinfo::{Pid, System};
+
+pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
+pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
+pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
+pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
+
+/// Struct to hold process attributes
+pub struct ProcessAttributes {
+    pub attributes: Vec<KeyValue>,
+}
+
+impl ProcessAttributes {
+    /// Creates a new instance of `ProcessAttributes` for the given PID.
+    pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
+        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+        let process = system
+            .process(pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
+
+        let attributes = vec![
+            KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
+            KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
+            KeyValue::new(
+                PROCESS_EXECUTABLE_PATH,
+                process
+                    .exe()
+                    .map(|path| path.to_string_lossy().into_owned())
+                    .unwrap_or_default(),
+            ),
+            KeyValue::new(
+                PROCESS_COMMAND,
+                process
+                    .cmd()
+                    .iter()
+                    .fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
+            ),
+        ];
+
+        Ok(ProcessAttributes { attributes })
+    }
+}

crates/obs/src/system/collector.rs

@@ -0,0 +1,156 @@
+use crate::system::attributes::ProcessAttributes;
+use crate::system::gpu::GpuCollector;
+use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use std::time::SystemTime;
+use sysinfo::{Networks, Pid, ProcessStatus, System};
+use tokio::time::{sleep, Duration};
+
+/// Collector is responsible for collecting system metrics and attributes.
+/// It uses the sysinfo crate to gather information about the system and processes.
+/// It also uses OpenTelemetry to record metrics.
+pub struct Collector {
+    metrics: Metrics,
+    attributes: ProcessAttributes,
+    gpu_collector: GpuCollector,
+    pid: Pid,
+    system: System,
+    networks: Networks,
+    core_count: usize,
+    interval_ms: u64,
+}
+
+impl Collector {
+    pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
+        let mut system = System::new_all();
+        let attributes = ProcessAttributes::new(pid, &mut system)?;
+        let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
+        let metrics = Metrics::new(&meter);
+        let gpu_collector = GpuCollector::new(pid)?;
+        let networks = Networks::new_with_refreshed_list();
+
+        Ok(Collector {
+            metrics,
+            attributes,
+            gpu_collector,
+            pid,
+            system,
+            networks,
+            core_count,
+            interval_ms,
+        })
+    }
+
+    pub async fn run(&mut self) -> Result<(), GlobalError> {
+        loop {
+            self.collect()?;
+            tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
+            sleep(Duration::from_millis(self.interval_ms)).await;
+        }
+    }
+
+    fn collect(&mut self) -> Result<(), GlobalError> {
+        self.system
+            .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
+
+        // refresh the network interface list and statistics
+        self.networks.refresh(false);
+
+        let process = self
+            .system
+            .process(self.pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
+
+        // CPU metrics
+        let cpu_usage = process.cpu_usage();
+        self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
+        self.metrics
+            .cpu_utilization
+            .record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
+
+        // Memory metrics
+        self.metrics
+            .memory_usage
+            .record(process.memory() as i64, &self.attributes.attributes);
+        self.metrics
+            .memory_virtual
+            .record(process.virtual_memory() as i64, &self.attributes.attributes);
+
+        // Disk I/O metrics
+        let disk_io = process.disk_usage();
+        self.metrics.disk_io.record(
+            disk_io.read_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
+        );
+        self.metrics.disk_io.record(
+            disk_io.written_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
+        );
+
+        // Network I/O indicators (corresponding to /system/network/internode)
+        let mut total_received: i64 = 0;
+        let mut total_transmitted: i64 = 0;
+
+        // statistics by interface
+        for (interface_name, data) in self.networks.iter() {
+            total_received += data.total_received() as i64;
+            total_transmitted += data.total_transmitted() as i64;
+
+            let received = data.received() as i64;
+            let transmitted = data.transmitted() as i64;
+            self.metrics.network_io_per_interface.record(
+                received,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "received"),
+                    ],
+                ]
+                .concat(),
+            );
+            self.metrics.network_io_per_interface.record(
+                transmitted,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "transmitted"),
+                    ],
+                ]
+                .concat(),
+            );
+        }
+        // global statistics
+        self.metrics.network_io.record(
+            total_received,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
+        );
+        self.metrics.network_io.record(
+            total_transmitted,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
+        );
+
+        // Process status indicator (corresponding to /system/process)
+        let status_value = match process.status() {
+            ProcessStatus::Run => 0,
+            ProcessStatus::Sleep => 1,
+            ProcessStatus::Zombie => 2,
+            _ => 3, // other status
+        };
+        self.metrics.process_status.record(
+            status_value,
+            &[
+                &self.attributes.attributes[..],
+                &[KeyValue::new(STATUS, format!("{:?}", process.status()))],
+            ]
+            .concat(),
+        );
+
+        // GPU Metrics (Optional) Non-MacOS
+        self.gpu_collector.collect(&self.metrics, &self.attributes)?;
+
+        Ok(())
+    }
+}

crates/obs/src/system/gpu.rs

@@ -0,0 +1,70 @@
+#[cfg(feature = "gpu")]
+use crate::system::attributes::ProcessAttributes;
+#[cfg(feature = "gpu")]
+use crate::system::metrics::Metrics;
+#[cfg(feature = "gpu")]
+use crate::GlobalError;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::enums::device::UsedGpuMemory;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::Nvml;
+#[cfg(feature = "gpu")]
+use sysinfo::Pid;
+#[cfg(feature = "gpu")]
+use tracing::warn;
+
+/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
+#[cfg(feature = "gpu")]
+pub struct GpuCollector {
+    nvml: Nvml,
+    pid: Pid,
+}
+
+#[cfg(feature = "gpu")]
+impl GpuCollector {
+    pub fn new(pid: Pid) -> Result<Self, GlobalError> {
+        let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
+        Ok(GpuCollector { nvml, pid })
+    }
+
+    pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
+        if let Ok(device) = self.nvml.device_by_index(0) {
+            if let Ok(gpu_stats) = device.running_compute_processes() {
+                for stat in gpu_stats.iter() {
+                    if stat.pid == self.pid.as_u32() {
+                        let memory_used = match stat.used_gpu_memory {
+                            UsedGpuMemory::Used(bytes) => bytes,
+                            UsedGpuMemory::Unavailable => 0,
+                        };
+                        metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
+                        return Ok(());
+                    }
+                }
+            } else {
+                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
+            }
+        } else {
+            return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
+        }
+        metrics.gpu_memory_usage.record(0, &attributes.attributes);
+        Ok(())
+    }
+}
+
+#[cfg(not(feature = "gpu"))]
+pub struct GpuCollector;
+
+#[cfg(not(feature = "gpu"))]
+impl GpuCollector {
+    pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
+        Ok(GpuCollector)
+    }
+
+    pub fn collect(
+        &self,
+        _metrics: &crate::system::metrics::Metrics,
+        _attributes: &crate::system::attributes::ProcessAttributes,
+    ) -> Result<(), crate::GlobalError> {
+        Ok(())
+    }
+}

crates/obs/src/system/metrics.rs

@@ -0,0 +1,100 @@
+pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
+pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
+pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
+pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
+pub const PROCESS_DISK_IO: &str = "process.disk.io";
+pub const PROCESS_NETWORK_IO: &str = "process.network.io";
+pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
+pub const PROCESS_STATUS: &str = "process.status";
+#[cfg(feature = "gpu")]
+pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
+pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
+pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
+pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
+
+/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
+/// It contains various metrics such as CPU usage, memory usage,
+/// disk I/O, network I/O, and process status.
+///
+/// The `Metrics` struct is designed to be used with OpenTelemetry's
+/// metrics API to record and export these metrics.
+///
+/// The `new` method initializes the metrics using the provided
+/// `opentelemetry::metrics::Meter`.
+pub struct Metrics {
+    pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
+    pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
+    pub memory_usage: opentelemetry::metrics::Gauge<i64>,
+    pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
+    pub disk_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
+    pub process_status: opentelemetry::metrics::Gauge<i64>,
+    #[cfg(feature = "gpu")]
+    pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
+}
+
+impl Metrics {
+    pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
+        let cpu_usage = meter
+            .f64_gauge(PROCESS_CPU_USAGE)
+            .with_description("The percentage of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let cpu_utilization = meter
+            .f64_gauge(PROCESS_CPU_UTILIZATION)
+            .with_description("The amount of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let memory_usage = meter
+            .i64_gauge(PROCESS_MEMORY_USAGE)
+            .with_description("The amount of physical memory in use.")
+            .with_unit("byte")
+            .build();
+        let memory_virtual = meter
+            .i64_gauge(PROCESS_MEMORY_VIRTUAL)
+            .with_description("The amount of committed virtual memory.")
+            .with_unit("byte")
+            .build();
+        let disk_io = meter
+            .i64_gauge(PROCESS_DISK_IO)
+            .with_description("Disk bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io = meter
+            .i64_gauge(PROCESS_NETWORK_IO)
+            .with_description("Network bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io_per_interface = meter
+            .i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
+            .with_description("Network bytes transferred (per interface).")
+            .with_unit("byte")
+            .build();
+
+        let process_status = meter
+            .i64_gauge(PROCESS_STATUS)
+            .with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
+            .build();
+
+        #[cfg(feature = "gpu")]
+        let gpu_memory_usage = meter
+            .u64_gauge(PROCESS_GPU_MEMORY_USAGE)
+            .with_description("The amount of physical GPU memory in use.")
+            .with_unit("byte")
+            .build();
+
+        Metrics {
+            cpu_usage,
+            cpu_utilization,
+            memory_usage,
+            memory_virtual,
+            disk_io,
+            network_io,
+            network_io_per_interface,
+            process_status,
+            #[cfg(feature = "gpu")]
+            gpu_memory_usage,
+        }
+    }
+}

crates/obs/src/system/mod.rs

@@ -0,0 +1,24 @@
+use crate::GlobalError;
+
+pub(crate) mod attributes;
+mod collector;
+pub(crate) mod gpu;
+pub(crate) mod metrics;
+
+/// Initialize the indicator collector for the current process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
+    let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}
+
+/// Initialize the metric collector for the specified PID process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
+    let pid = sysinfo::Pid::from_u32(pid);
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}

crates/obs/src/telemetry.rs

@@ -1,5 +1,6 @@
 use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
-use crate::{get_local_ip_with_default, OtelConfig};
+use crate::utils::get_local_ip_with_default;
+use crate::OtelConfig;
 use opentelemetry::trace::TracerProvider;
 use opentelemetry::{global, KeyValue};
 use opentelemetry_appender_tracing::layer;
@@ -14,11 +15,10 @@ use opentelemetry_semantic_conventions::{
     attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
     SCHEMA_URL,
 };
-use prometheus::{Encoder, Registry, TextEncoder};
+use smallvec::SmallVec;
+use std::borrow::Cow;
 use std::io::IsTerminal;
-use std::sync::Arc;
-use tokio::sync::{Mutex, OnceCell};
-use tracing::{info, warn};
+use tracing::info;
 use tracing_error::ErrorLayer;
 use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
@@ -67,66 +67,20 @@ impl Drop for OtelGuard {
     }
 }
 
-/// Global registry for Prometheus metrics
-static GLOBAL_REGISTRY: OnceCell<Arc<Mutex<Registry>>> = OnceCell::const_new();
-
-/// Get the global registry instance
-/// This function returns a reference to the global registry instance.
-///
-/// # Returns
-/// A reference to the global registry instance
-///
-/// # Example
-/// ```
-/// use rustfs_obs::get_global_registry;
-///
-/// let registry = get_global_registry();
-/// ```
-pub fn get_global_registry() -> Arc<Mutex<Registry>> {
-    GLOBAL_REGISTRY.get().unwrap().clone()
-}
-
-/// Prometheus metric endpoints
-/// This function returns a string containing the Prometheus metrics.
-/// The metrics are collected from the global registry.
-/// The function is used to expose the metrics via an HTTP endpoint.
-///
-/// # Returns
-/// A string containing the Prometheus metrics
-///
-/// # Example
-/// ```
-/// use rustfs_obs::metrics;
-///
-/// async fn main() {
-///    let metrics = metrics().await;
-///   println!("{}", metrics);
-/// }
-/// ```
-pub async fn metrics() -> String {
-    let encoder = TextEncoder::new();
-    // Get a reference to the registry for reading metrics
-    let registry = get_global_registry().lock().await.to_owned();
-    let metric_families = registry.gather();
-    if metric_families.is_empty() {
-        warn!("No metrics available in Prometheus registry");
-    } else {
-        info!("Metrics collected: {} families", metric_families.len());
-    }
-    let mut buffer = Vec::new();
-    encoder.encode(&metric_families, &mut buffer).unwrap();
-    String::from_utf8(buffer).unwrap_or_else(|_| "Error encoding metrics".to_string())
-}
-
 /// create OpenTelemetry Resource
 fn resource(config: &OtelConfig) -> Resource {
-    let config = config.clone();
     Resource::builder()
-        .with_service_name(config.service_name.unwrap_or(SERVICE_NAME.to_string()))
+        .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
         .with_schema_url(
             [
-                KeyValue::new(OTEL_SERVICE_VERSION, config.service_version.unwrap_or(SERVICE_VERSION.to_string())),
-                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, config.environment.unwrap_or(ENVIRONMENT.to_string())),
+                KeyValue::new(
+                    OTEL_SERVICE_VERSION,
+                    Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
+                ),
+                KeyValue::new(
+                    DEPLOYMENT_ENVIRONMENT_NAME,
+                    Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
+                ),
                 KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
             ],
             SCHEMA_URL,
@@ -141,167 +95,169 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
         .build()
 }
 
-/// Initialize Meter Provider
-fn init_meter_provider(config: &OtelConfig) -> SdkMeterProvider {
-    let mut builder = MeterProviderBuilder::default().with_resource(resource(config));
-    // If endpoint is empty, use stdout output
-    if config.endpoint.is_empty() {
-        builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-    } else {
-        // If endpoint is not empty, use otlp output
-        let exporter = opentelemetry_otlp::MetricExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
-            .build()
-            .unwrap();
-        builder = builder.with_reader(
-            PeriodicReader::builder(exporter)
-                .with_interval(std::time::Duration::from_secs(config.meter_interval.unwrap_or(METER_INTERVAL)))
-                .build(),
-        );
-        // If use_stdout is true, output to stdout at the same time
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-        }
-    }
-    let registry = Registry::new();
-    // Set global registry
-    GLOBAL_REGISTRY.set(Arc::new(Mutex::new(registry.clone()))).unwrap();
-    // Create Prometheus exporter
-    let prometheus_exporter = opentelemetry_prometheus::exporter().with_registry(registry).build().unwrap();
-    // Build meter provider
-    let meter_provider = builder.with_reader(prometheus_exporter).build();
-    global::set_meter_provider(meter_provider.clone());
-    meter_provider
-}
-
-/// Initialize Tracer Provider
-fn init_tracer_provider(config: &OtelConfig) -> SdkTracerProvider {
-    let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
-    let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
-        Sampler::TraceIdRatioBased(sample_ratio)
-    } else {
-        Sampler::AlwaysOn
-    };
-    let builder = SdkTracerProvider::builder()
-        .with_sampler(sampler)
-        .with_id_generator(RandomIdGenerator::default())
-        .with_resource(resource(config));
-
-    let tracer_provider = if config.endpoint.is_empty() {
-        builder
-            .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-            .build()
-    } else {
-        let exporter = opentelemetry_otlp::SpanExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .build()
-            .unwrap();
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder
-                .with_batch_exporter(exporter)
-                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-        } else {
-            builder.with_batch_exporter(exporter)
-        }
-        .build()
-    };
-
-    global::set_tracer_provider(tracer_provider.clone());
-    tracer_provider
-}
-
 /// Initialize Telemetry
 pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
-    let tracer_provider = init_tracer_provider(config);
-    let meter_provider = init_meter_provider(config);
+    // avoid repeated access to configuration fields
+    let endpoint = &config.endpoint;
+    let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
+    let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
+    let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
+    let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
 
-    // Initialize logger provider based on configuration
-    let logger_provider = {
-        let mut builder = SdkLoggerProvider::builder().with_resource(resource(config));
+    // Pre-create resource objects to avoid repeated construction
+    let res = resource(config);
 
-        if config.endpoint.is_empty() {
-            // Use stdout exporter when no endpoint is configured
-            builder = builder.with_simple_exporter(opentelemetry_stdout::LogExporter::default());
+    // initialize tracer provider
+    let tracer_provider = {
+        let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
+        let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
+            Sampler::TraceIdRatioBased(sample_ratio)
+        } else {
+            Sampler::AlwaysOn
+        };
+
+        let builder = SdkTracerProvider::builder()
+            .with_sampler(sampler)
+            .with_id_generator(RandomIdGenerator::default())
+            .with_resource(res.clone());
+
+        let tracer_provider = if endpoint.is_empty() {
+            builder
+                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+                .build()
+        } else {
+            let exporter = opentelemetry_otlp::SpanExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .build()
+                .unwrap();
+
+            let builder = if use_stdout {
+                builder
+                    .with_batch_exporter(exporter)
+                    .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+            } else {
+                builder.with_batch_exporter(exporter)
+            };
+
+            builder.build()
+        };
+
+        global::set_tracer_provider(tracer_provider.clone());
+        tracer_provider
+    };
+
+    // initialize meter provider
+    let meter_provider = {
+        let mut builder = MeterProviderBuilder::default().with_resource(res.clone());
+
+        if endpoint.is_empty() {
+            builder = builder.with_reader(create_periodic_reader(meter_interval));
+        } else {
+            let exporter = opentelemetry_otlp::MetricExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+                .build()
+                .unwrap();
+
+            builder = builder.with_reader(
+                PeriodicReader::builder(exporter)
+                    .with_interval(std::time::Duration::from_secs(meter_interval))
+                    .build(),
+            );
+
+            if use_stdout {
+                builder = builder.with_reader(create_periodic_reader(meter_interval));
+            }
+        }
+
+        let meter_provider = builder.build();
+        global::set_meter_provider(meter_provider.clone());
+        meter_provider
+    };
+
+    // initialize logger provider
+    let logger_provider = {
+        let mut builder = SdkLoggerProvider::builder().with_resource(res);
+
+        if endpoint.is_empty() {
+            builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
         } else {
-            // Configure OTLP exporter when endpoint is provided
             let exporter = opentelemetry_otlp::LogExporter::builder()
                 .with_tonic()
-                .with_endpoint(&config.endpoint)
+                .with_endpoint(endpoint)
                 .build()
                 .unwrap();
 
             builder = builder.with_batch_exporter(exporter);
 
-            // Add stdout exporter if requested
-            if config.use_stdout.unwrap_or(USE_STDOUT) {
+            if use_stdout {
                 builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
             }
         }
 
         builder.build()
     };
-    let config = config.clone();
-    let logger_level = config.logger_level.unwrap_or(LOGGER_LEVEL.to_string());
-    let logger_level = logger_level.as_str();
-    // Setup OpenTelemetryTracingBridge layer
-    let otel_layer = {
-        // Filter to prevent infinite telemetry loops
-        // This blocks events from OpenTelemetry and its dependent libraries (tonic, reqwest, etc.)
-        // from being sent back to OpenTelemetry itself
-        let filter_otel = match logger_level {
-            "trace" | "debug" => {
-                info!("OpenTelemetry tracing initialized with level: {}", logger_level);
-                let mut filter = EnvFilter::new(logger_level);
-                for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
-                    filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+
+    // configuring tracing
+    {
+        // optimize filter configuration
+        let otel_layer = {
+            let filter_otel = match logger_level {
+                "trace" | "debug" => {
+                    info!("OpenTelemetry tracing initialized with level: {}", logger_level);
+                    EnvFilter::new(logger_level)
                 }
-                filter
-            }
-            _ => {
-                let mut filter = EnvFilter::new(logger_level);
-                for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
-                    filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+                _ => {
+                    let mut filter = EnvFilter::new(logger_level);
+
+                    // use smallvec to avoid heap allocation
+                    let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
+
+                    for directive in directives {
+                        filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+                    }
+                    filter
                 }
-                filter
-            }
+            };
+            layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
         };
-        layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
-    };
 
-    let tracer = tracer_provider.tracer(config.service_name.unwrap_or(SERVICE_NAME.to_string()));
-    let registry = tracing_subscriber::registry()
-        .with(switch_level(logger_level))
-        .with(OpenTelemetryLayer::new(tracer))
-        .with(MetricsLayer::new(meter_provider.clone()))
-        .with(otel_layer)
-        .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
+        let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
 
-    // Configure formatting layer
-    let enable_color = std::io::stdout().is_terminal();
-    let fmt_layer = tracing_subscriber::fmt::layer()
-        .with_ansi(enable_color)
-        .with_thread_names(true)
-        .with_file(true)
-        .with_line_number(true);
+        // Configure registry to avoid repeated calls to filter methods
+        let level_filter = switch_level(logger_level);
+        let registry = tracing_subscriber::registry()
+            .with(level_filter)
+            .with(OpenTelemetryLayer::new(tracer))
+            .with(MetricsLayer::new(meter_provider.clone()))
+            .with(otel_layer)
+            .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
 
-    // Creating a formatting layer with explicit type to avoid type mismatches
-    let fmt_layer = fmt_layer.with_filter(
-        EnvFilter::new(logger_level).add_directive(
-            format!("opentelemetry={}", if config.endpoint.is_empty() { logger_level } else { "off" })
-                .parse()
-                .unwrap(),
-        ),
-    );
+        // configure the formatting layer
+        let enable_color = std::io::stdout().is_terminal();
+        let fmt_layer = tracing_subscriber::fmt::layer()
+            .with_ansi(enable_color)
+            .with_thread_names(true)
+            .with_file(true)
+            .with_line_number(true)
+            .with_filter(
+                EnvFilter::new(logger_level).add_directive(
+                    format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
+                        .parse()
+                        .unwrap(),
+                ),
+            );
 
-    registry.with(ErrorLayer::default()).with(fmt_layer).init();
-    if !config.endpoint.is_empty() {
-        info!(
-            "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
-            config.endpoint, logger_level
-        );
+        registry.with(ErrorLayer::default()).with(fmt_layer).init();
+
+        if !endpoint.is_empty() {
+            info!(
+                "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
+                endpoint, logger_level
+            );
+        }
     }
 
     OtelGuard {
@@ -313,12 +269,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 
 /// Switch log level
 fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
+    use tracing_subscriber::filter::LevelFilter;
     match logger_level {
-        "error" => tracing_subscriber::filter::LevelFilter::ERROR,
-        "warn" => tracing_subscriber::filter::LevelFilter::WARN,
-        "info" => tracing_subscriber::filter::LevelFilter::INFO,
-        "debug" => tracing_subscriber::filter::LevelFilter::DEBUG,
-        "trace" => tracing_subscriber::filter::LevelFilter::TRACE,
-        _ => tracing_subscriber::filter::LevelFilter::OFF,
+        "error" => LevelFilter::ERROR,
+        "warn" => LevelFilter::WARN,
+        "info" => LevelFilter::INFO,
+        "debug" => LevelFilter::DEBUG,
+        "trace" => LevelFilter::TRACE,
+        _ => LevelFilter::OFF,
     }
 }