improve code for opentelemetry and add system metrics

crates/obs/Cargo.toml

@@ -11,32 +11,35 @@ workspace = true
 
 [features]
 default = ["file"]
+file = []
+gpu = ["dep:nvml-wrapper"]
 kafka = ["dep:rdkafka"]
 webhook = ["dep:reqwest"]
-file = []
+full = ["file", "gpu", "kafka", "webhook"]
 
 [dependencies]
 async-trait = { workspace = true }
 chrono = { workspace = true }
 config = { workspace = true }
+nvml-wrapper = { workspace = true, optional = true }
 opentelemetry = { workspace = true }
 opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
 opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 opentelemetry-stdout = { workspace = true }
 opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
-opentelemetry-prometheus = { workspace = true }
 opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
-prometheus = { workspace = true }
 serde = { workspace = true }
+smallvec = { workspace = true, features = ["serde"] }
 tracing = { workspace = true, features = ["std", "attributes"] }
 tracing-core = { workspace = true }
 tracing-error = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
-tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
 rdkafka = { workspace = true, features = ["tokio"], optional = true }
 reqwest = { workspace = true, optional = true, default-features = false }
 serde_json = { workspace = true }
+sysinfo = { workspace = true }
 thiserror = { workspace = true }
 local-ip-address = { workspace = true }
 

crates/obs/src/global.rs

@@ -16,11 +16,24 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
 
 /// Error type for global guard operations
 #[derive(Debug, thiserror::Error)]
-pub enum GuardError {
+pub enum GlobalError {
     #[error("Failed to set global guard: {0}")]
     SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
     #[error("Global guard not initialized")]
     NotInitialized,
+    #[error("Global system metrics err: {0}")]
+    MetricsError(String),
+    #[error("Failed to get process ID: {0}")]
+    GetPidError(String),
+    #[error("Type conversion error: {0}")]
+    ConversionError(String),
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+    #[error("System metrics error: {0}")]
+    SystemMetricsError(String),
+    #[cfg(feature = "gpu")]
+    #[error("GPU metrics error: {0}")]
+    GpuMetricsError(String),
 }
 
 /// Set the global guard for OpenTelemetry
@@ -43,9 +56,9 @@ pub enum GuardError {
 ///     Ok(())
 /// }
 /// ```
-pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
+pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
     info!("Initializing global OpenTelemetry guard");
-    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GuardError::SetError)
+    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
 }
 
 /// Get the global guard for OpenTelemetry
@@ -65,8 +78,8 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
 ///     Ok(())
 /// }
 /// ```
-pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GuardError> {
-    GLOBAL_GUARD.get().cloned().ok_or(GuardError::NotInitialized)
+pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
+    GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
 }
 
 /// Try to get the global guard for OpenTelemetry
@@ -84,6 +97,6 @@ mod tests {
     #[tokio::test]
     async fn test_get_uninitialized_guard() {
         let result = get_global_guard();
-        assert!(matches!(result, Err(GuardError::NotInitialized)));
+        assert!(matches!(result, Err(GlobalError::NotInitialized)));
     }
 }

crates/obs/src/lib.rs

@@ -1,23 +1,24 @@
-/// # obs
-///
-/// `obs` is a logging and observability library for Rust.
-/// It provides a simple and easy-to-use interface for logging and observability.
-/// It is built on top of the `log` crate and `opentelemetry` crate.
-///
-/// ## Features
-/// - Structured logging
-/// - Distributed tracing
-/// - Metrics collection
-/// - Log processing worker
-/// - Multiple sinks
-/// - Configuration-based setup
-/// - Telemetry guard
-/// - Global logger
-/// - Log levels
-/// - Log entry types
-/// - Log record
-/// - Object version
-/// - Local IP address
+//! # RustFS Observability
+//!
+//! provides tools for system and service monitoring
+//!
+//! ## feature mark
+//!
+//! - `file`: enable file logging enabled by default
+//! - `gpu`: gpu monitoring function
+//! - `kafka`: enable kafka metric output
+//! - `webhook`: enable webhook notifications
+//! - `full`: includes all functions
+//!
+//! to enable gpu monitoring add in cargo toml
+//!
+//! ```toml
+//! # using gpu monitoring
+//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
+//!
+//! # use all functions
+//! rustfs-obs = { version = "0.1.0", features = ["full"] }
+//! ```
 ///
 /// ## Usage
 ///
@@ -31,10 +32,15 @@ mod config;
 mod entry;
 mod global;
 mod logger;
+mod metrics;
 mod sink;
 mod telemetry;
 mod utils;
 mod worker;
+
+#[cfg(feature = "gpu")]
+pub use crate::metrics::init_gpu_metrics;
+
 pub use config::load_config;
 pub use config::{AppConfig, OtelConfig};
 pub use entry::args::Args;
@@ -42,15 +48,15 @@ pub use entry::audit::{ApiDetails, AuditLogEntry};
 pub use entry::base::BaseLogEntry;
 pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
 pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
-pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GuardError};
+pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
 pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
 pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
 pub use logger::{log_init_state, InitLogStatus};
 pub use logger::{LogError, Logger};
+pub use metrics::{init_system_metrics, init_system_metrics_for_pid};
 pub use sink::Sink;
 use std::sync::Arc;
 pub use telemetry::init_telemetry;
-pub use telemetry::{get_global_registry, metrics};
 use tokio::sync::Mutex;
 pub use utils::{get_local_ip, get_local_ip_with_default};
 pub use worker::start_worker;

crates/obs/src/metrics.rs

@@ -0,0 +1,394 @@
+//! # Metrics
+//! This file is part of the RustFS project
+//! Current metrics observed are:
+//! - CPU
+//! - Memory
+//! - Disk
+//! - Network
+//!
+//! # Getting started
+//!
+//! ```
+//! use opentelemetry::global;
+//! use rustfs_obs::init_system_metrics;
+//!
+//! #[tokio::main]
+//! async fn main() {
+//!     let meter = global::meter("rustfs-system-meter");
+//!     let result = init_system_metrics(meter);
+//! }
+//! ```
+//!
+
+use crate::GlobalError;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::enums::device::UsedGpuMemory;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::Nvml;
+use opentelemetry::metrics::Meter;
+use opentelemetry::Key;
+use opentelemetry::KeyValue;
+use std::time::Duration;
+use sysinfo::{get_current_pid, System};
+use tokio::time::sleep;
+use tracing::warn;
+
+const PROCESS_PID: Key = Key::from_static_str("process.pid");
+const PROCESS_EXECUTABLE_NAME: Key = Key::from_static_str("process.executable.name");
+const PROCESS_EXECUTABLE_PATH: Key = Key::from_static_str("process.executable.path");
+const PROCESS_COMMAND: Key = Key::from_static_str("process.command");
+const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
+const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
+const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
+const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
+const PROCESS_DISK_IO: &str = "process.disk.io";
+const DIRECTION: Key = Key::from_static_str("direction");
+const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
+
+// add static variables that delay initialize nvml
+#[cfg(feature = "gpu")]
+static NVML_INSTANCE: OnceCell<Arc<Mutex<Option<Result<Nvml, nvml_wrapper::error::NvmlError>>>>> = OnceCell::const_new();
+
+// get or initialize an nvml instance
+#[cfg(feature = "gpu")]
+async fn get_or_init_nvml() -> &'static Arc<Mutex<Option<Result<Nvml, nvml_wrapper::error::NvmlError>>>> {
+    NVML_INSTANCE
+        .get_or_init(|| async { Arc::new(Mutex::new(Some(Nvml::init()))) })
+        .await
+}
+
+/// Record asynchronously information about the current process.
+/// This function is useful for monitoring the current process.
+///
+/// # Arguments
+/// * `meter` - The OpenTelemetry meter to use.
+///
+/// # Returns
+/// * `Ok(())` if successful
+/// * `Err(GlobalError)` if an error occurs
+///
+/// # Example
+/// ```
+/// use opentelemetry::global;
+/// use rustfs_obs::init_system_metrics;
+///
+/// #[tokio::main]
+/// async fn main() {
+///     let meter = global::meter("rustfs-system-meter");
+///     let result = init_system_metrics(meter);
+/// }
+/// ```
+pub async fn init_system_metrics(meter: Meter) -> Result<(), GlobalError> {
+    let pid = get_current_pid().map_err(|err| GlobalError::MetricsError(err.to_string()))?;
+    register_system_metrics(meter, pid).await
+}
+
+/// Record asynchronously information about a specific process by its PID.
+/// This function is useful for monitoring processes other than the current one.
+///
+/// # Arguments
+/// * `meter` - The OpenTelemetry meter to use.
+/// * `pid` - The PID of the process to monitor.
+///
+/// # Returns
+/// * `Ok(())` if successful
+/// * `Err(GlobalError)` if an error occurs
+///
+/// # Example
+/// ```
+/// use opentelemetry::global;
+/// use rustfs_obs::init_system_metrics_for_pid;
+///
+/// #[tokio::main]
+/// async fn main() {
+///     let meter = global::meter("rustfs-system-meter");
+///     // replace with the actual PID
+///     let pid = 1234;
+///     let result = init_system_metrics_for_pid(meter, pid).await;
+/// }
+/// ```
+///
+pub async fn init_system_metrics_for_pid(meter: Meter, pid: u32) -> Result<(), GlobalError> {
+    let pid = sysinfo::Pid::from_u32(pid);
+    register_system_metrics(meter, pid).await
+}
+
+/// Register system metrics for the current process.
+/// This function is useful for monitoring the current process.
+///
+/// # Arguments
+/// * `meter` - The OpenTelemetry meter to use.
+/// * `pid` - The PID of the process to monitor.
+///
+/// # Returns
+/// * `Ok(())` if successful
+/// * `Err(GlobalError)` if an error occurs
+///
+async fn register_system_metrics(meter: Meter, pid: sysinfo::Pid) -> Result<(), GlobalError> {
+    // cache core counts to avoid repeated calculations
+    let core_count = System::physical_core_count()
+        .ok_or_else(|| GlobalError::SystemMetricsError("Could not get physical core count".to_string()))?;
+    let core_count_f32 = core_count as f32;
+
+    // create metric meter
+    let (
+        process_cpu_utilization,
+        process_cpu_usage,
+        process_memory_usage,
+        process_memory_virtual,
+        process_disk_io,
+        process_gpu_memory_usage,
+    ) = create_metrics(&meter);
+
+    // initialize system object
+    let mut sys = System::new_all();
+    sys.refresh_all();
+
+    // Prepare public properties to avoid repeated construction in loops
+    let common_attributes = prepare_common_attributes(&sys, pid)?;
+
+    // get the metric export interval
+    let interval = get_export_interval();
+
+    // Use asynchronous tasks to process CPU, memory, and disk metrics to avoid blocking the main asynchronous tasks
+    let cpu_mem_task = tokio::spawn(async move {
+        loop {
+            sleep(Duration::from_millis(interval)).await;
+            if let Err(e) = update_process_metrics(
+                &mut sys,
+                pid,
+                &process_cpu_usage,
+                &process_cpu_utilization,
+                &process_memory_usage,
+                &process_memory_virtual,
+                &process_disk_io,
+                &common_attributes,
+                core_count_f32,
+            ) {
+                warn!("Failed to update process metrics: {}", e);
+            }
+        }
+    });
+
+    // Use another asynchronous task to handle GPU metrics
+    #[cfg(feature = "gpu")]
+    let gpu_task = tokio::spawn(async move {
+        loop {
+            sleep(Duration::from_millis(interval)).await;
+
+            // delayed initialization nvml
+            let nvml_arc = get_or_init_nvml().await;
+            let nvml_option = nvml_arc.lock().unwrap();
+
+            if let Err(e) = update_gpu_metrics(&nvml, pid, &process_gpu_memory_usage, &common_attributes) {
+                warn!("Failed to update GPU metrics: {}", e);
+            }
+        }
+    });
+
+    // record empty values when non gpu function
+    #[cfg(not(feature = "gpu"))]
+    let gpu_task = tokio::spawn(async move {
+        loop {
+            sleep(Duration::from_millis(interval)).await;
+            process_gpu_memory_usage.record(0, &common_attributes);
+        }
+    });
+
+    // Wait for the two tasks to complete (actually they will run forever)
+    let _ = tokio::join!(cpu_mem_task, gpu_task);
+
+    Ok(())
+}
+
+fn create_metrics(meter: &Meter) -> (F64Gauge, F64Gauge, I64Gauge, I64Gauge, I64Gauge, U64Gauge) {
+    let process_cpu_utilization = meter
+        .f64_gauge(PROCESS_CPU_USAGE)
+        .with_description("The percentage of CPU in use.")
+        .with_unit("percent")
+        .build();
+
+    let process_cpu_usage = meter
+        .f64_gauge(PROCESS_CPU_UTILIZATION)
+        .with_description("The amount of CPU in use.")
+        .with_unit("percent")
+        .build();
+
+    let process_memory_usage = meter
+        .i64_gauge(PROCESS_MEMORY_USAGE)
+        .with_description("The amount of physical memory in use.")
+        .with_unit("byte")
+        .build();
+
+    let process_memory_virtual = meter
+        .i64_gauge(PROCESS_MEMORY_VIRTUAL)
+        .with_description("The amount of committed virtual memory.")
+        .with_unit("byte")
+        .build();
+
+    let process_disk_io = meter
+        .i64_gauge(PROCESS_DISK_IO)
+        .with_description("Disk bytes transferred.")
+        .with_unit("byte")
+        .build();
+
+    let process_gpu_memory_usage = meter
+        .u64_gauge(PROCESS_GPU_MEMORY_USAGE)
+        .with_description("The amount of physical GPU memory in use.")
+        .with_unit("byte")
+        .build();
+
+    (
+        process_cpu_utilization,
+        process_cpu_usage,
+        process_memory_usage,
+        process_memory_virtual,
+        process_disk_io,
+        process_gpu_memory_usage,
+    )
+}
+
+fn prepare_common_attributes(sys: &System, pid: sysinfo::Pid) -> Result<[KeyValue; 4], GlobalError> {
+    let process = sys
+        .process(pid)
+        .ok_or_else(|| GlobalError::SystemMetricsError(format!("Process with PID {} not found", pid.as_u32())))?;
+
+    // optimize string operations and reduce allocation
+    let cmd = process.cmd().iter().filter_map(|s| s.to_str()).collect::<Vec<_>>().join(" ");
+
+    let executable_path = process
+        .exe()
+        .map(|path| path.to_string_lossy().into_owned())
+        .unwrap_or_default();
+
+    let name = process.name().to_os_string().into_string().unwrap_or_default();
+
+    Ok([
+        KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
+        KeyValue::new(PROCESS_EXECUTABLE_NAME, name),
+        KeyValue::new(PROCESS_EXECUTABLE_PATH, executable_path),
+        KeyValue::new(PROCESS_COMMAND, cmd),
+    ])
+}
+
+fn get_export_interval() -> u64 {
+    std::env::var("OTEL_METRIC_EXPORT_INTERVAL")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(30000)
+}
+
+fn update_process_metrics(
+    sys: &mut System,
+    pid: sysinfo::Pid,
+    process_cpu_usage: &F64Gauge,
+    process_cpu_utilization: &F64Gauge,
+    process_memory_usage: &I64Gauge,
+    process_memory_virtual: &I64Gauge,
+    process_disk_io: &I64Gauge,
+    common_attributes: &[KeyValue; 4],
+    core_count: f32,
+) -> Result<(), GlobalError> {
+    // Only refresh the data of the required process to reduce system call overhead
+    sys.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+
+    let process = match sys.process(pid) {
+        Some(p) => p,
+        None => {
+            return Err(GlobalError::SystemMetricsError(format!(
+                "Process with PID {} no longer exists",
+                pid.as_u32()
+            )))
+        }
+    };
+
+    // collect data in batches and record it again
+    let cpu_usage = process.cpu_usage();
+    process_cpu_usage.record(cpu_usage.into(), &[]);
+    process_cpu_utilization.record((cpu_usage / core_count).into(), &common_attributes);
+
+    // safe type conversion
+    let memory = process.memory();
+    let virtual_memory = process.virtual_memory();
+
+    // Avoid multiple error checks and use .map_err to handle errors in chain
+    let memory_i64 =
+        i64::try_from(memory).map_err(|_| GlobalError::ConversionError("Failed to convert memory usage to i64".to_string()))?;
+
+    let virtual_memory_i64 = i64::try_from(virtual_memory)
+        .map_err(|_| GlobalError::ConversionError("Failed to convert virtual memory to i64".to_string()))?;
+
+    process_memory_usage.record(memory_i64, common_attributes);
+    process_memory_virtual.record(virtual_memory_i64, common_attributes);
+
+    // process disk io metrics
+    let disk_io = process.disk_usage();
+
+    // batch conversion to reduce duplicate code
+    let read_bytes_i64 = i64::try_from(disk_io.read_bytes)
+        .map_err(|_| GlobalError::ConversionError("Failed to convert read bytes to i64".to_string()))?;
+
+    let written_bytes_i64 = i64::try_from(disk_io.written_bytes)
+        .map_err(|_| GlobalError::ConversionError("Failed to convert written bytes to i64".to_string()))?;
+
+    // Optimize attribute array stitching to reduce heap allocation
+    let mut read_attributes = [KeyValue::new(DIRECTION, "read")];
+    let read_attrs = [common_attributes, &read_attributes].concat();
+
+    let mut write_attributes = [KeyValue::new(DIRECTION, "write")];
+    let write_attrs = [common_attributes, &write_attributes].concat();
+
+    process_disk_io.record(read_bytes_i64, &read_attrs);
+    process_disk_io.record(written_bytes_i64, &write_attrs);
+
+    Ok(())
+}
+
+// GPU metric update function, conditional compilation based on feature flags
+#[cfg(feature = "gpu")]
+fn update_gpu_metrics(
+    nvml: &Result<Nvml, nvml_wrapper::error::NvmlError>,
+    pid: sysinfo::Pid,
+    process_gpu_memory_usage: &U64Gauge,
+    common_attributes: &[KeyValue; 4],
+) -> Result<(), GlobalError> {
+    match nvml {
+        Ok(nvml) => {
+            if let Ok(device) = nvml.device_by_index(0) {
+                if let Ok(gpu_stats) = device.running_compute_processes() {
+                    for stat in gpu_stats.iter() {
+                        if stat.pid == pid.as_u32() {
+                            let memory_used = match stat.used_gpu_memory {
+                                UsedGpuMemory::Used(bytes) => bytes,
+                                UsedGpuMemory::Unavailable => 0,
+                            };
+
+                            process_gpu_memory_usage.record(memory_used, common_attributes);
+                            return Ok(());
+                        }
+                    }
+                }
+            }
+            // If no GPU usage record of the process is found, the record is 0
+            process_gpu_memory_usage.record(0, common_attributes);
+            Ok(())
+        }
+        Err(e) => {
+            warn!("Could not get NVML, recording 0 for GPU memory usage: {}", e);
+            process_gpu_memory_usage.record(0, common_attributes);
+            Ok(())
+        }
+    }
+}
+
+#[cfg(not(feature = "gpu"))]
+fn update_gpu_metrics(
+    _: &(), // blank placeholder parameters
+    _: sysinfo::Pid,
+    process_gpu_memory_usage: &U64Gauge,
+    common_attributes: &[KeyValue; 4],
+) -> Result<(), GlobalError> {
+    // always logged when non gpu function 0
+    process_gpu_memory_usage.record(0, common_attributes);
+    Ok(())
+}

crates/obs/src/telemetry.rs

@@ -14,11 +14,10 @@ use opentelemetry_semantic_conventions::{
     attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
     SCHEMA_URL,
 };
-use prometheus::{Encoder, Registry, TextEncoder};
+use smallvec::SmallVec;
+use std::borrow::Cow;
 use std::io::IsTerminal;
-use std::sync::Arc;
-use tokio::sync::{Mutex, OnceCell};
-use tracing::{info, warn};
+use tracing::info;
 use tracing_error::ErrorLayer;
 use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
@@ -67,66 +66,20 @@ impl Drop for OtelGuard {
     }
 }
 
-/// Global registry for Prometheus metrics
-static GLOBAL_REGISTRY: OnceCell<Arc<Mutex<Registry>>> = OnceCell::const_new();
-
-/// Get the global registry instance
-/// This function returns a reference to the global registry instance.
-///
-/// # Returns
-/// A reference to the global registry instance
-///
-/// # Example
-/// ```
-/// use rustfs_obs::get_global_registry;
-///
-/// let registry = get_global_registry();
-/// ```
-pub fn get_global_registry() -> Arc<Mutex<Registry>> {
-    GLOBAL_REGISTRY.get().unwrap().clone()
-}
-
-/// Prometheus metric endpoints
-/// This function returns a string containing the Prometheus metrics.
-/// The metrics are collected from the global registry.
-/// The function is used to expose the metrics via an HTTP endpoint.
-///
-/// # Returns
-/// A string containing the Prometheus metrics
-///
-/// # Example
-/// ```
-/// use rustfs_obs::metrics;
-///
-/// async fn main() {
-///    let metrics = metrics().await;
-///   println!("{}", metrics);
-/// }
-/// ```
-pub async fn metrics() -> String {
-    let encoder = TextEncoder::new();
-    // Get a reference to the registry for reading metrics
-    let registry = get_global_registry().lock().await.to_owned();
-    let metric_families = registry.gather();
-    if metric_families.is_empty() {
-        warn!("No metrics available in Prometheus registry");
-    } else {
-        info!("Metrics collected: {} families", metric_families.len());
-    }
-    let mut buffer = Vec::new();
-    encoder.encode(&metric_families, &mut buffer).unwrap();
-    String::from_utf8(buffer).unwrap_or_else(|_| "Error encoding metrics".to_string())
-}
-
 /// create OpenTelemetry Resource
 fn resource(config: &OtelConfig) -> Resource {
-    let config = config.clone();
     Resource::builder()
-        .with_service_name(config.service_name.unwrap_or(SERVICE_NAME.to_string()))
+        .with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
         .with_schema_url(
             [
-                KeyValue::new(OTEL_SERVICE_VERSION, config.service_version.unwrap_or(SERVICE_VERSION.to_string())),
-                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, config.environment.unwrap_or(ENVIRONMENT.to_string())),
+                KeyValue::new(
+                    OTEL_SERVICE_VERSION,
+                    Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
+                ),
+                KeyValue::new(
+                    DEPLOYMENT_ENVIRONMENT_NAME,
+                    Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
+                ),
                 KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
             ],
             SCHEMA_URL,
@@ -141,163 +94,169 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
         .build()
 }
 
-/// Initialize Meter Provider
-fn init_meter_provider(config: &OtelConfig) -> SdkMeterProvider {
-    let mut builder = MeterProviderBuilder::default().with_resource(resource(config));
-    // If endpoint is empty, use stdout output
-    if config.endpoint.is_empty() {
-        builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-    } else {
-        // If endpoint is not empty, use otlp output
-        let exporter = opentelemetry_otlp::MetricExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
-            .build()
-            .unwrap();
-        builder = builder.with_reader(
-            PeriodicReader::builder(exporter)
-                .with_interval(std::time::Duration::from_secs(config.meter_interval.unwrap_or(METER_INTERVAL)))
-                .build(),
-        );
-        // If use_stdout is true, output to stdout at the same time
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
-        }
-    }
-    let registry = Registry::new();
-    // Set global registry
-    GLOBAL_REGISTRY.set(Arc::new(Mutex::new(registry.clone()))).unwrap();
-    // Create Prometheus exporter
-    let prometheus_exporter = opentelemetry_prometheus::exporter().with_registry(registry).build().unwrap();
-    // Build meter provider
-    let meter_provider = builder.with_reader(prometheus_exporter).build();
-    global::set_meter_provider(meter_provider.clone());
-    meter_provider
-}
-
-/// Initialize Tracer Provider
-fn init_tracer_provider(config: &OtelConfig) -> SdkTracerProvider {
-    let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
-    let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
-        Sampler::TraceIdRatioBased(sample_ratio)
-    } else {
-        Sampler::AlwaysOn
-    };
-    let builder = SdkTracerProvider::builder()
-        .with_sampler(sampler)
-        .with_id_generator(RandomIdGenerator::default())
-        .with_resource(resource(config));
-
-    let tracer_provider = if config.endpoint.is_empty() {
-        builder
-            .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-            .build()
-    } else {
-        let exporter = opentelemetry_otlp::SpanExporter::builder()
-            .with_tonic()
-            .with_endpoint(&config.endpoint)
-            .build()
-            .unwrap();
-        if config.use_stdout.unwrap_or(USE_STDOUT) {
-            builder
-                .with_batch_exporter(exporter)
-                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
-        } else {
-            builder.with_batch_exporter(exporter)
-        }
-        .build()
-    };
-
-    global::set_tracer_provider(tracer_provider.clone());
-    tracer_provider
-}
-
 /// Initialize Telemetry
 pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
-    let tracer_provider = init_tracer_provider(config);
-    let meter_provider = init_meter_provider(config);
+    // avoid repeated access to configuration fields
+    let endpoint = &config.endpoint;
+    let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
+    let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
+    let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
+    let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
 
-    // Initialize logger provider based on configuration
-    let logger_provider = {
-        let mut builder = SdkLoggerProvider::builder().with_resource(resource(config));
+    // Pre-create resource objects to avoid repeated construction
+    let res = resource(config);
 
-        if config.endpoint.is_empty() {
-            // Use stdout exporter when no endpoint is configured
-            builder = builder.with_simple_exporter(opentelemetry_stdout::LogExporter::default());
+    // initialize tracer provider
+    let tracer_provider = {
+        let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
+        let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
+            Sampler::TraceIdRatioBased(sample_ratio)
+        } else {
+            Sampler::AlwaysOn
+        };
+
+        let builder = SdkTracerProvider::builder()
+            .with_sampler(sampler)
+            .with_id_generator(RandomIdGenerator::default())
+            .with_resource(res.clone());
+
+        let tracer_provider = if endpoint.is_empty() {
+            builder
+                .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+                .build()
+        } else {
+            let exporter = opentelemetry_otlp::SpanExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .build()
+                .unwrap();
+
+            let builder = if use_stdout {
+                builder
+                    .with_batch_exporter(exporter)
+                    .with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
+            } else {
+                builder.with_batch_exporter(exporter)
+            };
+
+            builder.build()
+        };
+
+        global::set_tracer_provider(tracer_provider.clone());
+        tracer_provider
+    };
+
+    // initialize meter provider
+    let meter_provider = {
+        let mut builder = MeterProviderBuilder::default().with_resource(res.clone());
+
+        if endpoint.is_empty() {
+            builder = builder.with_reader(create_periodic_reader(meter_interval));
+        } else {
+            let exporter = opentelemetry_otlp::MetricExporter::builder()
+                .with_tonic()
+                .with_endpoint(endpoint)
+                .with_temporality(opentelemetry_sdk::metrics::Temporality::default())
+                .build()
+                .unwrap();
+
+            builder = builder.with_reader(
+                PeriodicReader::builder(exporter)
+                    .with_interval(std::time::Duration::from_secs(meter_interval))
+                    .build(),
+            );
+
+            if use_stdout {
+                builder = builder.with_reader(create_periodic_reader(meter_interval));
+            }
+        }
+
+        let meter_provider = builder.build();
+        global::set_meter_provider(meter_provider.clone());
+        meter_provider
+    };
+
+    // initialize logger provider
+    let logger_provider = {
+        let mut builder = SdkLoggerProvider::builder().with_resource(res);
+
+        if endpoint.is_empty() {
+            builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
         } else {
-            // Configure OTLP exporter when endpoint is provided
             let exporter = opentelemetry_otlp::LogExporter::builder()
                 .with_tonic()
-                .with_endpoint(&config.endpoint)
+                .with_endpoint(endpoint)
                 .build()
                 .unwrap();
 
             builder = builder.with_batch_exporter(exporter);
 
-            // Add stdout exporter if requested
-            if config.use_stdout.unwrap_or(USE_STDOUT) {
+            if use_stdout {
                 builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
             }
         }
 
         builder.build()
     };
-    let config = config.clone();
-    let logger_level = config.logger_level.unwrap_or(LOGGER_LEVEL.to_string());
-    let logger_level = logger_level.as_str();
-    // Setup OpenTelemetryTracingBridge layer
-    let otel_layer = {
-        // Filter to prevent infinite telemetry loops
-        // This blocks events from OpenTelemetry and its dependent libraries (tonic, reqwest, etc.)
-        // from being sent back to OpenTelemetry itself
-        let filter_otel = match logger_level {
-            "trace" | "debug" => {
-                info!("OpenTelemetry tracing initialized with level: {}", logger_level);
-                EnvFilter::new(logger_level)
-            }
-            _ => {
-                let mut filter = EnvFilter::new(logger_level);
-                for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
-                    filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+
+    // configuring tracing
+    {
+        // optimize filter configuration
+        let otel_layer = {
+            let filter_otel = match logger_level {
+                "trace" | "debug" => {
+                    info!("OpenTelemetry tracing initialized with level: {}", logger_level);
+                    EnvFilter::new(logger_level)
                 }
-                filter
-            }
+                _ => {
+                    let mut filter = EnvFilter::new(logger_level);
+
+                    // use smallvec to avoid heap allocation
+                    let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
+
+                    for directive in directives {
+                        filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
+                    }
+                    filter
+                }
+            };
+            layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
         };
-        layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
-    };
 
-    let tracer = tracer_provider.tracer(config.service_name.unwrap_or(SERVICE_NAME.to_string()));
-    let registry = tracing_subscriber::registry()
-        .with(switch_level(logger_level))
-        .with(OpenTelemetryLayer::new(tracer))
-        .with(MetricsLayer::new(meter_provider.clone()))
-        .with(otel_layer)
-        .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
+        let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
 
-    // Configure formatting layer
-    let enable_color = std::io::stdout().is_terminal();
-    let fmt_layer = tracing_subscriber::fmt::layer()
-        .with_ansi(enable_color)
-        .with_thread_names(true)
-        .with_file(true)
-        .with_line_number(true);
+        // Configure registry to avoid repeated calls to filter methods
+        let level_filter = switch_level(logger_level);
+        let registry = tracing_subscriber::registry()
+            .with(level_filter)
+            .with(OpenTelemetryLayer::new(tracer))
+            .with(MetricsLayer::new(meter_provider.clone()))
+            .with(otel_layer)
+            .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
 
-    // Creating a formatting layer with explicit type to avoid type mismatches
-    let fmt_layer = fmt_layer.with_filter(
-        EnvFilter::new(logger_level).add_directive(
-            format!("opentelemetry={}", if config.endpoint.is_empty() { logger_level } else { "off" })
-                .parse()
-                .unwrap(),
-        ),
-    );
+        // configure the formatting layer
+        let enable_color = std::io::stdout().is_terminal();
+        let fmt_layer = tracing_subscriber::fmt::layer()
+            .with_ansi(enable_color)
+            .with_thread_names(true)
+            .with_file(true)
+            .with_line_number(true)
+            .with_filter(
+                EnvFilter::new(logger_level).add_directive(
+                    format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
+                        .parse()
+                        .unwrap(),
+                ),
+            );
 
-    registry.with(ErrorLayer::default()).with(fmt_layer).init();
-    if !config.endpoint.is_empty() {
-        info!(
-            "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
-            config.endpoint, logger_level
-        );
+        registry.with(ErrorLayer::default()).with(fmt_layer).init();
+
+        if !endpoint.is_empty() {
+            info!(
+                "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
+                endpoint, logger_level
+            );
+        }
     }
 
     OtelGuard {
@@ -309,12 +268,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 
 /// Switch log level
 fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
+    use tracing_subscriber::filter::LevelFilter;
     match logger_level {
-        "error" => tracing_subscriber::filter::LevelFilter::ERROR,
-        "warn" => tracing_subscriber::filter::LevelFilter::WARN,
-        "info" => tracing_subscriber::filter::LevelFilter::INFO,
-        "debug" => tracing_subscriber::filter::LevelFilter::DEBUG,
-        "trace" => tracing_subscriber::filter::LevelFilter::TRACE,
-        _ => tracing_subscriber::filter::LevelFilter::OFF,
+        "error" => LevelFilter::ERROR,
+        "warn" => LevelFilter::WARN,
+        "info" => LevelFilter::INFO,
+        "debug" => LevelFilter::DEBUG,
+        "trace" => LevelFilter::TRACE,
+        _ => LevelFilter::OFF,
     }
 }