feat: add TraceLayer for HTTP service and improve metrics

- Add TraceLayer to HTTP server for request tracing
- Implement system metrics for process monitoring
- Optimize init_telemetry method for better resource management
- Add graceful shutdown handling for telemetry components
- Fix GracefulShutdown ownership issues with Arc wrapper

crates/obs/src/system/attributes.rs

@@ -0,0 +1,44 @@
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use sysinfo::{Pid, System};
+
+pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
+pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
+pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
+pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
+
+/// Struct to hold process attributes
+pub struct ProcessAttributes {
+    pub attributes: Vec<KeyValue>,
+}
+
+impl ProcessAttributes {
+    /// Creates a new instance of `ProcessAttributes` for the given PID.
+    pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
+        system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
+        let process = system
+            .process(pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
+
+        let attributes = vec![
+            KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
+            KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
+            KeyValue::new(
+                PROCESS_EXECUTABLE_PATH,
+                process
+                    .exe()
+                    .map(|path| path.to_string_lossy().into_owned())
+                    .unwrap_or_default(),
+            ),
+            KeyValue::new(
+                PROCESS_COMMAND,
+                process
+                    .cmd()
+                    .iter()
+                    .fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
+            ),
+        ];
+
+        Ok(ProcessAttributes { attributes })
+    }
+}

crates/obs/src/system/collector.rs

@@ -0,0 +1,156 @@
+use crate::system::attributes::ProcessAttributes;
+use crate::system::gpu::GpuCollector;
+use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
+use crate::GlobalError;
+use opentelemetry::KeyValue;
+use std::time::SystemTime;
+use sysinfo::{Networks, Pid, ProcessStatus, System};
+use tokio::time::{sleep, Duration};
+
+/// Collector is responsible for collecting system metrics and attributes.
+/// It uses the sysinfo crate to gather information about the system and processes.
+/// It also uses OpenTelemetry to record metrics.
+pub struct Collector {
+    metrics: Metrics,
+    attributes: ProcessAttributes,
+    gpu_collector: GpuCollector,
+    pid: Pid,
+    system: System,
+    networks: Networks,
+    core_count: usize,
+    interval_ms: u64,
+}
+
+impl Collector {
+    pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
+        let mut system = System::new_all();
+        let attributes = ProcessAttributes::new(pid, &mut system)?;
+        let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
+        let metrics = Metrics::new(&meter);
+        let gpu_collector = GpuCollector::new(pid)?;
+        let networks = Networks::new_with_refreshed_list();
+
+        Ok(Collector {
+            metrics,
+            attributes,
+            gpu_collector,
+            pid,
+            system,
+            networks,
+            core_count,
+            interval_ms,
+        })
+    }
+
+    pub async fn run(&mut self) -> Result<(), GlobalError> {
+        loop {
+            self.collect()?;
+            tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
+            sleep(Duration::from_millis(self.interval_ms)).await;
+        }
+    }
+
+    fn collect(&mut self) -> Result<(), GlobalError> {
+        self.system
+            .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
+
+        // 刷新网络接口列表和统计数据
+        self.networks.refresh(false); // 刷新网络统计数据
+
+        let process = self
+            .system
+            .process(self.pid)
+            .ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
+
+        // CPU 指标
+        let cpu_usage = process.cpu_usage();
+        self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
+        self.metrics
+            .cpu_utilization
+            .record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
+
+        // 内存指标
+        self.metrics
+            .memory_usage
+            .record(process.memory() as i64, &self.attributes.attributes);
+        self.metrics
+            .memory_virtual
+            .record(process.virtual_memory() as i64, &self.attributes.attributes);
+
+        // 磁盘I/O指标
+        let disk_io = process.disk_usage();
+        self.metrics.disk_io.record(
+            disk_io.read_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
+        );
+        self.metrics.disk_io.record(
+            disk_io.written_bytes as i64,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
+        );
+
+        // 网络I/O指标（对应 /system/network/internode）
+        let mut total_received: i64 = 0;
+        let mut total_transmitted: i64 = 0;
+
+        // 按接口统计
+        for (interface_name, data) in self.networks.iter() {
+            total_received += data.total_received() as i64;
+            total_transmitted += data.total_transmitted() as i64;
+
+            let received = data.received() as i64;
+            let transmitted = data.transmitted() as i64;
+            self.metrics.network_io_per_interface.record(
+                received,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "received"),
+                    ],
+                ]
+                .concat(),
+            );
+            self.metrics.network_io_per_interface.record(
+                transmitted,
+                &[
+                    &self.attributes.attributes[..],
+                    &[
+                        KeyValue::new(INTERFACE, interface_name.to_string()),
+                        KeyValue::new(DIRECTION, "transmitted"),
+                    ],
+                ]
+                .concat(),
+            );
+        }
+        // 全局统计
+        self.metrics.network_io.record(
+            total_received,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
+        );
+        self.metrics.network_io.record(
+            total_transmitted,
+            &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
+        );
+
+        // 进程状态指标（对应 /system/process）
+        let status_value = match process.status() {
+            ProcessStatus::Run => 0,
+            ProcessStatus::Sleep => 1,
+            ProcessStatus::Zombie => 2,
+            _ => 3, // 其他状态
+        };
+        self.metrics.process_status.record(
+            status_value,
+            &[
+                &self.attributes.attributes[..],
+                &[KeyValue::new(STATUS, format!("{:?}", process.status()))],
+            ]
+            .concat(),
+        );
+
+        // GPU 指标（可选）
+        self.gpu_collector.collect(&self.metrics, &self.attributes)?;
+
+        Ok(())
+    }
+}

crates/obs/src/system/gpu.rs

@@ -0,0 +1,70 @@
+#[cfg(feature = "gpu")]
+use crate::system::attributes::ProcessAttributes;
+#[cfg(feature = "gpu")]
+use crate::system::metrics::Metrics;
+#[cfg(feature = "gpu")]
+use crate::GlobalError;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::enums::device::UsedGpuMemory;
+#[cfg(feature = "gpu")]
+use nvml_wrapper::Nvml;
+#[cfg(feature = "gpu")]
+use sysinfo::Pid;
+#[cfg(feature = "gpu")]
+use tracing::warn;
+
+/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
+#[cfg(feature = "gpu")]
+pub struct GpuCollector {
+    nvml: Nvml,
+    pid: Pid,
+}
+
+#[cfg(feature = "gpu")]
+impl GpuCollector {
+    pub fn new(pid: Pid) -> Result<Self, GlobalError> {
+        let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
+        Ok(GpuCollector { nvml, pid })
+    }
+
+    pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
+        if let Ok(device) = self.nvml.device_by_index(0) {
+            if let Ok(gpu_stats) = device.running_compute_processes() {
+                for stat in gpu_stats.iter() {
+                    if stat.pid == self.pid.as_u32() {
+                        let memory_used = match stat.used_gpu_memory {
+                            UsedGpuMemory::Used(bytes) => bytes,
+                            UsedGpuMemory::Unavailable => 0,
+                        };
+                        metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
+                        return Ok(());
+                    }
+                }
+            } else {
+                warn!("Could not get GPU stats, recording 0 for GPU memory usage");
+            }
+        } else {
+            return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
+        }
+        metrics.gpu_memory_usage.record(0, &attributes.attributes);
+        Ok(())
+    }
+}
+
+#[cfg(not(feature = "gpu"))]
+pub struct GpuCollector;
+
+#[cfg(not(feature = "gpu"))]
+impl GpuCollector {
+    pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
+        Ok(GpuCollector)
+    }
+
+    pub fn collect(
+        &self,
+        _metrics: &crate::system::metrics::Metrics,
+        _attributes: &crate::system::attributes::ProcessAttributes,
+    ) -> Result<(), crate::GlobalError> {
+        Ok(())
+    }
+}

crates/obs/src/system/metrics.rs

@@ -0,0 +1,100 @@
+pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
+pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
+pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
+pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
+pub const PROCESS_DISK_IO: &str = "process.disk.io";
+pub const PROCESS_NETWORK_IO: &str = "process.network.io";
+pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
+pub const PROCESS_STATUS: &str = "process.status";
+#[cfg(feature = "gpu")]
+pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
+pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
+pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
+pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
+
+/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
+/// It contains various metrics such as CPU usage, memory usage,
+/// disk I/O, network I/O, and process status.
+///
+/// The `Metrics` struct is designed to be used with OpenTelemetry's
+/// metrics API to record and export these metrics.
+///
+/// The `new` method initializes the metrics using the provided
+/// `opentelemetry::metrics::Meter`.
+pub struct Metrics {
+    pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
+    pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
+    pub memory_usage: opentelemetry::metrics::Gauge<i64>,
+    pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
+    pub disk_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io: opentelemetry::metrics::Gauge<i64>,
+    pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
+    pub process_status: opentelemetry::metrics::Gauge<i64>,
+    #[cfg(feature = "gpu")]
+    pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
+}
+
+impl Metrics {
+    pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
+        let cpu_usage = meter
+            .f64_gauge(PROCESS_CPU_USAGE)
+            .with_description("The percentage of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let cpu_utilization = meter
+            .f64_gauge(PROCESS_CPU_UTILIZATION)
+            .with_description("The amount of CPU in use.")
+            .with_unit("percent")
+            .build();
+        let memory_usage = meter
+            .i64_gauge(PROCESS_MEMORY_USAGE)
+            .with_description("The amount of physical memory in use.")
+            .with_unit("byte")
+            .build();
+        let memory_virtual = meter
+            .i64_gauge(PROCESS_MEMORY_VIRTUAL)
+            .with_description("The amount of committed virtual memory.")
+            .with_unit("byte")
+            .build();
+        let disk_io = meter
+            .i64_gauge(PROCESS_DISK_IO)
+            .with_description("Disk bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io = meter
+            .i64_gauge(PROCESS_NETWORK_IO)
+            .with_description("Network bytes transferred.")
+            .with_unit("byte")
+            .build();
+        let network_io_per_interface = meter
+            .i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
+            .with_description("Network bytes transferred (per interface).")
+            .with_unit("byte")
+            .build();
+
+        let process_status = meter
+            .i64_gauge(PROCESS_STATUS)
+            .with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
+            .build();
+
+        #[cfg(feature = "gpu")]
+        let gpu_memory_usage = meter
+            .u64_gauge(PROCESS_GPU_MEMORY_USAGE)
+            .with_description("The amount of physical GPU memory in use.")
+            .with_unit("byte")
+            .build();
+
+        Metrics {
+            cpu_usage,
+            cpu_utilization,
+            memory_usage,
+            memory_virtual,
+            disk_io,
+            network_io,
+            network_io_per_interface,
+            process_status,
+            #[cfg(feature = "gpu")]
+            gpu_memory_usage,
+        }
+    }
+}

crates/obs/src/system/mod.rs

@@ -0,0 +1,24 @@
+use crate::GlobalError;
+
+pub(crate) mod attributes;
+mod collector;
+pub(crate) mod gpu;
+pub(crate) mod metrics;
+
+/// Initialize the indicator collector for the current process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
+    let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}
+
+/// Initialize the metric collector for the specified PID process
+/// This function will create a new `Collector` instance and start collecting metrics.
+/// It will run indefinitely until the process is terminated.
+pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
+    let pid = sysinfo::Pid::from_u32(pid);
+    let mut collector = collector::Collector::new(pid, meter, 30000)?;
+    collector.run().await
+}