mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
feat: add TraceLayer for HTTP service and improve metrics
- Add TraceLayer to HTTP server for request tracing - Implement system metrics for process monitoring - Optimize init_telemetry method for better resource management - Add graceful shutdown handling for telemetry components - Fix GracefulShutdown ownership issues with Arc wrapper
This commit is contained in:
houseme
@@ -25,7 +25,7 @@ batch_timeout_ms = 1000 # Default is 100ms if not specified
|
||||
|
||||
[sinks.file]
|
||||
enabled = true
|
||||
path = "logs/app.log"
|
||||
path = "deploy/logs/app.log"
|
||||
batch_size = 100
|
||||
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use opentelemetry::global;
|
||||
use rustfs_obs::{get_logger, init_obs, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
||||
use rustfs_obs::{get_logger, init_obs, init_process_observer, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{info, instrument};
|
||||
use tracing::{error, info, instrument};
|
||||
use tracing_core::Level;
|
||||
|
||||
#[tokio::main]
|
||||
@@ -38,6 +38,11 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
|
||||
&[opentelemetry::KeyValue::new("operation", "run")],
|
||||
);
|
||||
|
||||
match init_process_observer(meter).await {
|
||||
Ok(_) => info!("Process observer initialized successfully"),
|
||||
Err(e) => error!("Failed to initialize process observer: {:?}", e),
|
||||
}
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.message(Some("run logger api_handler info".to_string()))
|
||||
.request_id(Some("request_id".to_string()))
|
||||
|
||||
+14
-11
@@ -23,17 +23,20 @@ pub enum GlobalError {
|
||||
NotInitialized,
|
||||
#[error("Global system metrics err: {0}")]
|
||||
MetricsError(String),
|
||||
#[error("Failed to get process ID: {0}")]
|
||||
GetPidError(String),
|
||||
#[error("Type conversion error: {0}")]
|
||||
ConversionError(String),
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
#[error("System metrics error: {0}")]
|
||||
SystemMetricsError(String),
|
||||
#[cfg(feature = "gpu")]
|
||||
#[error("GPU metrics error: {0}")]
|
||||
GpuMetricsError(String),
|
||||
#[error("Failed to get current PID: {0}")]
|
||||
PidError(String),
|
||||
#[error("Process with PID {0} not found")]
|
||||
ProcessNotFound(u32),
|
||||
#[error("Failed to get physical core count")]
|
||||
CoreCountError,
|
||||
#[error("GPU initialization failed: {0}")]
|
||||
GpuInitError(String),
|
||||
#[error("GPU device not found: {0}")]
|
||||
GpuDeviceError(String),
|
||||
#[error("Failed to send log: {0}")]
|
||||
SendFailed(&'static str),
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(&'static str),
|
||||
}
|
||||
|
||||
/// Set the global guard for OpenTelemetry
|
||||
|
||||
+27
-13
@@ -32,34 +32,35 @@ mod config;
|
||||
mod entry;
|
||||
mod global;
|
||||
mod logger;
|
||||
mod metrics;
|
||||
mod sink;
|
||||
mod system;
|
||||
mod telemetry;
|
||||
mod utils;
|
||||
mod worker;
|
||||
|
||||
#[cfg(feature = "gpu")]
|
||||
pub use crate::metrics::init_gpu_metrics;
|
||||
|
||||
use crate::logger::InitLogStatus;
|
||||
pub use config::load_config;
|
||||
pub use config::{AppConfig, OtelConfig};
|
||||
#[cfg(feature = "file")]
|
||||
pub use config::FileSinkConfig;
|
||||
#[cfg(feature = "kafka")]
|
||||
pub use config::KafkaSinkConfig;
|
||||
#[cfg(feature = "webhook")]
|
||||
pub use config::WebhookSinkConfig;
|
||||
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
|
||||
pub use entry::args::Args;
|
||||
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
||||
pub use entry::base::BaseLogEntry;
|
||||
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
||||
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
||||
pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
|
||||
pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
|
||||
pub use logger::{log_init_state, InitLogStatus};
|
||||
pub use logger::{LogError, Logger};
|
||||
pub use metrics::{init_system_metrics, init_system_metrics_for_pid};
|
||||
pub use sink::Sink;
|
||||
pub use logger::Logger;
|
||||
pub use logger::{get_global_logger, init_global_logger, start_logger};
|
||||
pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
use std::sync::Arc;
|
||||
pub use system::{init_process_observer, init_process_observer_for_pid};
|
||||
pub use telemetry::init_telemetry;
|
||||
use tokio::sync::Mutex;
|
||||
pub use utils::{get_local_ip, get_local_ip_with_default};
|
||||
pub use worker::start_worker;
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Initialize the observability module
|
||||
///
|
||||
@@ -80,6 +81,19 @@ pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::Otel
|
||||
let guard = init_telemetry(&config.observability);
|
||||
let sinks = sink::create_sinks(&config).await;
|
||||
let logger = init_global_logger(&config, sinks).await;
|
||||
let obs_config = config.observability.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = InitLogStatus::init_start_log(&obs_config).await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Logger initialized successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to initialize logger: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(logger, guard)
|
||||
}
|
||||
|
||||
|
||||
+21
-81
@@ -1,5 +1,6 @@
|
||||
use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
|
||||
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, OtelConfig, ServerLogEntry, Sink, UnifiedLogEntry};
|
||||
use crate::sink::Sink;
|
||||
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
@@ -43,26 +44,26 @@ impl Logger {
|
||||
|
||||
/// Log a server entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
||||
}
|
||||
|
||||
/// Log an audit entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
||||
}
|
||||
|
||||
/// Log a console entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
||||
}
|
||||
|
||||
/// Asynchronous logging of unified log entries
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger"))]
|
||||
#[tracing::instrument(level = "error", skip_all)]
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
|
||||
// Extract information for tracing based on entry type
|
||||
match &entry {
|
||||
UnifiedLogEntry::Server(server) => {
|
||||
@@ -123,11 +124,11 @@ impl Logger {
|
||||
tracing::warn!("Log queue full, applying backpressure");
|
||||
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
Ok(Err(_)) => Err(LogError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(LogError::Timeout("Queue backpressure timeout")),
|
||||
Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
|
||||
}
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(LogError::SendFailed("Logger channel closed")),
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +161,7 @@ impl Logger {
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), LogError> {
|
||||
) -> Result<(), GlobalError> {
|
||||
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
||||
|
||||
let server_entry = ServerLogEntry::new(level, source.to_string())
|
||||
@@ -190,7 +191,7 @@ impl Logger {
|
||||
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), LogError> {
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
|
||||
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
||||
}
|
||||
|
||||
@@ -208,31 +209,12 @@ impl Logger {
|
||||
/// let _ = logger.shutdown().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn shutdown(self) -> Result<(), LogError> {
|
||||
pub async fn shutdown(self) -> Result<(), GlobalError> {
|
||||
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Log error type
|
||||
/// This enum defines the error types that can occur when logging.
|
||||
/// It is used to provide more detailed error information.
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::LogError;
|
||||
/// use thiserror::Error;
|
||||
///
|
||||
/// LogError::SendFailed("Failed to send log");
|
||||
/// LogError::Timeout("Operation timed out");
|
||||
/// ```
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LogError {
|
||||
#[error("Failed to send log: {0}")]
|
||||
SendFailed(&'static str),
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(&'static str),
|
||||
}
|
||||
|
||||
/// Start the log module
|
||||
/// This function starts the log module.
|
||||
/// It initializes the logger and starts the worker to process logs.
|
||||
@@ -297,48 +279,6 @@ pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
|
||||
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
||||
}
|
||||
|
||||
/// Get the global logger instance with a lock
|
||||
/// This function returns a reference to the global logger instance with a lock.
|
||||
/// It is used to ensure that the logger is thread-safe.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance with a lock
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::locked_logger;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let logger = locked_logger().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn locked_logger() -> tokio::sync::MutexGuard<'static, Logger> {
|
||||
get_global_logger().lock().await
|
||||
}
|
||||
|
||||
/// Initialize with default empty logger if needed (optional)
|
||||
/// This function initializes the logger with a default empty logger if needed.
|
||||
/// It is used to ensure that the logger is initialized before logging.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ensure_logger_initialized;
|
||||
///
|
||||
/// let logger = ensure_logger_initialized();
|
||||
/// ```
|
||||
pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
||||
if GLOBAL_LOGGER.get().is_none() {
|
||||
let config = AppConfig::default();
|
||||
let sinks = vec![];
|
||||
let logger = Arc::new(Mutex::new(start_logger(&config, sinks)));
|
||||
let _ = GLOBAL_LOGGER.set(logger);
|
||||
}
|
||||
GLOBAL_LOGGER.get().unwrap()
|
||||
}
|
||||
|
||||
/// Log information
|
||||
/// This function logs information messages.
|
||||
///
|
||||
@@ -357,7 +297,7 @@ pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
||||
/// let _ = log_info("This is an information message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
||||
}
|
||||
|
||||
@@ -375,7 +315,7 @@ pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// async fn example() {
|
||||
/// let _ = log_error("This is an error message", "example").await;
|
||||
/// }
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
||||
}
|
||||
|
||||
@@ -395,7 +335,7 @@ pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_warn("This is a warning message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
||||
}
|
||||
|
||||
@@ -415,7 +355,7 @@ pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_debug("This is a debug message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
||||
}
|
||||
|
||||
@@ -436,7 +376,7 @@ pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_trace("This is a trace message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
||||
}
|
||||
|
||||
@@ -467,7 +407,7 @@ pub async fn log_with_context(
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), LogError> {
|
||||
) -> Result<(), GlobalError> {
|
||||
get_global_logger()
|
||||
.lock()
|
||||
.await
|
||||
@@ -477,7 +417,7 @@ pub async fn log_with_context(
|
||||
|
||||
/// Log initialization status
|
||||
#[derive(Debug)]
|
||||
pub struct InitLogStatus {
|
||||
pub(crate) struct InitLogStatus {
|
||||
pub timestamp: SystemTime,
|
||||
pub service_name: String,
|
||||
pub version: String,
|
||||
@@ -508,14 +448,14 @@ impl InitLogStatus {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), LogError> {
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
|
||||
let status = Self::new_config(config);
|
||||
log_init_state(Some(status)).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Log initialization details during system startup
|
||||
pub async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), LogError> {
|
||||
async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
|
||||
let status = status.unwrap_or_default();
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
|
||||
@@ -1,394 +0,0 @@
|
||||
//! # Metrics
|
||||
//! This file is part of the RustFS project
|
||||
//! Current metrics observed are:
|
||||
//! - CPU
|
||||
//! - Memory
|
||||
//! - Disk
|
||||
//! - Network
|
||||
//!
|
||||
//! # Getting started
|
||||
//!
|
||||
//! ```
|
||||
//! use opentelemetry::global;
|
||||
//! use rustfs_obs::init_system_metrics;
|
||||
//!
|
||||
//! #[tokio::main]
|
||||
//! async fn main() {
|
||||
//! let meter = global::meter("rustfs-system-meter");
|
||||
//! let result = init_system_metrics(meter);
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
|
||||
use crate::GlobalError;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::enums::device::UsedGpuMemory;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::Nvml;
|
||||
use opentelemetry::metrics::Meter;
|
||||
use opentelemetry::Key;
|
||||
use opentelemetry::KeyValue;
|
||||
use std::time::Duration;
|
||||
use sysinfo::{get_current_pid, System};
|
||||
use tokio::time::sleep;
|
||||
use tracing::warn;
|
||||
|
||||
const PROCESS_PID: Key = Key::from_static_str("process.pid");
|
||||
const PROCESS_EXECUTABLE_NAME: Key = Key::from_static_str("process.executable.name");
|
||||
const PROCESS_EXECUTABLE_PATH: Key = Key::from_static_str("process.executable.path");
|
||||
const PROCESS_COMMAND: Key = Key::from_static_str("process.command");
|
||||
const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
|
||||
const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
|
||||
const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
|
||||
const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
|
||||
const PROCESS_DISK_IO: &str = "process.disk.io";
|
||||
const DIRECTION: Key = Key::from_static_str("direction");
|
||||
const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
|
||||
|
||||
// add static variables that delay initialize nvml
|
||||
#[cfg(feature = "gpu")]
|
||||
static NVML_INSTANCE: OnceCell<Arc<Mutex<Option<Result<Nvml, nvml_wrapper::error::NvmlError>>>>> = OnceCell::const_new();
|
||||
|
||||
// get or initialize an nvml instance
|
||||
#[cfg(feature = "gpu")]
|
||||
async fn get_or_init_nvml() -> &'static Arc<Mutex<Option<Result<Nvml, nvml_wrapper::error::NvmlError>>>> {
|
||||
NVML_INSTANCE
|
||||
.get_or_init(|| async { Arc::new(Mutex::new(Some(Nvml::init()))) })
|
||||
.await
|
||||
}
|
||||
|
||||
/// Record asynchronously information about the current process.
|
||||
/// This function is useful for monitoring the current process.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `meter` - The OpenTelemetry meter to use.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` if successful
|
||||
/// * `Err(GlobalError)` if an error occurs
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use opentelemetry::global;
|
||||
/// use rustfs_obs::init_system_metrics;
|
||||
///
|
||||
/// #[tokio::main]
|
||||
/// async fn main() {
|
||||
/// let meter = global::meter("rustfs-system-meter");
|
||||
/// let result = init_system_metrics(meter);
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn init_system_metrics(meter: Meter) -> Result<(), GlobalError> {
|
||||
let pid = get_current_pid().map_err(|err| GlobalError::MetricsError(err.to_string()))?;
|
||||
register_system_metrics(meter, pid).await
|
||||
}
|
||||
|
||||
/// Record asynchronously information about a specific process by its PID.
|
||||
/// This function is useful for monitoring processes other than the current one.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `meter` - The OpenTelemetry meter to use.
|
||||
/// * `pid` - The PID of the process to monitor.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` if successful
|
||||
/// * `Err(GlobalError)` if an error occurs
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use opentelemetry::global;
|
||||
/// use rustfs_obs::init_system_metrics_for_pid;
|
||||
///
|
||||
/// #[tokio::main]
|
||||
/// async fn main() {
|
||||
/// let meter = global::meter("rustfs-system-meter");
|
||||
/// // replace with the actual PID
|
||||
/// let pid = 1234;
|
||||
/// let result = init_system_metrics_for_pid(meter, pid).await;
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
pub async fn init_system_metrics_for_pid(meter: Meter, pid: u32) -> Result<(), GlobalError> {
|
||||
let pid = sysinfo::Pid::from_u32(pid);
|
||||
register_system_metrics(meter, pid).await
|
||||
}
|
||||
|
||||
/// Register system metrics for the current process.
|
||||
/// This function is useful for monitoring the current process.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `meter` - The OpenTelemetry meter to use.
|
||||
/// * `pid` - The PID of the process to monitor.
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` if successful
|
||||
/// * `Err(GlobalError)` if an error occurs
|
||||
///
|
||||
async fn register_system_metrics(meter: Meter, pid: sysinfo::Pid) -> Result<(), GlobalError> {
|
||||
// cache core counts to avoid repeated calculations
|
||||
let core_count = System::physical_core_count()
|
||||
.ok_or_else(|| GlobalError::SystemMetricsError("Could not get physical core count".to_string()))?;
|
||||
let core_count_f32 = core_count as f32;
|
||||
|
||||
// create metric meter
|
||||
let (
|
||||
process_cpu_utilization,
|
||||
process_cpu_usage,
|
||||
process_memory_usage,
|
||||
process_memory_virtual,
|
||||
process_disk_io,
|
||||
process_gpu_memory_usage,
|
||||
) = create_metrics(&meter);
|
||||
|
||||
// initialize system object
|
||||
let mut sys = System::new_all();
|
||||
sys.refresh_all();
|
||||
|
||||
// Prepare public properties to avoid repeated construction in loops
|
||||
let common_attributes = prepare_common_attributes(&sys, pid)?;
|
||||
|
||||
// get the metric export interval
|
||||
let interval = get_export_interval();
|
||||
|
||||
// Use asynchronous tasks to process CPU, memory, and disk metrics to avoid blocking the main asynchronous tasks
|
||||
let cpu_mem_task = tokio::spawn(async move {
|
||||
loop {
|
||||
sleep(Duration::from_millis(interval)).await;
|
||||
if let Err(e) = update_process_metrics(
|
||||
&mut sys,
|
||||
pid,
|
||||
&process_cpu_usage,
|
||||
&process_cpu_utilization,
|
||||
&process_memory_usage,
|
||||
&process_memory_virtual,
|
||||
&process_disk_io,
|
||||
&common_attributes,
|
||||
core_count_f32,
|
||||
) {
|
||||
warn!("Failed to update process metrics: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Use another asynchronous task to handle GPU metrics
|
||||
#[cfg(feature = "gpu")]
|
||||
let gpu_task = tokio::spawn(async move {
|
||||
loop {
|
||||
sleep(Duration::from_millis(interval)).await;
|
||||
|
||||
// delayed initialization nvml
|
||||
let nvml_arc = get_or_init_nvml().await;
|
||||
let nvml_option = nvml_arc.lock().unwrap();
|
||||
|
||||
if let Err(e) = update_gpu_metrics(&nvml, pid, &process_gpu_memory_usage, &common_attributes) {
|
||||
warn!("Failed to update GPU metrics: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// record empty values when non gpu function
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
let gpu_task = tokio::spawn(async move {
|
||||
loop {
|
||||
sleep(Duration::from_millis(interval)).await;
|
||||
process_gpu_memory_usage.record(0, &common_attributes);
|
||||
}
|
||||
});
|
||||
|
||||
// Wait for the two tasks to complete (actually they will run forever)
|
||||
let _ = tokio::join!(cpu_mem_task, gpu_task);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_metrics(meter: &Meter) -> (F64Gauge, F64Gauge, I64Gauge, I64Gauge, I64Gauge, U64Gauge) {
|
||||
let process_cpu_utilization = meter
|
||||
.f64_gauge(PROCESS_CPU_USAGE)
|
||||
.with_description("The percentage of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
|
||||
let process_cpu_usage = meter
|
||||
.f64_gauge(PROCESS_CPU_UTILIZATION)
|
||||
.with_description("The amount of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
|
||||
let process_memory_usage = meter
|
||||
.i64_gauge(PROCESS_MEMORY_USAGE)
|
||||
.with_description("The amount of physical memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
let process_memory_virtual = meter
|
||||
.i64_gauge(PROCESS_MEMORY_VIRTUAL)
|
||||
.with_description("The amount of committed virtual memory.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
let process_disk_io = meter
|
||||
.i64_gauge(PROCESS_DISK_IO)
|
||||
.with_description("Disk bytes transferred.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
let process_gpu_memory_usage = meter
|
||||
.u64_gauge(PROCESS_GPU_MEMORY_USAGE)
|
||||
.with_description("The amount of physical GPU memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
(
|
||||
process_cpu_utilization,
|
||||
process_cpu_usage,
|
||||
process_memory_usage,
|
||||
process_memory_virtual,
|
||||
process_disk_io,
|
||||
process_gpu_memory_usage,
|
||||
)
|
||||
}
|
||||
|
||||
fn prepare_common_attributes(sys: &System, pid: sysinfo::Pid) -> Result<[KeyValue; 4], GlobalError> {
|
||||
let process = sys
|
||||
.process(pid)
|
||||
.ok_or_else(|| GlobalError::SystemMetricsError(format!("Process with PID {} not found", pid.as_u32())))?;
|
||||
|
||||
// optimize string operations and reduce allocation
|
||||
let cmd = process.cmd().iter().filter_map(|s| s.to_str()).collect::<Vec<_>>().join(" ");
|
||||
|
||||
let executable_path = process
|
||||
.exe()
|
||||
.map(|path| path.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
let name = process.name().to_os_string().into_string().unwrap_or_default();
|
||||
|
||||
Ok([
|
||||
KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
|
||||
KeyValue::new(PROCESS_EXECUTABLE_NAME, name),
|
||||
KeyValue::new(PROCESS_EXECUTABLE_PATH, executable_path),
|
||||
KeyValue::new(PROCESS_COMMAND, cmd),
|
||||
])
|
||||
}
|
||||
|
||||
fn get_export_interval() -> u64 {
|
||||
std::env::var("OTEL_METRIC_EXPORT_INTERVAL")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30000)
|
||||
}
|
||||
|
||||
fn update_process_metrics(
|
||||
sys: &mut System,
|
||||
pid: sysinfo::Pid,
|
||||
process_cpu_usage: &F64Gauge,
|
||||
process_cpu_utilization: &F64Gauge,
|
||||
process_memory_usage: &I64Gauge,
|
||||
process_memory_virtual: &I64Gauge,
|
||||
process_disk_io: &I64Gauge,
|
||||
common_attributes: &[KeyValue; 4],
|
||||
core_count: f32,
|
||||
) -> Result<(), GlobalError> {
|
||||
// Only refresh the data of the required process to reduce system call overhead
|
||||
sys.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
|
||||
|
||||
let process = match sys.process(pid) {
|
||||
Some(p) => p,
|
||||
None => {
|
||||
return Err(GlobalError::SystemMetricsError(format!(
|
||||
"Process with PID {} no longer exists",
|
||||
pid.as_u32()
|
||||
)))
|
||||
}
|
||||
};
|
||||
|
||||
// collect data in batches and record it again
|
||||
let cpu_usage = process.cpu_usage();
|
||||
process_cpu_usage.record(cpu_usage.into(), &[]);
|
||||
process_cpu_utilization.record((cpu_usage / core_count).into(), &common_attributes);
|
||||
|
||||
// safe type conversion
|
||||
let memory = process.memory();
|
||||
let virtual_memory = process.virtual_memory();
|
||||
|
||||
// Avoid multiple error checks and use .map_err to handle errors in chain
|
||||
let memory_i64 =
|
||||
i64::try_from(memory).map_err(|_| GlobalError::ConversionError("Failed to convert memory usage to i64".to_string()))?;
|
||||
|
||||
let virtual_memory_i64 = i64::try_from(virtual_memory)
|
||||
.map_err(|_| GlobalError::ConversionError("Failed to convert virtual memory to i64".to_string()))?;
|
||||
|
||||
process_memory_usage.record(memory_i64, common_attributes);
|
||||
process_memory_virtual.record(virtual_memory_i64, common_attributes);
|
||||
|
||||
// process disk io metrics
|
||||
let disk_io = process.disk_usage();
|
||||
|
||||
// batch conversion to reduce duplicate code
|
||||
let read_bytes_i64 = i64::try_from(disk_io.read_bytes)
|
||||
.map_err(|_| GlobalError::ConversionError("Failed to convert read bytes to i64".to_string()))?;
|
||||
|
||||
let written_bytes_i64 = i64::try_from(disk_io.written_bytes)
|
||||
.map_err(|_| GlobalError::ConversionError("Failed to convert written bytes to i64".to_string()))?;
|
||||
|
||||
// Optimize attribute array stitching to reduce heap allocation
|
||||
let mut read_attributes = [KeyValue::new(DIRECTION, "read")];
|
||||
let read_attrs = [common_attributes, &read_attributes].concat();
|
||||
|
||||
let mut write_attributes = [KeyValue::new(DIRECTION, "write")];
|
||||
let write_attrs = [common_attributes, &write_attributes].concat();
|
||||
|
||||
process_disk_io.record(read_bytes_i64, &read_attrs);
|
||||
process_disk_io.record(written_bytes_i64, &write_attrs);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// GPU metric update function, conditional compilation based on feature flags
|
||||
#[cfg(feature = "gpu")]
|
||||
fn update_gpu_metrics(
|
||||
nvml: &Result<Nvml, nvml_wrapper::error::NvmlError>,
|
||||
pid: sysinfo::Pid,
|
||||
process_gpu_memory_usage: &U64Gauge,
|
||||
common_attributes: &[KeyValue; 4],
|
||||
) -> Result<(), GlobalError> {
|
||||
match nvml {
|
||||
Ok(nvml) => {
|
||||
if let Ok(device) = nvml.device_by_index(0) {
|
||||
if let Ok(gpu_stats) = device.running_compute_processes() {
|
||||
for stat in gpu_stats.iter() {
|
||||
if stat.pid == pid.as_u32() {
|
||||
let memory_used = match stat.used_gpu_memory {
|
||||
UsedGpuMemory::Used(bytes) => bytes,
|
||||
UsedGpuMemory::Unavailable => 0,
|
||||
};
|
||||
|
||||
process_gpu_memory_usage.record(memory_used, common_attributes);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// If no GPU usage record of the process is found, the record is 0
|
||||
process_gpu_memory_usage.record(0, common_attributes);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Could not get NVML, recording 0 for GPU memory usage: {}", e);
|
||||
process_gpu_memory_usage.record(0, common_attributes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
fn update_gpu_metrics(
|
||||
_: &(), // blank placeholder parameters
|
||||
_: sysinfo::Pid,
|
||||
process_gpu_memory_usage: &U64Gauge,
|
||||
common_attributes: &[KeyValue; 4],
|
||||
) -> Result<(), GlobalError> {
|
||||
// always logged when non gpu function 0
|
||||
process_gpu_memory_usage.record(0, common_attributes);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use crate::GlobalError;
|
||||
use opentelemetry::KeyValue;
|
||||
use sysinfo::{Pid, System};
|
||||
|
||||
pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
|
||||
pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
|
||||
pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
|
||||
pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
|
||||
|
||||
/// Struct to hold process attributes
|
||||
pub struct ProcessAttributes {
|
||||
pub attributes: Vec<KeyValue>,
|
||||
}
|
||||
|
||||
impl ProcessAttributes {
|
||||
/// Creates a new instance of `ProcessAttributes` for the given PID.
|
||||
pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
|
||||
system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
|
||||
let process = system
|
||||
.process(pid)
|
||||
.ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
|
||||
|
||||
let attributes = vec![
|
||||
KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
|
||||
KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
|
||||
KeyValue::new(
|
||||
PROCESS_EXECUTABLE_PATH,
|
||||
process
|
||||
.exe()
|
||||
.map(|path| path.to_string_lossy().into_owned())
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
KeyValue::new(
|
||||
PROCESS_COMMAND,
|
||||
process
|
||||
.cmd()
|
||||
.iter()
|
||||
.fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
|
||||
),
|
||||
];
|
||||
|
||||
Ok(ProcessAttributes { attributes })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
use crate::system::attributes::ProcessAttributes;
|
||||
use crate::system::gpu::GpuCollector;
|
||||
use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
|
||||
use crate::GlobalError;
|
||||
use opentelemetry::KeyValue;
|
||||
use std::time::SystemTime;
|
||||
use sysinfo::{Networks, Pid, ProcessStatus, System};
|
||||
use tokio::time::{sleep, Duration};
|
||||
|
||||
/// Collector is responsible for collecting system metrics and attributes.
|
||||
/// It uses the sysinfo crate to gather information about the system and processes.
|
||||
/// It also uses OpenTelemetry to record metrics.
|
||||
pub struct Collector {
|
||||
metrics: Metrics,
|
||||
attributes: ProcessAttributes,
|
||||
gpu_collector: GpuCollector,
|
||||
pid: Pid,
|
||||
system: System,
|
||||
networks: Networks,
|
||||
core_count: usize,
|
||||
interval_ms: u64,
|
||||
}
|
||||
|
||||
impl Collector {
|
||||
pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
|
||||
let mut system = System::new_all();
|
||||
let attributes = ProcessAttributes::new(pid, &mut system)?;
|
||||
let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
|
||||
let metrics = Metrics::new(&meter);
|
||||
let gpu_collector = GpuCollector::new(pid)?;
|
||||
let networks = Networks::new_with_refreshed_list();
|
||||
|
||||
Ok(Collector {
|
||||
metrics,
|
||||
attributes,
|
||||
gpu_collector,
|
||||
pid,
|
||||
system,
|
||||
networks,
|
||||
core_count,
|
||||
interval_ms,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run(&mut self) -> Result<(), GlobalError> {
|
||||
loop {
|
||||
self.collect()?;
|
||||
tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
|
||||
sleep(Duration::from_millis(self.interval_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn collect(&mut self) -> Result<(), GlobalError> {
|
||||
self.system
|
||||
.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
|
||||
|
||||
// 刷新网络接口列表和统计数据
|
||||
self.networks.refresh(false); // 刷新网络统计数据
|
||||
|
||||
let process = self
|
||||
.system
|
||||
.process(self.pid)
|
||||
.ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
|
||||
|
||||
// CPU 指标
|
||||
let cpu_usage = process.cpu_usage();
|
||||
self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
|
||||
self.metrics
|
||||
.cpu_utilization
|
||||
.record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
|
||||
|
||||
// 内存指标
|
||||
self.metrics
|
||||
.memory_usage
|
||||
.record(process.memory() as i64, &self.attributes.attributes);
|
||||
self.metrics
|
||||
.memory_virtual
|
||||
.record(process.virtual_memory() as i64, &self.attributes.attributes);
|
||||
|
||||
// 磁盘I/O指标
|
||||
let disk_io = process.disk_usage();
|
||||
self.metrics.disk_io.record(
|
||||
disk_io.read_bytes as i64,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
|
||||
);
|
||||
self.metrics.disk_io.record(
|
||||
disk_io.written_bytes as i64,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
|
||||
);
|
||||
|
||||
// 网络I/O指标（对应 /system/network/internode）
|
||||
let mut total_received: i64 = 0;
|
||||
let mut total_transmitted: i64 = 0;
|
||||
|
||||
// 按接口统计
|
||||
for (interface_name, data) in self.networks.iter() {
|
||||
total_received += data.total_received() as i64;
|
||||
total_transmitted += data.total_transmitted() as i64;
|
||||
|
||||
let received = data.received() as i64;
|
||||
let transmitted = data.transmitted() as i64;
|
||||
self.metrics.network_io_per_interface.record(
|
||||
received,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[
|
||||
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||
KeyValue::new(DIRECTION, "received"),
|
||||
],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
self.metrics.network_io_per_interface.record(
|
||||
transmitted,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[
|
||||
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||
KeyValue::new(DIRECTION, "transmitted"),
|
||||
],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
}
|
||||
// 全局统计
|
||||
self.metrics.network_io.record(
|
||||
total_received,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
|
||||
);
|
||||
self.metrics.network_io.record(
|
||||
total_transmitted,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
|
||||
);
|
||||
|
||||
// 进程状态指标（对应 /system/process）
|
||||
let status_value = match process.status() {
|
||||
ProcessStatus::Run => 0,
|
||||
ProcessStatus::Sleep => 1,
|
||||
ProcessStatus::Zombie => 2,
|
||||
_ => 3, // 其他状态
|
||||
};
|
||||
self.metrics.process_status.record(
|
||||
status_value,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[KeyValue::new(STATUS, format!("{:?}", process.status()))],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
|
||||
// GPU 指标（可选）
|
||||
self.gpu_collector.collect(&self.metrics, &self.attributes)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::system::attributes::ProcessAttributes;
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::system::metrics::Metrics;
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::GlobalError;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::enums::device::UsedGpuMemory;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::Nvml;
|
||||
#[cfg(feature = "gpu")]
|
||||
use sysinfo::Pid;
|
||||
#[cfg(feature = "gpu")]
|
||||
use tracing::warn;
|
||||
|
||||
/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
|
||||
#[cfg(feature = "gpu")]
|
||||
pub struct GpuCollector {
|
||||
nvml: Nvml,
|
||||
pid: Pid,
|
||||
}
|
||||
|
||||
#[cfg(feature = "gpu")]
|
||||
impl GpuCollector {
|
||||
pub fn new(pid: Pid) -> Result<Self, GlobalError> {
|
||||
let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
|
||||
Ok(GpuCollector { nvml, pid })
|
||||
}
|
||||
|
||||
pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
|
||||
if let Ok(device) = self.nvml.device_by_index(0) {
|
||||
if let Ok(gpu_stats) = device.running_compute_processes() {
|
||||
for stat in gpu_stats.iter() {
|
||||
if stat.pid == self.pid.as_u32() {
|
||||
let memory_used = match stat.used_gpu_memory {
|
||||
UsedGpuMemory::Used(bytes) => bytes,
|
||||
UsedGpuMemory::Unavailable => 0,
|
||||
};
|
||||
metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Could not get GPU stats, recording 0 for GPU memory usage");
|
||||
}
|
||||
} else {
|
||||
return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
|
||||
}
|
||||
metrics.gpu_memory_usage.record(0, &attributes.attributes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
pub struct GpuCollector;
|
||||
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
impl GpuCollector {
|
||||
pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
|
||||
Ok(GpuCollector)
|
||||
}
|
||||
|
||||
pub fn collect(
|
||||
&self,
|
||||
_metrics: &crate::system::metrics::Metrics,
|
||||
_attributes: &crate::system::attributes::ProcessAttributes,
|
||||
) -> Result<(), crate::GlobalError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
|
||||
pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
|
||||
pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
|
||||
pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
|
||||
pub const PROCESS_DISK_IO: &str = "process.disk.io";
|
||||
pub const PROCESS_NETWORK_IO: &str = "process.network.io";
|
||||
pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
|
||||
pub const PROCESS_STATUS: &str = "process.status";
|
||||
#[cfg(feature = "gpu")]
|
||||
pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
|
||||
pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
|
||||
pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
|
||||
pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
|
||||
|
||||
/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
|
||||
/// It contains various metrics such as CPU usage, memory usage,
|
||||
/// disk I/O, network I/O, and process status.
|
||||
///
|
||||
/// The `Metrics` struct is designed to be used with OpenTelemetry's
|
||||
/// metrics API to record and export these metrics.
|
||||
///
|
||||
/// The `new` method initializes the metrics using the provided
|
||||
/// `opentelemetry::metrics::Meter`.
|
||||
pub struct Metrics {
|
||||
pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
|
||||
pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
|
||||
pub memory_usage: opentelemetry::metrics::Gauge<i64>,
|
||||
pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
|
||||
pub disk_io: opentelemetry::metrics::Gauge<i64>,
|
||||
pub network_io: opentelemetry::metrics::Gauge<i64>,
|
||||
pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
|
||||
pub process_status: opentelemetry::metrics::Gauge<i64>,
|
||||
#[cfg(feature = "gpu")]
|
||||
pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
|
||||
let cpu_usage = meter
|
||||
.f64_gauge(PROCESS_CPU_USAGE)
|
||||
.with_description("The percentage of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
let cpu_utilization = meter
|
||||
.f64_gauge(PROCESS_CPU_UTILIZATION)
|
||||
.with_description("The amount of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
let memory_usage = meter
|
||||
.i64_gauge(PROCESS_MEMORY_USAGE)
|
||||
.with_description("The amount of physical memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let memory_virtual = meter
|
||||
.i64_gauge(PROCESS_MEMORY_VIRTUAL)
|
||||
.with_description("The amount of committed virtual memory.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let disk_io = meter
|
||||
.i64_gauge(PROCESS_DISK_IO)
|
||||
.with_description("Disk bytes transferred.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let network_io = meter
|
||||
.i64_gauge(PROCESS_NETWORK_IO)
|
||||
.with_description("Network bytes transferred.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let network_io_per_interface = meter
|
||||
.i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
|
||||
.with_description("Network bytes transferred (per interface).")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
let process_status = meter
|
||||
.i64_gauge(PROCESS_STATUS)
|
||||
.with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
|
||||
.build();
|
||||
|
||||
#[cfg(feature = "gpu")]
|
||||
let gpu_memory_usage = meter
|
||||
.u64_gauge(PROCESS_GPU_MEMORY_USAGE)
|
||||
.with_description("The amount of physical GPU memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
Metrics {
|
||||
cpu_usage,
|
||||
cpu_utilization,
|
||||
memory_usage,
|
||||
memory_virtual,
|
||||
disk_io,
|
||||
network_io,
|
||||
network_io_per_interface,
|
||||
process_status,
|
||||
#[cfg(feature = "gpu")]
|
||||
gpu_memory_usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
use crate::GlobalError;
|
||||
|
||||
pub(crate) mod attributes;
|
||||
mod collector;
|
||||
pub(crate) mod gpu;
|
||||
pub(crate) mod metrics;
|
||||
|
||||
/// Initialize the indicator collector for the current process
|
||||
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||
/// It will run indefinitely until the process is terminated.
|
||||
pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
|
||||
let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
|
||||
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||
collector.run().await
|
||||
}
|
||||
|
||||
/// Initialize the metric collector for the specified PID process
|
||||
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||
/// It will run indefinitely until the process is terminated.
|
||||
pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
|
||||
let pid = sysinfo::Pid::from_u32(pid);
|
||||
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||
collector.run().await
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
|
||||
use crate::{get_local_ip_with_default, OtelConfig};
|
||||
use crate::utils::get_local_ip_with_default;
|
||||
use crate::OtelConfig;
|
||||
use opentelemetry::trace::TracerProvider;
|
||||
use opentelemetry::{global, KeyValue};
|
||||
use opentelemetry_appender_tracing::layer;
|
||||
|
||||
Reference in New Issue
Block a user