mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 23:56:53 +00:00
feat: add TraceLayer for HTTP service and improve metrics (#361)
* improve code for opentelemetry and add system metrics * feat: add TraceLayer for HTTP service and improve metrics - Add TraceLayer to HTTP server for request tracing - Implement system metrics for process monitoring - Optimize init_telemetry method for better resource management - Add graceful shutdown handling for telemetry components - Fix GracefulShutdown ownership issues with Arc wrapper * improve code for init_process_observer * remove tomlfmt.toml * Translation comment * improve code for console CompressionLayer params
This commit is contained in:
houseme
@@ -149,7 +149,6 @@ impl NotifierConfig {
|
||||
)
|
||||
.build()
|
||||
.unwrap_or_default();
|
||||
println!("Loaded config: {:?}", app_config);
|
||||
match app_config.try_deserialize::<NotifierConfig>() {
|
||||
Ok(app_config) => {
|
||||
println!("Parsed AppConfig: {:?} \n", app_config);
|
||||
|
||||
@@ -11,32 +11,35 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = ["file"]
|
||||
file = []
|
||||
gpu = ["dep:nvml-wrapper"]
|
||||
kafka = ["dep:rdkafka"]
|
||||
webhook = ["dep:reqwest"]
|
||||
file = []
|
||||
full = ["file", "gpu", "kafka", "webhook"]
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
config = { workspace = true }
|
||||
nvml-wrapper = { workspace = true, optional = true }
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
opentelemetry-stdout = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
|
||||
opentelemetry-prometheus = { workspace = true }
|
||||
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
||||
prometheus = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
tracing-core = { workspace = true }
|
||||
tracing-error = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
||||
reqwest = { workspace = true, optional = true, default-features = false }
|
||||
serde_json = { workspace = true }
|
||||
sysinfo = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
local-ip-address = { workspace = true }
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ batch_timeout_ms = 1000 # Default is 100ms if not specified
|
||||
|
||||
[sinks.file]
|
||||
enabled = true
|
||||
path = "logs/app.log"
|
||||
path = "deploy/logs/app.log"
|
||||
batch_size = 100
|
||||
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use opentelemetry::global;
|
||||
use rustfs_obs::{get_logger, init_obs, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
||||
use rustfs_obs::{get_logger, init_obs, init_process_observer, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{info, instrument};
|
||||
use tracing::{error, info, instrument};
|
||||
use tracing_core::Level;
|
||||
|
||||
#[tokio::main]
|
||||
@@ -38,6 +38,11 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
|
||||
&[opentelemetry::KeyValue::new("operation", "run")],
|
||||
);
|
||||
|
||||
match init_process_observer(meter).await {
|
||||
Ok(_) => info!("Process observer initialized successfully"),
|
||||
Err(e) => error!("Failed to initialize process observer: {:?}", e),
|
||||
}
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.message(Some("run logger api_handler info".to_string()))
|
||||
.request_id(Some("request_id".to_string()))
|
||||
|
||||
@@ -178,7 +178,6 @@ pub struct AppConfig {
|
||||
pub logger: Option<LoggerConfig>,
|
||||
}
|
||||
|
||||
// 为 AppConfig 实现 Default
|
||||
impl AppConfig {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
@@ -189,6 +188,7 @@ impl AppConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// implement default for AppConfig
|
||||
impl Default for AppConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
|
||||
@@ -16,11 +16,27 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
|
||||
|
||||
/// Error type for global guard operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum GuardError {
|
||||
pub enum GlobalError {
|
||||
#[error("Failed to set global guard: {0}")]
|
||||
SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
|
||||
#[error("Global guard not initialized")]
|
||||
NotInitialized,
|
||||
#[error("Global system metrics err: {0}")]
|
||||
MetricsError(String),
|
||||
#[error("Failed to get current PID: {0}")]
|
||||
PidError(String),
|
||||
#[error("Process with PID {0} not found")]
|
||||
ProcessNotFound(u32),
|
||||
#[error("Failed to get physical core count")]
|
||||
CoreCountError,
|
||||
#[error("GPU initialization failed: {0}")]
|
||||
GpuInitError(String),
|
||||
#[error("GPU device not found: {0}")]
|
||||
GpuDeviceError(String),
|
||||
#[error("Failed to send log: {0}")]
|
||||
SendFailed(&'static str),
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(&'static str),
|
||||
}
|
||||
|
||||
/// Set the global guard for OpenTelemetry
|
||||
@@ -43,9 +59,9 @@ pub enum GuardError {
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
|
||||
pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
|
||||
info!("Initializing global OpenTelemetry guard");
|
||||
GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GuardError::SetError)
|
||||
GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
|
||||
}
|
||||
|
||||
/// Get the global guard for OpenTelemetry
|
||||
@@ -65,8 +81,8 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GuardError> {
|
||||
GLOBAL_GUARD.get().cloned().ok_or(GuardError::NotInitialized)
|
||||
pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
|
||||
GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
|
||||
}
|
||||
|
||||
/// Try to get the global guard for OpenTelemetry
|
||||
@@ -84,6 +100,6 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_get_uninitialized_guard() {
|
||||
let result = get_global_guard();
|
||||
assert!(matches!(result, Err(GuardError::NotInitialized)));
|
||||
assert!(matches!(result, Err(GlobalError::NotInitialized)));
|
||||
}
|
||||
}
|
||||
|
||||
+50
-30
@@ -1,23 +1,24 @@
|
||||
/// # obs
|
||||
///
|
||||
/// `obs` is a logging and observability library for Rust.
|
||||
/// It provides a simple and easy-to-use interface for logging and observability.
|
||||
/// It is built on top of the `log` crate and `opentelemetry` crate.
|
||||
///
|
||||
/// ## Features
|
||||
/// - Structured logging
|
||||
/// - Distributed tracing
|
||||
/// - Metrics collection
|
||||
/// - Log processing worker
|
||||
/// - Multiple sinks
|
||||
/// - Configuration-based setup
|
||||
/// - Telemetry guard
|
||||
/// - Global logger
|
||||
/// - Log levels
|
||||
/// - Log entry types
|
||||
/// - Log record
|
||||
/// - Object version
|
||||
/// - Local IP address
|
||||
//! # RustFS Observability
|
||||
//!
|
||||
//! provides tools for system and service monitoring
|
||||
//!
|
||||
//! ## feature mark
|
||||
//!
|
||||
//! - `file`: enable file logging enabled by default
|
||||
//! - `gpu`: gpu monitoring function
|
||||
//! - `kafka`: enable kafka metric output
|
||||
//! - `webhook`: enable webhook notifications
|
||||
//! - `full`: includes all functions
|
||||
//!
|
||||
//! to enable gpu monitoring add in cargo toml
|
||||
//!
|
||||
//! ```toml
|
||||
//! # using gpu monitoring
|
||||
//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
|
||||
//!
|
||||
//! # use all functions
|
||||
//! rustfs-obs = { version = "0.1.0", features = ["full"] }
|
||||
//! ```
|
||||
///
|
||||
/// ## Usage
|
||||
///
|
||||
@@ -32,28 +33,34 @@ mod entry;
|
||||
mod global;
|
||||
mod logger;
|
||||
mod sink;
|
||||
mod system;
|
||||
mod telemetry;
|
||||
mod utils;
|
||||
mod worker;
|
||||
|
||||
use crate::logger::InitLogStatus;
|
||||
pub use config::load_config;
|
||||
pub use config::{AppConfig, OtelConfig};
|
||||
#[cfg(feature = "file")]
|
||||
pub use config::FileSinkConfig;
|
||||
#[cfg(feature = "kafka")]
|
||||
pub use config::KafkaSinkConfig;
|
||||
#[cfg(feature = "webhook")]
|
||||
pub use config::WebhookSinkConfig;
|
||||
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
|
||||
pub use entry::args::Args;
|
||||
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
||||
pub use entry::base::BaseLogEntry;
|
||||
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
||||
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
||||
pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GuardError};
|
||||
pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
|
||||
pub use logger::{log_init_state, InitLogStatus};
|
||||
pub use logger::{LogError, Logger};
|
||||
pub use sink::Sink;
|
||||
pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
|
||||
pub use logger::Logger;
|
||||
pub use logger::{get_global_logger, init_global_logger, start_logger};
|
||||
pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
use std::sync::Arc;
|
||||
pub use system::{init_process_observer, init_process_observer_for_pid};
|
||||
pub use telemetry::init_telemetry;
|
||||
pub use telemetry::{get_global_registry, metrics};
|
||||
use tokio::sync::Mutex;
|
||||
pub use utils::{get_local_ip, get_local_ip_with_default};
|
||||
pub use worker::start_worker;
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Initialize the observability module
|
||||
///
|
||||
@@ -74,6 +81,19 @@ pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::Otel
|
||||
let guard = init_telemetry(&config.observability);
|
||||
let sinks = sink::create_sinks(&config).await;
|
||||
let logger = init_global_logger(&config, sinks).await;
|
||||
let obs_config = config.observability.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = InitLogStatus::init_start_log(&obs_config).await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Logger initialized successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to initialize logger: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(logger, guard)
|
||||
}
|
||||
|
||||
|
||||
+21
-81
@@ -1,5 +1,6 @@
|
||||
use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
|
||||
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, OtelConfig, ServerLogEntry, Sink, UnifiedLogEntry};
|
||||
use crate::sink::Sink;
|
||||
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
@@ -43,26 +44,26 @@ impl Logger {
|
||||
|
||||
/// Log a server entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
||||
}
|
||||
|
||||
/// Log an audit entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
||||
}
|
||||
|
||||
/// Log a console entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
||||
}
|
||||
|
||||
/// Asynchronous logging of unified log entries
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger"))]
|
||||
#[tracing::instrument(level = "error", skip_all)]
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), LogError> {
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
|
||||
// Extract information for tracing based on entry type
|
||||
match &entry {
|
||||
UnifiedLogEntry::Server(server) => {
|
||||
@@ -123,11 +124,11 @@ impl Logger {
|
||||
tracing::warn!("Log queue full, applying backpressure");
|
||||
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
Ok(Err(_)) => Err(LogError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(LogError::Timeout("Queue backpressure timeout")),
|
||||
Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
|
||||
}
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(LogError::SendFailed("Logger channel closed")),
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +161,7 @@ impl Logger {
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), LogError> {
|
||||
) -> Result<(), GlobalError> {
|
||||
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
||||
|
||||
let server_entry = ServerLogEntry::new(level, source.to_string())
|
||||
@@ -190,7 +191,7 @@ impl Logger {
|
||||
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), LogError> {
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
|
||||
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
||||
}
|
||||
|
||||
@@ -208,31 +209,12 @@ impl Logger {
|
||||
/// let _ = logger.shutdown().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn shutdown(self) -> Result<(), LogError> {
|
||||
pub async fn shutdown(self) -> Result<(), GlobalError> {
|
||||
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Log error type
|
||||
/// This enum defines the error types that can occur when logging.
|
||||
/// It is used to provide more detailed error information.
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::LogError;
|
||||
/// use thiserror::Error;
|
||||
///
|
||||
/// LogError::SendFailed("Failed to send log");
|
||||
/// LogError::Timeout("Operation timed out");
|
||||
/// ```
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LogError {
|
||||
#[error("Failed to send log: {0}")]
|
||||
SendFailed(&'static str),
|
||||
#[error("Operation timed out: {0}")]
|
||||
Timeout(&'static str),
|
||||
}
|
||||
|
||||
/// Start the log module
|
||||
/// This function starts the log module.
|
||||
/// It initializes the logger and starts the worker to process logs.
|
||||
@@ -297,48 +279,6 @@ pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
|
||||
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
||||
}
|
||||
|
||||
/// Get the global logger instance with a lock
|
||||
/// This function returns a reference to the global logger instance with a lock.
|
||||
/// It is used to ensure that the logger is thread-safe.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance with a lock
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::locked_logger;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let logger = locked_logger().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn locked_logger() -> tokio::sync::MutexGuard<'static, Logger> {
|
||||
get_global_logger().lock().await
|
||||
}
|
||||
|
||||
/// Initialize with default empty logger if needed (optional)
|
||||
/// This function initializes the logger with a default empty logger if needed.
|
||||
/// It is used to ensure that the logger is initialized before logging.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ensure_logger_initialized;
|
||||
///
|
||||
/// let logger = ensure_logger_initialized();
|
||||
/// ```
|
||||
pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
||||
if GLOBAL_LOGGER.get().is_none() {
|
||||
let config = AppConfig::default();
|
||||
let sinks = vec![];
|
||||
let logger = Arc::new(Mutex::new(start_logger(&config, sinks)));
|
||||
let _ = GLOBAL_LOGGER.set(logger);
|
||||
}
|
||||
GLOBAL_LOGGER.get().unwrap()
|
||||
}
|
||||
|
||||
/// Log information
|
||||
/// This function logs information messages.
|
||||
///
|
||||
@@ -357,7 +297,7 @@ pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
||||
/// let _ = log_info("This is an information message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
||||
}
|
||||
|
||||
@@ -375,7 +315,7 @@ pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// async fn example() {
|
||||
/// let _ = log_error("This is an error message", "example").await;
|
||||
/// }
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
||||
}
|
||||
|
||||
@@ -395,7 +335,7 @@ pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_warn("This is a warning message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
||||
}
|
||||
|
||||
@@ -415,7 +355,7 @@ pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_debug("This is a debug message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
||||
}
|
||||
|
||||
@@ -436,7 +376,7 @@ pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
||||
/// let _ = log_trace("This is a trace message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), LogError> {
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
||||
}
|
||||
|
||||
@@ -467,7 +407,7 @@ pub async fn log_with_context(
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), LogError> {
|
||||
) -> Result<(), GlobalError> {
|
||||
get_global_logger()
|
||||
.lock()
|
||||
.await
|
||||
@@ -477,7 +417,7 @@ pub async fn log_with_context(
|
||||
|
||||
/// Log initialization status
|
||||
#[derive(Debug)]
|
||||
pub struct InitLogStatus {
|
||||
pub(crate) struct InitLogStatus {
|
||||
pub timestamp: SystemTime,
|
||||
pub service_name: String,
|
||||
pub version: String,
|
||||
@@ -508,14 +448,14 @@ impl InitLogStatus {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), LogError> {
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
|
||||
let status = Self::new_config(config);
|
||||
log_init_state(Some(status)).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Log initialization details during system startup
|
||||
pub async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), LogError> {
|
||||
async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
|
||||
let status = status.unwrap_or_default();
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
use crate::GlobalError;
|
||||
use opentelemetry::KeyValue;
|
||||
use sysinfo::{Pid, System};
|
||||
|
||||
pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
|
||||
pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
|
||||
pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
|
||||
pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
|
||||
|
||||
/// Struct to hold process attributes
|
||||
pub struct ProcessAttributes {
|
||||
pub attributes: Vec<KeyValue>,
|
||||
}
|
||||
|
||||
impl ProcessAttributes {
|
||||
/// Creates a new instance of `ProcessAttributes` for the given PID.
|
||||
pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
|
||||
system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
|
||||
let process = system
|
||||
.process(pid)
|
||||
.ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
|
||||
|
||||
let attributes = vec![
|
||||
KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
|
||||
KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
|
||||
KeyValue::new(
|
||||
PROCESS_EXECUTABLE_PATH,
|
||||
process
|
||||
.exe()
|
||||
.map(|path| path.to_string_lossy().into_owned())
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
KeyValue::new(
|
||||
PROCESS_COMMAND,
|
||||
process
|
||||
.cmd()
|
||||
.iter()
|
||||
.fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
|
||||
),
|
||||
];
|
||||
|
||||
Ok(ProcessAttributes { attributes })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
use crate::system::attributes::ProcessAttributes;
|
||||
use crate::system::gpu::GpuCollector;
|
||||
use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
|
||||
use crate::GlobalError;
|
||||
use opentelemetry::KeyValue;
|
||||
use std::time::SystemTime;
|
||||
use sysinfo::{Networks, Pid, ProcessStatus, System};
|
||||
use tokio::time::{sleep, Duration};
|
||||
|
||||
/// Collector is responsible for collecting system metrics and attributes.
|
||||
/// It uses the sysinfo crate to gather information about the system and processes.
|
||||
/// It also uses OpenTelemetry to record metrics.
|
||||
pub struct Collector {
|
||||
metrics: Metrics,
|
||||
attributes: ProcessAttributes,
|
||||
gpu_collector: GpuCollector,
|
||||
pid: Pid,
|
||||
system: System,
|
||||
networks: Networks,
|
||||
core_count: usize,
|
||||
interval_ms: u64,
|
||||
}
|
||||
|
||||
impl Collector {
|
||||
pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
|
||||
let mut system = System::new_all();
|
||||
let attributes = ProcessAttributes::new(pid, &mut system)?;
|
||||
let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
|
||||
let metrics = Metrics::new(&meter);
|
||||
let gpu_collector = GpuCollector::new(pid)?;
|
||||
let networks = Networks::new_with_refreshed_list();
|
||||
|
||||
Ok(Collector {
|
||||
metrics,
|
||||
attributes,
|
||||
gpu_collector,
|
||||
pid,
|
||||
system,
|
||||
networks,
|
||||
core_count,
|
||||
interval_ms,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run(&mut self) -> Result<(), GlobalError> {
|
||||
loop {
|
||||
self.collect()?;
|
||||
tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
|
||||
sleep(Duration::from_millis(self.interval_ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn collect(&mut self) -> Result<(), GlobalError> {
|
||||
self.system
|
||||
.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
|
||||
|
||||
// refresh the network interface list and statistics
|
||||
self.networks.refresh(false);
|
||||
|
||||
let process = self
|
||||
.system
|
||||
.process(self.pid)
|
||||
.ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
|
||||
|
||||
// CPU metrics
|
||||
let cpu_usage = process.cpu_usage();
|
||||
self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
|
||||
self.metrics
|
||||
.cpu_utilization
|
||||
.record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
|
||||
|
||||
// Memory metrics
|
||||
self.metrics
|
||||
.memory_usage
|
||||
.record(process.memory() as i64, &self.attributes.attributes);
|
||||
self.metrics
|
||||
.memory_virtual
|
||||
.record(process.virtual_memory() as i64, &self.attributes.attributes);
|
||||
|
||||
// Disk I/O metrics
|
||||
let disk_io = process.disk_usage();
|
||||
self.metrics.disk_io.record(
|
||||
disk_io.read_bytes as i64,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
|
||||
);
|
||||
self.metrics.disk_io.record(
|
||||
disk_io.written_bytes as i64,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
|
||||
);
|
||||
|
||||
// Network I/O indicators (corresponding to /system/network/internode)
|
||||
let mut total_received: i64 = 0;
|
||||
let mut total_transmitted: i64 = 0;
|
||||
|
||||
// statistics by interface
|
||||
for (interface_name, data) in self.networks.iter() {
|
||||
total_received += data.total_received() as i64;
|
||||
total_transmitted += data.total_transmitted() as i64;
|
||||
|
||||
let received = data.received() as i64;
|
||||
let transmitted = data.transmitted() as i64;
|
||||
self.metrics.network_io_per_interface.record(
|
||||
received,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[
|
||||
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||
KeyValue::new(DIRECTION, "received"),
|
||||
],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
self.metrics.network_io_per_interface.record(
|
||||
transmitted,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[
|
||||
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||
KeyValue::new(DIRECTION, "transmitted"),
|
||||
],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
}
|
||||
// global statistics
|
||||
self.metrics.network_io.record(
|
||||
total_received,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
|
||||
);
|
||||
self.metrics.network_io.record(
|
||||
total_transmitted,
|
||||
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
|
||||
);
|
||||
|
||||
// Process status indicator (corresponding to /system/process)
|
||||
let status_value = match process.status() {
|
||||
ProcessStatus::Run => 0,
|
||||
ProcessStatus::Sleep => 1,
|
||||
ProcessStatus::Zombie => 2,
|
||||
_ => 3, // other status
|
||||
};
|
||||
self.metrics.process_status.record(
|
||||
status_value,
|
||||
&[
|
||||
&self.attributes.attributes[..],
|
||||
&[KeyValue::new(STATUS, format!("{:?}", process.status()))],
|
||||
]
|
||||
.concat(),
|
||||
);
|
||||
|
||||
// GPU Metrics (Optional) Non-MacOS
|
||||
self.gpu_collector.collect(&self.metrics, &self.attributes)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::system::attributes::ProcessAttributes;
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::system::metrics::Metrics;
|
||||
#[cfg(feature = "gpu")]
|
||||
use crate::GlobalError;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::enums::device::UsedGpuMemory;
|
||||
#[cfg(feature = "gpu")]
|
||||
use nvml_wrapper::Nvml;
|
||||
#[cfg(feature = "gpu")]
|
||||
use sysinfo::Pid;
|
||||
#[cfg(feature = "gpu")]
|
||||
use tracing::warn;
|
||||
|
||||
/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
|
||||
#[cfg(feature = "gpu")]
|
||||
pub struct GpuCollector {
|
||||
nvml: Nvml,
|
||||
pid: Pid,
|
||||
}
|
||||
|
||||
#[cfg(feature = "gpu")]
|
||||
impl GpuCollector {
|
||||
pub fn new(pid: Pid) -> Result<Self, GlobalError> {
|
||||
let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
|
||||
Ok(GpuCollector { nvml, pid })
|
||||
}
|
||||
|
||||
pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
|
||||
if let Ok(device) = self.nvml.device_by_index(0) {
|
||||
if let Ok(gpu_stats) = device.running_compute_processes() {
|
||||
for stat in gpu_stats.iter() {
|
||||
if stat.pid == self.pid.as_u32() {
|
||||
let memory_used = match stat.used_gpu_memory {
|
||||
UsedGpuMemory::Used(bytes) => bytes,
|
||||
UsedGpuMemory::Unavailable => 0,
|
||||
};
|
||||
metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Could not get GPU stats, recording 0 for GPU memory usage");
|
||||
}
|
||||
} else {
|
||||
return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
|
||||
}
|
||||
metrics.gpu_memory_usage.record(0, &attributes.attributes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
pub struct GpuCollector;
|
||||
|
||||
#[cfg(not(feature = "gpu"))]
|
||||
impl GpuCollector {
|
||||
pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
|
||||
Ok(GpuCollector)
|
||||
}
|
||||
|
||||
pub fn collect(
|
||||
&self,
|
||||
_metrics: &crate::system::metrics::Metrics,
|
||||
_attributes: &crate::system::attributes::ProcessAttributes,
|
||||
) -> Result<(), crate::GlobalError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
|
||||
pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
|
||||
pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
|
||||
pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
|
||||
pub const PROCESS_DISK_IO: &str = "process.disk.io";
|
||||
pub const PROCESS_NETWORK_IO: &str = "process.network.io";
|
||||
pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
|
||||
pub const PROCESS_STATUS: &str = "process.status";
|
||||
#[cfg(feature = "gpu")]
|
||||
pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
|
||||
pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
|
||||
pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
|
||||
pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
|
||||
|
||||
/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
|
||||
/// It contains various metrics such as CPU usage, memory usage,
|
||||
/// disk I/O, network I/O, and process status.
|
||||
///
|
||||
/// The `Metrics` struct is designed to be used with OpenTelemetry's
|
||||
/// metrics API to record and export these metrics.
|
||||
///
|
||||
/// The `new` method initializes the metrics using the provided
|
||||
/// `opentelemetry::metrics::Meter`.
|
||||
pub struct Metrics {
|
||||
pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
|
||||
pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
|
||||
pub memory_usage: opentelemetry::metrics::Gauge<i64>,
|
||||
pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
|
||||
pub disk_io: opentelemetry::metrics::Gauge<i64>,
|
||||
pub network_io: opentelemetry::metrics::Gauge<i64>,
|
||||
pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
|
||||
pub process_status: opentelemetry::metrics::Gauge<i64>,
|
||||
#[cfg(feature = "gpu")]
|
||||
pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
|
||||
let cpu_usage = meter
|
||||
.f64_gauge(PROCESS_CPU_USAGE)
|
||||
.with_description("The percentage of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
let cpu_utilization = meter
|
||||
.f64_gauge(PROCESS_CPU_UTILIZATION)
|
||||
.with_description("The amount of CPU in use.")
|
||||
.with_unit("percent")
|
||||
.build();
|
||||
let memory_usage = meter
|
||||
.i64_gauge(PROCESS_MEMORY_USAGE)
|
||||
.with_description("The amount of physical memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let memory_virtual = meter
|
||||
.i64_gauge(PROCESS_MEMORY_VIRTUAL)
|
||||
.with_description("The amount of committed virtual memory.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let disk_io = meter
|
||||
.i64_gauge(PROCESS_DISK_IO)
|
||||
.with_description("Disk bytes transferred.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let network_io = meter
|
||||
.i64_gauge(PROCESS_NETWORK_IO)
|
||||
.with_description("Network bytes transferred.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
let network_io_per_interface = meter
|
||||
.i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
|
||||
.with_description("Network bytes transferred (per interface).")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
let process_status = meter
|
||||
.i64_gauge(PROCESS_STATUS)
|
||||
.with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
|
||||
.build();
|
||||
|
||||
#[cfg(feature = "gpu")]
|
||||
let gpu_memory_usage = meter
|
||||
.u64_gauge(PROCESS_GPU_MEMORY_USAGE)
|
||||
.with_description("The amount of physical GPU memory in use.")
|
||||
.with_unit("byte")
|
||||
.build();
|
||||
|
||||
Metrics {
|
||||
cpu_usage,
|
||||
cpu_utilization,
|
||||
memory_usage,
|
||||
memory_virtual,
|
||||
disk_io,
|
||||
network_io,
|
||||
network_io_per_interface,
|
||||
process_status,
|
||||
#[cfg(feature = "gpu")]
|
||||
gpu_memory_usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
use crate::GlobalError;
|
||||
|
||||
pub(crate) mod attributes;
|
||||
mod collector;
|
||||
pub(crate) mod gpu;
|
||||
pub(crate) mod metrics;
|
||||
|
||||
/// Initialize the indicator collector for the current process
|
||||
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||
/// It will run indefinitely until the process is terminated.
|
||||
pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
|
||||
let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
|
||||
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||
collector.run().await
|
||||
}
|
||||
|
||||
/// Initialize the metric collector for the specified PID process
|
||||
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||
/// It will run indefinitely until the process is terminated.
|
||||
pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
|
||||
let pid = sysinfo::Pid::from_u32(pid);
|
||||
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||
collector.run().await
|
||||
}
|
||||
+159
-202
@@ -1,5 +1,6 @@
|
||||
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
|
||||
use crate::{get_local_ip_with_default, OtelConfig};
|
||||
use crate::utils::get_local_ip_with_default;
|
||||
use crate::OtelConfig;
|
||||
use opentelemetry::trace::TracerProvider;
|
||||
use opentelemetry::{global, KeyValue};
|
||||
use opentelemetry_appender_tracing::layer;
|
||||
@@ -14,11 +15,10 @@ use opentelemetry_semantic_conventions::{
|
||||
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
|
||||
SCHEMA_URL,
|
||||
};
|
||||
use prometheus::{Encoder, Registry, TextEncoder};
|
||||
use smallvec::SmallVec;
|
||||
use std::borrow::Cow;
|
||||
use std::io::IsTerminal;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{Mutex, OnceCell};
|
||||
use tracing::{info, warn};
|
||||
use tracing::info;
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
|
||||
@@ -67,66 +67,20 @@ impl Drop for OtelGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Global registry for Prometheus metrics
|
||||
static GLOBAL_REGISTRY: OnceCell<Arc<Mutex<Registry>>> = OnceCell::const_new();
|
||||
|
||||
/// Get the global registry instance
|
||||
/// This function returns a reference to the global registry instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global registry instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::get_global_registry;
|
||||
///
|
||||
/// let registry = get_global_registry();
|
||||
/// ```
|
||||
pub fn get_global_registry() -> Arc<Mutex<Registry>> {
|
||||
GLOBAL_REGISTRY.get().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Prometheus metric endpoints
|
||||
/// This function returns a string containing the Prometheus metrics.
|
||||
/// The metrics are collected from the global registry.
|
||||
/// The function is used to expose the metrics via an HTTP endpoint.
|
||||
///
|
||||
/// # Returns
|
||||
/// A string containing the Prometheus metrics
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::metrics;
|
||||
///
|
||||
/// async fn main() {
|
||||
/// let metrics = metrics().await;
|
||||
/// println!("{}", metrics);
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn metrics() -> String {
|
||||
let encoder = TextEncoder::new();
|
||||
// Get a reference to the registry for reading metrics
|
||||
let registry = get_global_registry().lock().await.to_owned();
|
||||
let metric_families = registry.gather();
|
||||
if metric_families.is_empty() {
|
||||
warn!("No metrics available in Prometheus registry");
|
||||
} else {
|
||||
info!("Metrics collected: {} families", metric_families.len());
|
||||
}
|
||||
let mut buffer = Vec::new();
|
||||
encoder.encode(&metric_families, &mut buffer).unwrap();
|
||||
String::from_utf8(buffer).unwrap_or_else(|_| "Error encoding metrics".to_string())
|
||||
}
|
||||
|
||||
/// create OpenTelemetry Resource
|
||||
fn resource(config: &OtelConfig) -> Resource {
|
||||
let config = config.clone();
|
||||
Resource::builder()
|
||||
.with_service_name(config.service_name.unwrap_or(SERVICE_NAME.to_string()))
|
||||
.with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
|
||||
.with_schema_url(
|
||||
[
|
||||
KeyValue::new(OTEL_SERVICE_VERSION, config.service_version.unwrap_or(SERVICE_VERSION.to_string())),
|
||||
KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, config.environment.unwrap_or(ENVIRONMENT.to_string())),
|
||||
KeyValue::new(
|
||||
OTEL_SERVICE_VERSION,
|
||||
Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
|
||||
),
|
||||
KeyValue::new(
|
||||
DEPLOYMENT_ENVIRONMENT_NAME,
|
||||
Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
|
||||
),
|
||||
KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
|
||||
],
|
||||
SCHEMA_URL,
|
||||
@@ -141,167 +95,169 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Initialize Meter Provider
|
||||
fn init_meter_provider(config: &OtelConfig) -> SdkMeterProvider {
|
||||
let mut builder = MeterProviderBuilder::default().with_resource(resource(config));
|
||||
// If endpoint is empty, use stdout output
|
||||
if config.endpoint.is_empty() {
|
||||
builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
|
||||
} else {
|
||||
// If endpoint is not empty, use otlp output
|
||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(&config.endpoint)
|
||||
.with_temporality(opentelemetry_sdk::metrics::Temporality::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
builder = builder.with_reader(
|
||||
PeriodicReader::builder(exporter)
|
||||
.with_interval(std::time::Duration::from_secs(config.meter_interval.unwrap_or(METER_INTERVAL)))
|
||||
.build(),
|
||||
);
|
||||
// If use_stdout is true, output to stdout at the same time
|
||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
||||
builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
|
||||
}
|
||||
}
|
||||
let registry = Registry::new();
|
||||
// Set global registry
|
||||
GLOBAL_REGISTRY.set(Arc::new(Mutex::new(registry.clone()))).unwrap();
|
||||
// Create Prometheus exporter
|
||||
let prometheus_exporter = opentelemetry_prometheus::exporter().with_registry(registry).build().unwrap();
|
||||
// Build meter provider
|
||||
let meter_provider = builder.with_reader(prometheus_exporter).build();
|
||||
global::set_meter_provider(meter_provider.clone());
|
||||
meter_provider
|
||||
}
|
||||
|
||||
/// Initialize Tracer Provider
|
||||
fn init_tracer_provider(config: &OtelConfig) -> SdkTracerProvider {
|
||||
let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
|
||||
let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
|
||||
Sampler::TraceIdRatioBased(sample_ratio)
|
||||
} else {
|
||||
Sampler::AlwaysOn
|
||||
};
|
||||
let builder = SdkTracerProvider::builder()
|
||||
.with_sampler(sampler)
|
||||
.with_id_generator(RandomIdGenerator::default())
|
||||
.with_resource(resource(config));
|
||||
|
||||
let tracer_provider = if config.endpoint.is_empty() {
|
||||
builder
|
||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||
.build()
|
||||
} else {
|
||||
let exporter = opentelemetry_otlp::SpanExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(&config.endpoint)
|
||||
.build()
|
||||
.unwrap();
|
||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
||||
builder
|
||||
.with_batch_exporter(exporter)
|
||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||
} else {
|
||||
builder.with_batch_exporter(exporter)
|
||||
}
|
||||
.build()
|
||||
};
|
||||
|
||||
global::set_tracer_provider(tracer_provider.clone());
|
||||
tracer_provider
|
||||
}
|
||||
|
||||
/// Initialize Telemetry
|
||||
pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
||||
let tracer_provider = init_tracer_provider(config);
|
||||
let meter_provider = init_meter_provider(config);
|
||||
// avoid repeated access to configuration fields
|
||||
let endpoint = &config.endpoint;
|
||||
let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
|
||||
let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
|
||||
let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
|
||||
let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
|
||||
|
||||
// Initialize logger provider based on configuration
|
||||
let logger_provider = {
|
||||
let mut builder = SdkLoggerProvider::builder().with_resource(resource(config));
|
||||
// Pre-create resource objects to avoid repeated construction
|
||||
let res = resource(config);
|
||||
|
||||
if config.endpoint.is_empty() {
|
||||
// Use stdout exporter when no endpoint is configured
|
||||
builder = builder.with_simple_exporter(opentelemetry_stdout::LogExporter::default());
|
||||
// initialize tracer provider
|
||||
let tracer_provider = {
|
||||
let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
|
||||
let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
|
||||
Sampler::TraceIdRatioBased(sample_ratio)
|
||||
} else {
|
||||
Sampler::AlwaysOn
|
||||
};
|
||||
|
||||
let builder = SdkTracerProvider::builder()
|
||||
.with_sampler(sampler)
|
||||
.with_id_generator(RandomIdGenerator::default())
|
||||
.with_resource(res.clone());
|
||||
|
||||
let tracer_provider = if endpoint.is_empty() {
|
||||
builder
|
||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||
.build()
|
||||
} else {
|
||||
let exporter = opentelemetry_otlp::SpanExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let builder = if use_stdout {
|
||||
builder
|
||||
.with_batch_exporter(exporter)
|
||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||
} else {
|
||||
builder.with_batch_exporter(exporter)
|
||||
};
|
||||
|
||||
builder.build()
|
||||
};
|
||||
|
||||
global::set_tracer_provider(tracer_provider.clone());
|
||||
tracer_provider
|
||||
};
|
||||
|
||||
// initialize meter provider
|
||||
let meter_provider = {
|
||||
let mut builder = MeterProviderBuilder::default().with_resource(res.clone());
|
||||
|
||||
if endpoint.is_empty() {
|
||||
builder = builder.with_reader(create_periodic_reader(meter_interval));
|
||||
} else {
|
||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(endpoint)
|
||||
.with_temporality(opentelemetry_sdk::metrics::Temporality::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
builder = builder.with_reader(
|
||||
PeriodicReader::builder(exporter)
|
||||
.with_interval(std::time::Duration::from_secs(meter_interval))
|
||||
.build(),
|
||||
);
|
||||
|
||||
if use_stdout {
|
||||
builder = builder.with_reader(create_periodic_reader(meter_interval));
|
||||
}
|
||||
}
|
||||
|
||||
let meter_provider = builder.build();
|
||||
global::set_meter_provider(meter_provider.clone());
|
||||
meter_provider
|
||||
};
|
||||
|
||||
// initialize logger provider
|
||||
let logger_provider = {
|
||||
let mut builder = SdkLoggerProvider::builder().with_resource(res);
|
||||
|
||||
if endpoint.is_empty() {
|
||||
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
||||
} else {
|
||||
// Configure OTLP exporter when endpoint is provided
|
||||
let exporter = opentelemetry_otlp::LogExporter::builder()
|
||||
.with_tonic()
|
||||
.with_endpoint(&config.endpoint)
|
||||
.with_endpoint(endpoint)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
builder = builder.with_batch_exporter(exporter);
|
||||
|
||||
// Add stdout exporter if requested
|
||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
||||
if use_stdout {
|
||||
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
||||
}
|
||||
}
|
||||
|
||||
builder.build()
|
||||
};
|
||||
let config = config.clone();
|
||||
let logger_level = config.logger_level.unwrap_or(LOGGER_LEVEL.to_string());
|
||||
let logger_level = logger_level.as_str();
|
||||
// Setup OpenTelemetryTracingBridge layer
|
||||
let otel_layer = {
|
||||
// Filter to prevent infinite telemetry loops
|
||||
// This blocks events from OpenTelemetry and its dependent libraries (tonic, reqwest, etc.)
|
||||
// from being sent back to OpenTelemetry itself
|
||||
let filter_otel = match logger_level {
|
||||
"trace" | "debug" => {
|
||||
info!("OpenTelemetry tracing initialized with level: {}", logger_level);
|
||||
let mut filter = EnvFilter::new(logger_level);
|
||||
for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
|
||||
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
||||
|
||||
// configuring tracing
|
||||
{
|
||||
// optimize filter configuration
|
||||
let otel_layer = {
|
||||
let filter_otel = match logger_level {
|
||||
"trace" | "debug" => {
|
||||
info!("OpenTelemetry tracing initialized with level: {}", logger_level);
|
||||
EnvFilter::new(logger_level)
|
||||
}
|
||||
filter
|
||||
}
|
||||
_ => {
|
||||
let mut filter = EnvFilter::new(logger_level);
|
||||
for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
|
||||
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
||||
_ => {
|
||||
let mut filter = EnvFilter::new(logger_level);
|
||||
|
||||
// use smallvec to avoid heap allocation
|
||||
let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
|
||||
|
||||
for directive in directives {
|
||||
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
||||
}
|
||||
filter
|
||||
}
|
||||
filter
|
||||
}
|
||||
};
|
||||
layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
|
||||
};
|
||||
layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
|
||||
};
|
||||
|
||||
let tracer = tracer_provider.tracer(config.service_name.unwrap_or(SERVICE_NAME.to_string()));
|
||||
let registry = tracing_subscriber::registry()
|
||||
.with(switch_level(logger_level))
|
||||
.with(OpenTelemetryLayer::new(tracer))
|
||||
.with(MetricsLayer::new(meter_provider.clone()))
|
||||
.with(otel_layer)
|
||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
|
||||
let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
|
||||
|
||||
// Configure formatting layer
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(enable_color)
|
||||
.with_thread_names(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true);
|
||||
// Configure registry to avoid repeated calls to filter methods
|
||||
let level_filter = switch_level(logger_level);
|
||||
let registry = tracing_subscriber::registry()
|
||||
.with(level_filter)
|
||||
.with(OpenTelemetryLayer::new(tracer))
|
||||
.with(MetricsLayer::new(meter_provider.clone()))
|
||||
.with(otel_layer)
|
||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
|
||||
|
||||
// Creating a formatting layer with explicit type to avoid type mismatches
|
||||
let fmt_layer = fmt_layer.with_filter(
|
||||
EnvFilter::new(logger_level).add_directive(
|
||||
format!("opentelemetry={}", if config.endpoint.is_empty() { logger_level } else { "off" })
|
||||
.parse()
|
||||
.unwrap(),
|
||||
),
|
||||
);
|
||||
// configure the formatting layer
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_ansi(enable_color)
|
||||
.with_thread_names(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_filter(
|
||||
EnvFilter::new(logger_level).add_directive(
|
||||
format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
|
||||
.parse()
|
||||
.unwrap(),
|
||||
),
|
||||
);
|
||||
|
||||
registry.with(ErrorLayer::default()).with(fmt_layer).init();
|
||||
if !config.endpoint.is_empty() {
|
||||
info!(
|
||||
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
|
||||
config.endpoint, logger_level
|
||||
);
|
||||
registry.with(ErrorLayer::default()).with(fmt_layer).init();
|
||||
|
||||
if !endpoint.is_empty() {
|
||||
info!(
|
||||
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
|
||||
endpoint, logger_level
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
OtelGuard {
|
||||
@@ -313,12 +269,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
||||
|
||||
/// Switch log level
|
||||
fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
match logger_level {
|
||||
"error" => tracing_subscriber::filter::LevelFilter::ERROR,
|
||||
"warn" => tracing_subscriber::filter::LevelFilter::WARN,
|
||||
"info" => tracing_subscriber::filter::LevelFilter::INFO,
|
||||
"debug" => tracing_subscriber::filter::LevelFilter::DEBUG,
|
||||
"trace" => tracing_subscriber::filter::LevelFilter::TRACE,
|
||||
_ => tracing_subscriber::filter::LevelFilter::OFF,
|
||||
"error" => LevelFilter::ERROR,
|
||||
"warn" => LevelFilter::WARN,
|
||||
"info" => LevelFilter::INFO,
|
||||
"debug" => LevelFilter::DEBUG,
|
||||
"trace" => LevelFilter::TRACE,
|
||||
_ => LevelFilter::OFF,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user