gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-19 11:06:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): Optional continuous CPU profiling with grafana pyroscope (#2035)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: heihutu <30542132+heihutu@users.noreply.github.com>
This commit is contained in:
evan slack
2026-03-03 02:28:58 -05:00
committed by GitHub
parent 08e1f4670b
commit ac4b13def1
11 changed files with 300 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/config.rs
+10 -3
View File
@@ -33,13 +33,14 @@ use rustfs_config::observability::{
ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL,
ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED,
ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED,
ENV_OBS_USE_STDOUT,
ENV_OBS_PROFILING_ENDPOINT, ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME,
ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT,
};
use rustfs_config::{
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED,
DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO,
SERVICE_VERSION, USE_STDOUT,
};
use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_str, get_env_u64, get_env_usize};
use serde::{Deserialize, Serialize};
@@ -92,12 +93,16 @@ pub struct OtelConfig {
pub metric_endpoint: Option<String>,
/// Dedicated log endpoint; overrides `endpoint` + `/v1/logs` fallback.
pub log_endpoint: Option<String>,
/// Dedicated profiling endpoint.
pub profiling_endpoint: Option<String>,
/// Whether to export distributed traces (default: `true`).
pub traces_export_enabled: Option<bool>,
/// Whether to export metrics (default: `true`).
pub metrics_export_enabled: Option<bool>,
/// Whether to export logs via OTLP (default: `true`).
pub logs_export_enabled: Option<bool>,
/// Whether to export profiles via pyroscope (default: `true`).
pub profiling_export_enabled: Option<bool>,
/// **[OTLP-only]** Mirror all signals to stdout in addition to OTLP export.
/// Only applies when an OTLP endpoint is configured.
pub use_stdout: Option<bool>,
@@ -216,9 +221,11 @@ impl OtelConfig {
trace_endpoint: get_env_opt_str(ENV_OBS_TRACE_ENDPOINT),
metric_endpoint: get_env_opt_str(ENV_OBS_METRIC_ENDPOINT),
log_endpoint: get_env_opt_str(ENV_OBS_LOG_ENDPOINT),
profiling_endpoint: get_env_opt_str(ENV_OBS_PROFILING_ENDPOINT),
traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)),
metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)),
logs_export_enabled: Some(get_env_bool(ENV_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED)),
profiling_export_enabled: Some(get_env_bool(ENV_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_PROFILING_EXPORT_ENABLED)),
use_stdout: Some(use_stdout),
sample_ratio: Some(get_env_f64(ENV_OBS_SAMPLE_RATIO, SAMPLE_RATIO)),
meter_interval: Some(get_env_u64(ENV_OBS_METER_INTERVAL, METER_INTERVAL)),
crates/obs/src/telemetry/guard.rs
+26 -7
View File
@@ -20,12 +20,17 @@
//! 1. Tracer provider — flushes pending spans.
//! 2. Meter provider — flushes pending metrics.
//! 3. Logger provider — flushes pending log records.
//! 4. Cleanup task — aborted to prevent lingering background work.
//! 5. Tracing worker guard — flushes buffered log lines written by
//! 4. Profiling agent — flushes pending profiles.
//! 5. Cleanup task — aborted to prevent lingering background work.
//! 6. Tracing worker guard — flushes buffered log lines written by
//! `tracing_appender`.
//! 6. Stdout worker guard — flushes buffered log lines written to stdout.
//! 7. Stdout worker guard — flushes buffered log lines written to stdout.
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
#[cfg(unix)]
use pyroscope::PyroscopeAgent;
#[cfg(unix)]
use pyroscope::pyroscope::PyroscopeAgentRunning;
/// RAII guard that owns all active OpenTelemetry providers and the
/// `tracing_appender` worker guard.
@@ -40,6 +45,8 @@ pub struct OtelGuard {
pub(crate) meter_provider: Option<SdkMeterProvider>,
/// Optional logger provider for OTLP log export.
pub(crate) logger_provider: Option<SdkLoggerProvider>,
#[cfg(unix)]
pub(crate) profiling_agent: Option<PyroscopeAgent<PyroscopeAgentRunning>>,
/// Handle to the background log-cleanup task; aborted on drop.
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
/// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -51,11 +58,13 @@ pub struct OtelGuard {
impl std::fmt::Debug for OtelGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OtelGuard")
.field("tracer_provider", &self.tracer_provider.is_some())
let mut s = f.debug_struct("OtelGuard");
s.field("tracer_provider", &self.tracer_provider.is_some())
.field("meter_provider", &self.meter_provider.is_some())
.field("logger_provider", &self.logger_provider.is_some())
.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("logger_provider", &self.logger_provider.is_some());
#[cfg(unix)]
s.field("profiling_agent", &self.profiling_agent.is_some());
s.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("tracing_guard", &self.tracing_guard.is_some())
.field("stdout_guard", &self.stdout_guard.is_some())
.finish()
@@ -86,6 +95,16 @@ impl Drop for OtelGuard {
eprintln!("Logger shutdown error: {err:?}");
}
#[cfg(unix)]
if let Some(agent) = self.profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
Ok(stopped) => {
stopped.shutdown();
}
}
}
if let Some(handle) = self.cleanup_handle.take() {
handle.abort();
eprintln!("Log cleanup task stopped");
crates/obs/src/telemetry/local.rs
+4
View File
@@ -139,6 +139,8 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(unix)]
profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard: None,
cleanup_handle: None,
@@ -275,6 +277,8 @@ fn init_file_logging_internal(
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(unix)]
profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard,
cleanup_handle: Some(cleanup_handle),
crates/obs/src/telemetry/otel.rs
+50 -1
View File
@@ -49,9 +49,13 @@ use opentelemetry_sdk::{
metrics::{PeriodicReader, SdkMeterProvider},
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
};
#[cfg(unix)]
use pyroscope::PyroscopeAgent;
#[cfg(unix)]
use pyroscope::pyroscope::PyroscopeAgentRunning;
use rustfs_config::{
APP_NAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED,
DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
};
use std::{io::IsTerminal, time::Duration};
use tracing::info;
@@ -137,6 +141,9 @@ pub(super) fn init_observability_http(
// ── Logger provider (HTTP) ────────────────────────────────────────────────
let logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
#[cfg(unix)]
let profiling_agent = init_profiler(config);
// ── Tracing subscriber registry ───────────────────────────────────────────
// Build an optional stdout formatting layer. When `log_stdout_enabled` is
// false the field is `None` and tracing-subscriber will skip it.
@@ -188,6 +195,8 @@ pub(super) fn init_observability_http(
tracer_provider,
meter_provider,
logger_provider,
#[cfg(unix)]
profiling_agent,
tracing_guard: None,
stdout_guard: None,
cleanup_handle: None,
@@ -309,6 +318,46 @@ fn build_logger_provider(
Ok(Some(builder.build()))
}
/// Starts the Pyroscope continuous profiling agent if `ENV_OBS_PROFILING_ENDPOINT` is set.
/// No-op and returns None on non-unix platforms.
#[cfg(unix)]
fn init_profiler(config: &OtelConfig) -> Option<PyroscopeAgent<PyroscopeAgentRunning>> {
use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
use pyroscope::pyroscope::PyroscopeAgentBuilder;
use rustfs_config::VERSION;
if !config
.profiling_export_enabled
.unwrap_or(DEFAULT_OBS_PROFILING_EXPORT_ENABLED)
{
return None;
}
let endpoint = config.profiling_endpoint.as_ref()?.as_str();
if endpoint.is_empty() {
return None;
}
// Configure Pyroscope Agent
let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
let version = config.service_version.as_deref().unwrap_or(VERSION);
let sample_rate = 100; // 100 Hz
let agent = PyroscopeAgentBuilder::new(endpoint, service_name, sample_rate, "pyroscope-rs", "1.0.1", backend)
.tags(vec![("version", version)]) // TODO: add git commit tag
.build()
.ok()?;
match agent.start() {
Ok(agent) => Some(agent),
Err(err) => {
eprintln!("Pyroscope agent start error: {err:?}");
None
}
}
}
/// Create a stdout periodic metrics reader for the given interval.
fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout::MetricExporter> {
PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default())