gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-31 17:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): Optional continuous CPU profiling with grafana pyroscope (#2035)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: heihutu <30542132+heihutu@users.noreply.github.com>
This commit is contained in:
evan slack
2026-03-03 02:28:58 -05:00
committed by GitHub
parent 08e1f4670b
commit ac4b13def1
11 changed files with 300 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/telemetry/guard.rs
+26 -7
View File
@@ -20,12 +20,17 @@
//! 1. Tracer provider — flushes pending spans.
//! 2. Meter provider — flushes pending metrics.
//! 3. Logger provider — flushes pending log records.
//! 4. Cleanup task — aborted to prevent lingering background work.
//! 5. Tracing worker guard — flushes buffered log lines written by
//! 4. Profiling agent — flushes pending profiles.
//! 5. Cleanup task — aborted to prevent lingering background work.
//! 6. Tracing worker guard — flushes buffered log lines written by
//! `tracing_appender`.
//! 6. Stdout worker guard — flushes buffered log lines written to stdout.
//! 7. Stdout worker guard — flushes buffered log lines written to stdout.
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
#[cfg(unix)]
use pyroscope::PyroscopeAgent;
#[cfg(unix)]
use pyroscope::pyroscope::PyroscopeAgentRunning;
/// RAII guard that owns all active OpenTelemetry providers and the
/// `tracing_appender` worker guard.
@@ -40,6 +45,8 @@ pub struct OtelGuard {
pub(crate) meter_provider: Option<SdkMeterProvider>,
/// Optional logger provider for OTLP log export.
pub(crate) logger_provider: Option<SdkLoggerProvider>,
#[cfg(unix)]
pub(crate) profiling_agent: Option<PyroscopeAgent<PyroscopeAgentRunning>>,
/// Handle to the background log-cleanup task; aborted on drop.
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
/// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -51,11 +58,13 @@ pub struct OtelGuard {
impl std::fmt::Debug for OtelGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OtelGuard")
.field("tracer_provider", &self.tracer_provider.is_some())
let mut s = f.debug_struct("OtelGuard");
s.field("tracer_provider", &self.tracer_provider.is_some())
.field("meter_provider", &self.meter_provider.is_some())
.field("logger_provider", &self.logger_provider.is_some())
.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("logger_provider", &self.logger_provider.is_some());
#[cfg(unix)]
s.field("profiling_agent", &self.profiling_agent.is_some());
s.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("tracing_guard", &self.tracing_guard.is_some())
.field("stdout_guard", &self.stdout_guard.is_some())
.finish()
@@ -86,6 +95,16 @@ impl Drop for OtelGuard {
eprintln!("Logger shutdown error: {err:?}");
}
#[cfg(unix)]
if let Some(agent) = self.profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
Ok(stopped) => {
stopped.shutdown();
}
}
}
if let Some(handle) = self.cleanup_handle.take() {
handle.abort();
eprintln!("Log cleanup task stopped");