diff --git a/.docker/observability/docker-compose.yml b/.docker/observability/docker-compose.yml index 43e502df0..d2c88e0af 100644 --- a/.docker/observability/docker-compose.yml +++ b/.docker/observability/docker-compose.yml @@ -32,6 +32,7 @@ services: - RUSTFS_SECRET_KEY=rustfsadmin - RUSTFS_OBS_LOGGER_LEVEL=info - RUSTFS_OBS_ENDPOINT=http://otel-collector:4318 + - RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040 volumes: - rustfs-data:/data/rustfs networks: @@ -79,8 +80,8 @@ services: - tempo-data:/var/tempo ports: - "3200:3200" # tempo - - "4317" # otlp grpc - - "4318" # otlp http + - "4317" # otlp grpc + - "4318" # otlp http networks: - otel-network restart: unless-stopped @@ -188,12 +189,12 @@ services: volumes: - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro ports: - - "1888:1888" # pprof - - "8888:8888" # Prometheus metrics for Collector - - "8889:8889" # Prometheus metrics for application indicators + - "1888:1888" # pprof + - "8888:8888" # Prometheus metrics for Collector + - "8889:8889" # Prometheus metrics for application indicators - "13133:13133" # health check - - "4317:4317" # OTLP gRPC - - "4318:4318" # OTLP HTTP + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP - "55679:55679" # zpages networks: - otel-network @@ -209,6 +210,19 @@ services: timeout: 5s retries: 3 + # --- Profiles --- + + pyroscope: + image: grafana/pyroscope:latest + container_name: pyroscope + ports: + - "4040:4040" + command: + - -self-profiling.disable-push=true + networks: + - otel-network + restart: unless-stopped + # --- Visualization --- grafana: @@ -231,7 +245,8 @@ services: - tempo - loki healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] + test: + ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] interval: 10s timeout: 5s retries: 3 diff --git a/.docker/observability/grafana/provisioning/datasources/datasources.yaml b/.docker/observability/grafana/provisioning/datasources/datasources.yaml index 9b6c6f4dc..f1ee62f17 100644 --- a/.docker/observability/grafana/provisioning/datasources/datasources.yaml +++ b/.docker/observability/grafana/provisioning/datasources/datasources.yaml @@ -76,3 +76,9 @@ datasources: spanEndTimeShift: '-1s' filterByTraceID: true filterBySpanID: false + + - name: Pyroscope + type: grafana-pyroscope-datasource + url: http://pyroscope:4040 + jsonData: + minStep: '15s' diff --git a/Cargo.lock b/Cargo.lock index 637d5f40d..cc3ad60e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ - "gimli", + "gimli 0.32.3", ] [[package]] @@ -17,6 +17,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aead" version = "0.6.0-rc.10" @@ -202,7 +208,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ - "object", + "object 0.37.3", ] [[package]] @@ -1104,7 +1110,7 @@ dependencies = [ "cfg-if", "libc", "miniz_oxide", - "object", + "object 0.37.3", "rustc-demangle", "windows-link", ] @@ -1210,7 +1216,7 @@ dependencies = [ "cfg-if", "constant_time_eq", "cpufeatures 0.2.17", - "memmap2", + "memmap2 0.9.10", "rayon-core", ] @@ -1724,6 +1730,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpp_demangle" version = "0.5.1" @@ -2161,6 +2176,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dary_heap" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" + [[package]] name = "dashmap" version = "6.1.0" @@ -3320,6 +3341,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "faster-hex" version = "0.10.0" @@ -3456,6 +3483,20 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "framehop" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe" +dependencies = [ + "arrayvec", + "cfg-if", + "fallible-iterator", + "gimli 0.31.1", + "macho-unwind-info", + "pe-unwind-info", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3638,6 +3679,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +dependencies = [ + "fallible-iterator", + "stable_deref_trait", +] + [[package]] name = "gimli" version = "0.32.3" @@ -4734,6 +4785,30 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libflate" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74" +dependencies = [ + "adler32", + "core2", + "crc32fast", + "dary_heap", + "libflate_lz77", +] + +[[package]] +name = "libflate_lz77" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c" +dependencies = [ + "core2", + "hashbrown 0.16.1", + "rle-decode-fast", +] + [[package]] name = "libloading" version = "0.8.9" @@ -4938,6 +5013,17 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "macho-unwind-info" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149" +dependencies = [ + "thiserror 2.0.18", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "mappings" version = "0.7.2" @@ -5004,6 +5090,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memmap2" version = "0.9.10" @@ -5110,6 +5205,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "names" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "neli" version = "0.7.4" @@ -5414,6 +5518,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "flate2", + "memchr", +] + [[package]] name = "object" version = "0.37.3" @@ -5815,6 +5929,19 @@ dependencies = [ "hmac 0.13.0-rc.5", ] +[[package]] +name = "pe-unwind-info" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59" +dependencies = [ + "arrayvec", + "bitflags 2.11.0", + "thiserror 2.0.18", + "zerocopy", + "zerocopy-derive", +] + [[package]] name = "pem" version = "3.0.6" @@ -6077,19 +6204,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" [[package]] -name = "pprof" -version = "0.15.0" +name = "pprof-pyroscope-fork" +version = "0.1500.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +checksum = "a81f1489b882945ce89a4c93dbb50b1bf31839188d3c83fb5e5eded6bad6ee2d" dependencies = [ "aligned-vec", "backtrace", "cfg-if", "findshlibs", + "framehop", "inferno 0.11.21", "libc", "log", + "memmap2 0.5.10", "nix 0.26.4", + "object 0.29.0", "once_cell", "protobuf", "protobuf-codegen", @@ -6342,6 +6472,26 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "pyroscope" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae9336ab02211eea97f17d2f9ffc9de0810e588b78334d268ed79ff4bfcd1210" +dependencies = [ + "lazy_static", + "libc", + "libflate", + "log", + "names", + "pprof-pyroscope-fork", + "prost", + "reqwest 0.13.2", + "serde_json", + "thiserror 2.0.18", + "url", + "uuid", +] + [[package]] name = "quick-xml" version = "0.26.0" @@ -6818,6 +6968,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + [[package]] name = "rmcp" version = "0.17.0" @@ -7028,7 +7184,7 @@ dependencies = [ "moka", "opentelemetry", "pin-project-lite", - "pprof", + "pprof-pyroscope-fork", "rand 0.10.0", "reqwest 0.13.2", "rmp-serde", @@ -7517,6 +7673,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry-stdout", "opentelemetry_sdk", + "pyroscope", "rustfs-config", "rustfs-utils", "serde", @@ -8820,7 +8977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23" dependencies = [ "debugid", - "memmap2", + "memmap2 0.9.10", "stable_deref_trait", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 0d1f77a5b..c7bda233e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -275,6 +275,7 @@ opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rus opentelemetry_sdk = { version = "0.31.0" } opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] } opentelemetry-stdout = { version = "0.31.0" } +pyroscope = { version= "1.0.1", features= ["backend-pprof-rs"]} # FTP and SFTP libunftp = { version = "0.23.0", features = ["experimental"] } @@ -291,7 +292,9 @@ tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profilin # Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] } # Used to generate CPU performance analysis data and flame diagrams -pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] } +# pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] } +# Pyroscope uses a patched pprof, until they merge back upstream, replace all references. Otherwise, two pprof libs with symbol collision. +pprof = { package = "pprof-pyroscope-fork", version = "0.1500.1", features = ["flamegraph", "protobuf-codec"]} [workspace.metadata.cargo-shear] ignored = ["rustfs", "rustfs-mcp"] diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 844dd1609..0b6cb2b85 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -221,6 +221,12 @@ pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true; /// Environment variable: RUSTFS_OBS_LOGS_EXPORT_ENABLED pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true; +/// Default profiling export enabled +/// It is used to enable or disable exporting profiles +/// Default value: true +/// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED +pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true; + /// Default log local logging enabled for rustfs /// This is the default log local logging enabled for rustfs. /// It is used to enable or disable local logging of the application. diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index 68a571966..6b90dad60 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -22,6 +22,7 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT"; pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT"; pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT"; pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT"; +pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT"; pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT"; pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO"; pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL"; @@ -33,6 +34,7 @@ pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT"; pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED"; pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENABLED"; pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED"; +pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED"; pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL"; pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED"; @@ -90,6 +92,7 @@ mod tests { assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT"); assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT"); assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT"); + assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT"); assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT"); assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO"); assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL"); @@ -106,6 +109,7 @@ mod tests { assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED"); assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED"); assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED"); + assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED"); // Test log cleanup related env keys assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES"); assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES"); diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 0303438d0..bbe0fbefa 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -56,6 +56,7 @@ tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", " tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } sysinfo = { workspace = true } thiserror = { workspace = true } +pyroscope = { workspace = true, features= ["backend-pprof-rs"]} walkdir = { workspace = true } diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index eff8044ea..7788ef7fc 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -33,13 +33,14 @@ use rustfs_config::observability::{ ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL, ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED, - ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, - ENV_OBS_USE_STDOUT, + ENV_OBS_PROFILING_ENDPOINT, ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, + ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT, }; use rustfs_config::{ APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED, - DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, + DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, + SERVICE_VERSION, USE_STDOUT, }; use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_str, get_env_u64, get_env_usize}; use serde::{Deserialize, Serialize}; @@ -92,12 +93,16 @@ pub struct OtelConfig { pub metric_endpoint: Option, /// Dedicated log endpoint; overrides `endpoint` + `/v1/logs` fallback. pub log_endpoint: Option, + /// Dedicated profiling endpoint. + pub profiling_endpoint: Option, /// Whether to export distributed traces (default: `true`). pub traces_export_enabled: Option, /// Whether to export metrics (default: `true`). pub metrics_export_enabled: Option, /// Whether to export logs via OTLP (default: `true`). pub logs_export_enabled: Option, + /// Whether to export profiles via pyroscope (default: `true`). + pub profiling_export_enabled: Option, /// **[OTLP-only]** Mirror all signals to stdout in addition to OTLP export. /// Only applies when an OTLP endpoint is configured. pub use_stdout: Option, @@ -216,9 +221,11 @@ impl OtelConfig { trace_endpoint: get_env_opt_str(ENV_OBS_TRACE_ENDPOINT), metric_endpoint: get_env_opt_str(ENV_OBS_METRIC_ENDPOINT), log_endpoint: get_env_opt_str(ENV_OBS_LOG_ENDPOINT), + profiling_endpoint: get_env_opt_str(ENV_OBS_PROFILING_ENDPOINT), traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)), metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)), logs_export_enabled: Some(get_env_bool(ENV_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED)), + profiling_export_enabled: Some(get_env_bool(ENV_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_PROFILING_EXPORT_ENABLED)), use_stdout: Some(use_stdout), sample_ratio: Some(get_env_f64(ENV_OBS_SAMPLE_RATIO, SAMPLE_RATIO)), meter_interval: Some(get_env_u64(ENV_OBS_METER_INTERVAL, METER_INTERVAL)), diff --git a/crates/obs/src/telemetry/guard.rs b/crates/obs/src/telemetry/guard.rs index 0cc59756f..b36753d73 100644 --- a/crates/obs/src/telemetry/guard.rs +++ b/crates/obs/src/telemetry/guard.rs @@ -20,12 +20,17 @@ //! 1. Tracer provider — flushes pending spans. //! 2. Meter provider — flushes pending metrics. //! 3. Logger provider — flushes pending log records. -//! 4. Cleanup task — aborted to prevent lingering background work. -//! 5. Tracing worker guard — flushes buffered log lines written by +//! 4. Profiling agent — flushes pending profiles. +//! 5. Cleanup task — aborted to prevent lingering background work. +//! 6. Tracing worker guard — flushes buffered log lines written by //! `tracing_appender`. -//! 6. Stdout worker guard — flushes buffered log lines written to stdout. +//! 7. Stdout worker guard — flushes buffered log lines written to stdout. use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider}; +#[cfg(unix)] +use pyroscope::PyroscopeAgent; +#[cfg(unix)] +use pyroscope::pyroscope::PyroscopeAgentRunning; /// RAII guard that owns all active OpenTelemetry providers and the /// `tracing_appender` worker guard. @@ -40,6 +45,8 @@ pub struct OtelGuard { pub(crate) meter_provider: Option, /// Optional logger provider for OTLP log export. pub(crate) logger_provider: Option, + #[cfg(unix)] + pub(crate) profiling_agent: Option>, /// Handle to the background log-cleanup task; aborted on drop. pub(crate) cleanup_handle: Option>, /// Worker guard that keeps the non-blocking `tracing_appender` thread @@ -51,11 +58,13 @@ pub struct OtelGuard { impl std::fmt::Debug for OtelGuard { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("OtelGuard") - .field("tracer_provider", &self.tracer_provider.is_some()) + let mut s = f.debug_struct("OtelGuard"); + s.field("tracer_provider", &self.tracer_provider.is_some()) .field("meter_provider", &self.meter_provider.is_some()) - .field("logger_provider", &self.logger_provider.is_some()) - .field("cleanup_handle", &self.cleanup_handle.is_some()) + .field("logger_provider", &self.logger_provider.is_some()); + #[cfg(unix)] + s.field("profiling_agent", &self.profiling_agent.is_some()); + s.field("cleanup_handle", &self.cleanup_handle.is_some()) .field("tracing_guard", &self.tracing_guard.is_some()) .field("stdout_guard", &self.stdout_guard.is_some()) .finish() @@ -86,6 +95,16 @@ impl Drop for OtelGuard { eprintln!("Logger shutdown error: {err:?}"); } + #[cfg(unix)] + if let Some(agent) = self.profiling_agent.take() { + match agent.stop() { + Err(err) => eprintln!("Profiling agent stop error: {err:?}"), + Ok(stopped) => { + stopped.shutdown(); + } + } + } + if let Some(handle) = self.cleanup_handle.take() { handle.abort(); eprintln!("Log cleanup task stopped"); diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index e728cb262..4ca69a1e7 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -139,6 +139,8 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo tracer_provider: None, meter_provider: None, logger_provider: None, + #[cfg(unix)] + profiling_agent: None, tracing_guard: Some(guard), stdout_guard: None, cleanup_handle: None, @@ -275,6 +277,8 @@ fn init_file_logging_internal( tracer_provider: None, meter_provider: None, logger_provider: None, + #[cfg(unix)] + profiling_agent: None, tracing_guard: Some(guard), stdout_guard, cleanup_handle: Some(cleanup_handle), diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index c1d556cfe..8da6d42c2 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -49,9 +49,13 @@ use opentelemetry_sdk::{ metrics::{PeriodicReader, SdkMeterProvider}, trace::{RandomIdGenerator, Sampler, SdkTracerProvider}, }; +#[cfg(unix)] +use pyroscope::PyroscopeAgent; +#[cfg(unix)] +use pyroscope::pyroscope::PyroscopeAgentRunning; use rustfs_config::{ APP_NAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED, - DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO, + DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO, }; use std::{io::IsTerminal, time::Duration}; use tracing::info; @@ -137,6 +141,9 @@ pub(super) fn init_observability_http( // ── Logger provider (HTTP) ──────────────────────────────────────────────── let logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?; + #[cfg(unix)] + let profiling_agent = init_profiler(config); + // ── Tracing subscriber registry ─────────────────────────────────────────── // Build an optional stdout formatting layer. When `log_stdout_enabled` is // false the field is `None` and tracing-subscriber will skip it. @@ -188,6 +195,8 @@ pub(super) fn init_observability_http( tracer_provider, meter_provider, logger_provider, + #[cfg(unix)] + profiling_agent, tracing_guard: None, stdout_guard: None, cleanup_handle: None, @@ -309,6 +318,46 @@ fn build_logger_provider( Ok(Some(builder.build())) } +/// Starts the Pyroscope continuous profiling agent if `ENV_OBS_PROFILING_ENDPOINT` is set. +/// No-op and returns None on non-unix platforms. +#[cfg(unix)] +fn init_profiler(config: &OtelConfig) -> Option> { + use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend}; + use pyroscope::pyroscope::PyroscopeAgentBuilder; + use rustfs_config::VERSION; + + if !config + .profiling_export_enabled + .unwrap_or(DEFAULT_OBS_PROFILING_EXPORT_ENABLED) + { + return None; + } + + let endpoint = config.profiling_endpoint.as_ref()?.as_str(); + if endpoint.is_empty() { + return None; + } + + // Configure Pyroscope Agent + let backend = pprof_backend(PprofConfig::default(), BackendConfig::default()); + let service_name = config.service_name.as_deref().unwrap_or(APP_NAME); + let version = config.service_version.as_deref().unwrap_or(VERSION); + let sample_rate = 100; // 100 Hz + + let agent = PyroscopeAgentBuilder::new(endpoint, service_name, sample_rate, "pyroscope-rs", "1.0.1", backend) + .tags(vec![("version", version)]) // TODO: add git commit tag + .build() + .ok()?; + + match agent.start() { + Ok(agent) => Some(agent), + Err(err) => { + eprintln!("Pyroscope agent start error: {err:?}"); + None + } + } +} + /// Create a stdout periodic metrics reader for the given interval. fn create_periodic_reader(interval: u64) -> PeriodicReader { PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default())