mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
feat(obs): Optional continuous CPU profiling with grafana pyroscope (#2035)
Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: loverustfs <hello@rustfs.com> Co-authored-by: heihutu <30542132+heihutu@users.noreply.github.com>
This commit is contained in:
evan slack
@@ -32,6 +32,7 @@ services:
|
||||
- RUSTFS_SECRET_KEY=rustfsadmin
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=info
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
|
||||
volumes:
|
||||
- rustfs-data:/data/rustfs
|
||||
networks:
|
||||
@@ -79,8 +80,8 @@ services:
|
||||
- tempo-data:/var/tempo
|
||||
ports:
|
||||
- "3200:3200" # tempo
|
||||
- "4317" # otlp grpc
|
||||
- "4318" # otlp http
|
||||
- "4317" # otlp grpc
|
||||
- "4318" # otlp http
|
||||
networks:
|
||||
- otel-network
|
||||
restart: unless-stopped
|
||||
@@ -188,12 +189,12 @@ services:
|
||||
volumes:
|
||||
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
ports:
|
||||
- "1888:1888" # pprof
|
||||
- "8888:8888" # Prometheus metrics for Collector
|
||||
- "8889:8889" # Prometheus metrics for application indicators
|
||||
- "1888:1888" # pprof
|
||||
- "8888:8888" # Prometheus metrics for Collector
|
||||
- "8889:8889" # Prometheus metrics for application indicators
|
||||
- "13133:13133" # health check
|
||||
- "4317:4317" # OTLP gRPC
|
||||
- "4318:4318" # OTLP HTTP
|
||||
- "4317:4317" # OTLP gRPC
|
||||
- "4318:4318" # OTLP HTTP
|
||||
- "55679:55679" # zpages
|
||||
networks:
|
||||
- otel-network
|
||||
@@ -209,6 +210,19 @@ services:
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# --- Profiles ---
|
||||
|
||||
pyroscope:
|
||||
image: grafana/pyroscope:latest
|
||||
container_name: pyroscope
|
||||
ports:
|
||||
- "4040:4040"
|
||||
command:
|
||||
- -self-profiling.disable-push=true
|
||||
networks:
|
||||
- otel-network
|
||||
restart: unless-stopped
|
||||
|
||||
# --- Visualization ---
|
||||
|
||||
grafana:
|
||||
@@ -231,7 +245,8 @@ services:
|
||||
- tempo
|
||||
- loki
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
|
||||
test:
|
||||
["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
@@ -76,3 +76,9 @@ datasources:
|
||||
spanEndTimeShift: '-1s'
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
|
||||
- name: Pyroscope
|
||||
type: grafana-pyroscope-datasource
|
||||
url: http://pyroscope:4040
|
||||
jsonData:
|
||||
minStep: '15s'
|
||||
|
||||
Generated
+166
-9
@@ -8,7 +8,7 @@ version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
"gimli 0.32.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -17,6 +17,12 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "adler32"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.6.0-rc.10"
|
||||
@@ -202,7 +208,7 @@ version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
|
||||
dependencies = [
|
||||
"object",
|
||||
"object 0.37.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1104,7 +1110,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"object 0.37.3",
|
||||
"rustc-demangle",
|
||||
"windows-link",
|
||||
]
|
||||
@@ -1210,7 +1216,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures 0.2.17",
|
||||
"memmap2",
|
||||
"memmap2 0.9.10",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
@@ -1724,6 +1730,15 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "core2"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpp_demangle"
|
||||
version = "0.5.1"
|
||||
@@ -2161,6 +2176,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dary_heap"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.1.0"
|
||||
@@ -3320,6 +3341,12 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "faster-hex"
|
||||
version = "0.10.0"
|
||||
@@ -3456,6 +3483,20 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "framehop"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"cfg-if",
|
||||
"fallible-iterator",
|
||||
"gimli 0.31.1",
|
||||
"macho-unwind-info",
|
||||
"pe-unwind-info",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
@@ -3638,6 +3679,16 @@ dependencies = [
|
||||
"polyval",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.31.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
|
||||
dependencies = [
|
||||
"fallible-iterator",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.32.3"
|
||||
@@ -4734,6 +4785,30 @@ version = "0.2.182"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
version = "2.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"core2",
|
||||
"crc32fast",
|
||||
"dary_heap",
|
||||
"libflate_lz77",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libflate_lz77"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c"
|
||||
dependencies = [
|
||||
"core2",
|
||||
"hashbrown 0.16.1",
|
||||
"rle-decode-fast",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.8.9"
|
||||
@@ -4938,6 +5013,17 @@ dependencies = [
|
||||
"sha2 0.10.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "macho-unwind-info"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149"
|
||||
dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mappings"
|
||||
version = "0.7.2"
|
||||
@@ -5004,6 +5090,15 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.5.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memmap2"
|
||||
version = "0.9.10"
|
||||
@@ -5110,6 +5205,15 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
||||
|
||||
[[package]]
|
||||
name = "names"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bddcd3bf5144b6392de80e04c347cd7fab2508f6df16a85fc496ecd5cec39bc"
|
||||
dependencies = [
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "neli"
|
||||
version = "0.7.4"
|
||||
@@ -5414,6 +5518,16 @@ dependencies = [
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.29.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.37.3"
|
||||
@@ -5815,6 +5929,19 @@ dependencies = [
|
||||
"hmac 0.13.0-rc.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pe-unwind-info"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.11.0",
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "3.0.6"
|
||||
@@ -6077,19 +6204,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24"
|
||||
|
||||
[[package]]
|
||||
name = "pprof"
|
||||
version = "0.15.0"
|
||||
name = "pprof-pyroscope-fork"
|
||||
version = "0.1500.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187"
|
||||
checksum = "a81f1489b882945ce89a4c93dbb50b1bf31839188d3c83fb5e5eded6bad6ee2d"
|
||||
dependencies = [
|
||||
"aligned-vec",
|
||||
"backtrace",
|
||||
"cfg-if",
|
||||
"findshlibs",
|
||||
"framehop",
|
||||
"inferno 0.11.21",
|
||||
"libc",
|
||||
"log",
|
||||
"memmap2 0.5.10",
|
||||
"nix 0.26.4",
|
||||
"object 0.29.0",
|
||||
"once_cell",
|
||||
"protobuf",
|
||||
"protobuf-codegen",
|
||||
@@ -6342,6 +6472,26 @@ dependencies = [
|
||||
"pulldown-cmark",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyroscope"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae9336ab02211eea97f17d2f9ffc9de0810e588b78334d268ed79ff4bfcd1210"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"libflate",
|
||||
"log",
|
||||
"names",
|
||||
"pprof-pyroscope-fork",
|
||||
"prost",
|
||||
"reqwest 0.13.2",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.26.0"
|
||||
@@ -6818,6 +6968,12 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rle-decode-fast"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "0.17.0"
|
||||
@@ -7028,7 +7184,7 @@ dependencies = [
|
||||
"moka",
|
||||
"opentelemetry",
|
||||
"pin-project-lite",
|
||||
"pprof",
|
||||
"pprof-pyroscope-fork",
|
||||
"rand 0.10.0",
|
||||
"reqwest 0.13.2",
|
||||
"rmp-serde",
|
||||
@@ -7517,6 +7673,7 @@ dependencies = [
|
||||
"opentelemetry-semantic-conventions",
|
||||
"opentelemetry-stdout",
|
||||
"opentelemetry_sdk",
|
||||
"pyroscope",
|
||||
"rustfs-config",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
@@ -8820,7 +8977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "751a2823d606b5d0a7616499e4130a516ebd01a44f39811be2b9600936509c23"
|
||||
dependencies = [
|
||||
"debugid",
|
||||
"memmap2",
|
||||
"memmap2 0.9.10",
|
||||
"stable_deref_trait",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
+4
-1
@@ -275,6 +275,7 @@ opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rus
|
||||
opentelemetry_sdk = { version = "0.31.0" }
|
||||
opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
|
||||
opentelemetry-stdout = { version = "0.31.0" }
|
||||
pyroscope = { version= "1.0.1", features= ["backend-pprof-rs"]}
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0", features = ["experimental"] }
|
||||
@@ -291,7 +292,9 @@ tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats", "profilin
|
||||
# Used to generate pprof-compatible memory profiling data and support symbolization and flame graphs
|
||||
jemalloc_pprof = { version = "0.8.2", features = ["symbolize", "flamegraph"] }
|
||||
# Used to generate CPU performance analysis data and flame diagrams
|
||||
pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
|
||||
# pprof = { version = "0.15.0", features = ["flamegraph", "protobuf-codec"] }
|
||||
# Pyroscope uses a patched pprof, until they merge back upstream, replace all references. Otherwise, two pprof libs with symbol collision.
|
||||
pprof = { package = "pprof-pyroscope-fork", version = "0.1500.1", features = ["flamegraph", "protobuf-codec"]}
|
||||
|
||||
[workspace.metadata.cargo-shear]
|
||||
ignored = ["rustfs", "rustfs-mcp"]
|
||||
|
||||
@@ -221,6 +221,12 @@ pub const DEFAULT_OBS_METRICS_EXPORT_ENABLED: bool = true;
|
||||
/// Environment variable: RUSTFS_OBS_LOGS_EXPORT_ENABLED
|
||||
pub const DEFAULT_OBS_LOGS_EXPORT_ENABLED: bool = true;
|
||||
|
||||
/// Default profiling export enabled
|
||||
/// It is used to enable or disable exporting profiles
|
||||
/// Default value: true
|
||||
/// Environment variable: RUSTFS_OBS_PROFILING_EXPORT_ENABLED
|
||||
pub const DEFAULT_OBS_PROFILING_EXPORT_ENABLED: bool = true;
|
||||
|
||||
/// Default log local logging enabled for rustfs
|
||||
/// This is the default log local logging enabled for rustfs.
|
||||
/// It is used to enable or disable local logging of the application.
|
||||
|
||||
@@ -22,6 +22,7 @@ pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
pub const ENV_OBS_TRACE_ENDPOINT: &str = "RUSTFS_OBS_TRACE_ENDPOINT";
|
||||
pub const ENV_OBS_METRIC_ENDPOINT: &str = "RUSTFS_OBS_METRIC_ENDPOINT";
|
||||
pub const ENV_OBS_LOG_ENDPOINT: &str = "RUSTFS_OBS_LOG_ENDPOINT";
|
||||
pub const ENV_OBS_PROFILING_ENDPOINT: &str = "RUSTFS_OBS_PROFILING_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
|
||||
@@ -33,6 +34,7 @@ pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
|
||||
pub const ENV_OBS_TRACES_EXPORT_ENABLED: &str = "RUSTFS_OBS_TRACES_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_METRICS_EXPORT_ENABLED: &str = "RUSTFS_OBS_METRICS_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_LOGS_EXPORT_ENABLED: &str = "RUSTFS_OBS_LOGS_EXPORT_ENABLED";
|
||||
pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_ENABLED";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
@@ -90,6 +92,7 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_TRACE_ENDPOINT, "RUSTFS_OBS_TRACE_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_METRIC_ENDPOINT, "RUSTFS_OBS_METRIC_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_LOG_ENDPOINT, "RUSTFS_OBS_LOG_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_PROFILING_ENDPOINT, "RUSTFS_OBS_PROFILING_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
|
||||
assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
|
||||
assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
|
||||
@@ -106,6 +109,7 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_METRICS_EXPORT_ENABLED, "RUSTFS_OBS_METRICS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -56,6 +56,7 @@ tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
sysinfo = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
pyroscope = { workspace = true, features= ["backend-pprof-rs"]}
|
||||
walkdir = { workspace = true }
|
||||
|
||||
|
||||
|
||||
@@ -33,13 +33,14 @@ use rustfs_config::observability::{
|
||||
ENV_OBS_LOG_MATCH_MODE, ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES,
|
||||
ENV_OBS_LOG_MIN_FILE_AGE_SECONDS, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOG_STDOUT_ENABLED, ENV_OBS_LOGGER_LEVEL,
|
||||
ENV_OBS_LOGS_EXPORT_ENABLED, ENV_OBS_METER_INTERVAL, ENV_OBS_METRIC_ENDPOINT, ENV_OBS_METRICS_EXPORT_ENABLED,
|
||||
ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED,
|
||||
ENV_OBS_USE_STDOUT,
|
||||
ENV_OBS_PROFILING_ENDPOINT, ENV_OBS_PROFILING_EXPORT_ENABLED, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME,
|
||||
ENV_OBS_SERVICE_VERSION, ENV_OBS_TRACE_ENDPOINT, ENV_OBS_TRACES_EXPORT_ENABLED, ENV_OBS_USE_STDOUT,
|
||||
};
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_FILENAME,
|
||||
DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED,
|
||||
DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT,
|
||||
DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO,
|
||||
SERVICE_VERSION, USE_STDOUT,
|
||||
};
|
||||
use rustfs_utils::{get_env_bool, get_env_f64, get_env_opt_str, get_env_str, get_env_u64, get_env_usize};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -92,12 +93,16 @@ pub struct OtelConfig {
|
||||
pub metric_endpoint: Option<String>,
|
||||
/// Dedicated log endpoint; overrides `endpoint` + `/v1/logs` fallback.
|
||||
pub log_endpoint: Option<String>,
|
||||
/// Dedicated profiling endpoint.
|
||||
pub profiling_endpoint: Option<String>,
|
||||
/// Whether to export distributed traces (default: `true`).
|
||||
pub traces_export_enabled: Option<bool>,
|
||||
/// Whether to export metrics (default: `true`).
|
||||
pub metrics_export_enabled: Option<bool>,
|
||||
/// Whether to export logs via OTLP (default: `true`).
|
||||
pub logs_export_enabled: Option<bool>,
|
||||
/// Whether to export profiles via pyroscope (default: `true`).
|
||||
pub profiling_export_enabled: Option<bool>,
|
||||
/// **[OTLP-only]** Mirror all signals to stdout in addition to OTLP export.
|
||||
/// Only applies when an OTLP endpoint is configured.
|
||||
pub use_stdout: Option<bool>,
|
||||
@@ -216,9 +221,11 @@ impl OtelConfig {
|
||||
trace_endpoint: get_env_opt_str(ENV_OBS_TRACE_ENDPOINT),
|
||||
metric_endpoint: get_env_opt_str(ENV_OBS_METRIC_ENDPOINT),
|
||||
log_endpoint: get_env_opt_str(ENV_OBS_LOG_ENDPOINT),
|
||||
profiling_endpoint: get_env_opt_str(ENV_OBS_PROFILING_ENDPOINT),
|
||||
traces_export_enabled: Some(get_env_bool(ENV_OBS_TRACES_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED)),
|
||||
metrics_export_enabled: Some(get_env_bool(ENV_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED)),
|
||||
logs_export_enabled: Some(get_env_bool(ENV_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED)),
|
||||
profiling_export_enabled: Some(get_env_bool(ENV_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_PROFILING_EXPORT_ENABLED)),
|
||||
use_stdout: Some(use_stdout),
|
||||
sample_ratio: Some(get_env_f64(ENV_OBS_SAMPLE_RATIO, SAMPLE_RATIO)),
|
||||
meter_interval: Some(get_env_u64(ENV_OBS_METER_INTERVAL, METER_INTERVAL)),
|
||||
|
||||
@@ -20,12 +20,17 @@
|
||||
//! 1. Tracer provider — flushes pending spans.
|
||||
//! 2. Meter provider — flushes pending metrics.
|
||||
//! 3. Logger provider — flushes pending log records.
|
||||
//! 4. Cleanup task — aborted to prevent lingering background work.
|
||||
//! 5. Tracing worker guard — flushes buffered log lines written by
|
||||
//! 4. Profiling agent — flushes pending profiles.
|
||||
//! 5. Cleanup task — aborted to prevent lingering background work.
|
||||
//! 6. Tracing worker guard — flushes buffered log lines written by
|
||||
//! `tracing_appender`.
|
||||
//! 6. Stdout worker guard — flushes buffered log lines written to stdout.
|
||||
//! 7. Stdout worker guard — flushes buffered log lines written to stdout.
|
||||
|
||||
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
|
||||
#[cfg(unix)]
|
||||
use pyroscope::PyroscopeAgent;
|
||||
#[cfg(unix)]
|
||||
use pyroscope::pyroscope::PyroscopeAgentRunning;
|
||||
|
||||
/// RAII guard that owns all active OpenTelemetry providers and the
|
||||
/// `tracing_appender` worker guard.
|
||||
@@ -40,6 +45,8 @@ pub struct OtelGuard {
|
||||
pub(crate) meter_provider: Option<SdkMeterProvider>,
|
||||
/// Optional logger provider for OTLP log export.
|
||||
pub(crate) logger_provider: Option<SdkLoggerProvider>,
|
||||
#[cfg(unix)]
|
||||
pub(crate) profiling_agent: Option<PyroscopeAgent<PyroscopeAgentRunning>>,
|
||||
/// Handle to the background log-cleanup task; aborted on drop.
|
||||
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
/// Worker guard that keeps the non-blocking `tracing_appender` thread
|
||||
@@ -51,11 +58,13 @@ pub struct OtelGuard {
|
||||
|
||||
impl std::fmt::Debug for OtelGuard {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OtelGuard")
|
||||
.field("tracer_provider", &self.tracer_provider.is_some())
|
||||
let mut s = f.debug_struct("OtelGuard");
|
||||
s.field("tracer_provider", &self.tracer_provider.is_some())
|
||||
.field("meter_provider", &self.meter_provider.is_some())
|
||||
.field("logger_provider", &self.logger_provider.is_some())
|
||||
.field("cleanup_handle", &self.cleanup_handle.is_some())
|
||||
.field("logger_provider", &self.logger_provider.is_some());
|
||||
#[cfg(unix)]
|
||||
s.field("profiling_agent", &self.profiling_agent.is_some());
|
||||
s.field("cleanup_handle", &self.cleanup_handle.is_some())
|
||||
.field("tracing_guard", &self.tracing_guard.is_some())
|
||||
.field("stdout_guard", &self.stdout_guard.is_some())
|
||||
.finish()
|
||||
@@ -86,6 +95,16 @@ impl Drop for OtelGuard {
|
||||
eprintln!("Logger shutdown error: {err:?}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(agent) = self.profiling_agent.take() {
|
||||
match agent.stop() {
|
||||
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
|
||||
Ok(stopped) => {
|
||||
stopped.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(handle) = self.cleanup_handle.take() {
|
||||
handle.abort();
|
||||
eprintln!("Log cleanup task stopped");
|
||||
|
||||
@@ -139,6 +139,8 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
tracer_provider: None,
|
||||
meter_provider: None,
|
||||
logger_provider: None,
|
||||
#[cfg(unix)]
|
||||
profiling_agent: None,
|
||||
tracing_guard: Some(guard),
|
||||
stdout_guard: None,
|
||||
cleanup_handle: None,
|
||||
@@ -275,6 +277,8 @@ fn init_file_logging_internal(
|
||||
tracer_provider: None,
|
||||
meter_provider: None,
|
||||
logger_provider: None,
|
||||
#[cfg(unix)]
|
||||
profiling_agent: None,
|
||||
tracing_guard: Some(guard),
|
||||
stdout_guard,
|
||||
cleanup_handle: Some(cleanup_handle),
|
||||
|
||||
@@ -49,9 +49,13 @@ use opentelemetry_sdk::{
|
||||
metrics::{PeriodicReader, SdkMeterProvider},
|
||||
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use pyroscope::PyroscopeAgent;
|
||||
#[cfg(unix)]
|
||||
use pyroscope::pyroscope::PyroscopeAgentRunning;
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED,
|
||||
DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
|
||||
DEFAULT_OBS_PROFILING_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
|
||||
};
|
||||
use std::{io::IsTerminal, time::Duration};
|
||||
use tracing::info;
|
||||
@@ -137,6 +141,9 @@ pub(super) fn init_observability_http(
|
||||
// ── Logger provider (HTTP) ────────────────────────────────────────────────
|
||||
let logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
let profiling_agent = init_profiler(config);
|
||||
|
||||
// ── Tracing subscriber registry ───────────────────────────────────────────
|
||||
// Build an optional stdout formatting layer. When `log_stdout_enabled` is
|
||||
// false the field is `None` and tracing-subscriber will skip it.
|
||||
@@ -188,6 +195,8 @@ pub(super) fn init_observability_http(
|
||||
tracer_provider,
|
||||
meter_provider,
|
||||
logger_provider,
|
||||
#[cfg(unix)]
|
||||
profiling_agent,
|
||||
tracing_guard: None,
|
||||
stdout_guard: None,
|
||||
cleanup_handle: None,
|
||||
@@ -309,6 +318,46 @@ fn build_logger_provider(
|
||||
Ok(Some(builder.build()))
|
||||
}
|
||||
|
||||
/// Starts the Pyroscope continuous profiling agent if `ENV_OBS_PROFILING_ENDPOINT` is set.
|
||||
/// No-op and returns None on non-unix platforms.
|
||||
#[cfg(unix)]
|
||||
fn init_profiler(config: &OtelConfig) -> Option<PyroscopeAgent<PyroscopeAgentRunning>> {
|
||||
use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
|
||||
use pyroscope::pyroscope::PyroscopeAgentBuilder;
|
||||
use rustfs_config::VERSION;
|
||||
|
||||
if !config
|
||||
.profiling_export_enabled
|
||||
.unwrap_or(DEFAULT_OBS_PROFILING_EXPORT_ENABLED)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let endpoint = config.profiling_endpoint.as_ref()?.as_str();
|
||||
if endpoint.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Configure Pyroscope Agent
|
||||
let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
|
||||
let version = config.service_version.as_deref().unwrap_or(VERSION);
|
||||
let sample_rate = 100; // 100 Hz
|
||||
|
||||
let agent = PyroscopeAgentBuilder::new(endpoint, service_name, sample_rate, "pyroscope-rs", "1.0.1", backend)
|
||||
.tags(vec![("version", version)]) // TODO: add git commit tag
|
||||
.build()
|
||||
.ok()?;
|
||||
|
||||
match agent.start() {
|
||||
Ok(agent) => Some(agent),
|
||||
Err(err) => {
|
||||
eprintln!("Pyroscope agent start error: {err:?}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a stdout periodic metrics reader for the given interval.
|
||||
fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout::MetricExporter> {
|
||||
PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default())
|
||||
|
||||
Reference in New Issue
Block a user