mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-26 05:56:50 +00:00
feat: add TraceLayer for HTTP service and improve metrics (#361)
* improve code for opentelemetry and add system metrics * feat: add TraceLayer for HTTP service and improve metrics - Add TraceLayer to HTTP server for request tracing - Implement system metrics for process monitoring - Optimize init_telemetry method for better resource management - Add graceful shutdown handling for telemetry components - Fix GracefulShutdown ownership issues with Arc wrapper * improve code for init_process_observer * remove tomlfmt.toml * Translation comment * improve code for console CompressionLayer params
This commit is contained in:
houseme
Generated
+118
-35
@@ -525,7 +525,6 @@ version = "0.4.19"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"
|
checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"brotli",
|
|
||||||
"bzip2",
|
"bzip2",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
@@ -5251,6 +5250,15 @@ dependencies = [
|
|||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ntapi"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
|
||||||
|
dependencies = [
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nu-ansi-term"
|
name = "nu-ansi-term"
|
||||||
version = "0.46.0"
|
version = "0.46.0"
|
||||||
@@ -5413,6 +5421,29 @@ version = "0.3.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
|
checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nvml-wrapper"
|
||||||
|
version = "0.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0c9bff0aa1d48904a1385ea2a8b97576fbdcbc9a3cfccd0d31fe978e1c4038c5"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.9.0",
|
||||||
|
"libloading 0.8.6",
|
||||||
|
"nvml-wrapper-sys",
|
||||||
|
"static_assertions",
|
||||||
|
"thiserror 1.0.69",
|
||||||
|
"wrapcenum-derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nvml-wrapper-sys"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "698d45156f28781a4e79652b6ebe2eaa0589057d588d3aec1333f6466f13fcb5"
|
||||||
|
dependencies = [
|
||||||
|
"libloading 0.8.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "objc"
|
name = "objc"
|
||||||
version = "0.2.7"
|
version = "0.2.7"
|
||||||
@@ -5716,19 +5747,6 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "opentelemetry-prometheus"
|
|
||||||
version = "0.29.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "098a71a4430bb712be6130ed777335d2e5b19bc8566de5f2edddfce906def6ab"
|
|
||||||
dependencies = [
|
|
||||||
"once_cell",
|
|
||||||
"opentelemetry",
|
|
||||||
"opentelemetry_sdk",
|
|
||||||
"prometheus",
|
|
||||||
"tracing",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "opentelemetry-proto"
|
name = "opentelemetry-proto"
|
||||||
version = "0.29.0"
|
version = "0.29.0"
|
||||||
@@ -6487,21 +6505,6 @@ dependencies = [
|
|||||||
"version_check",
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "prometheus"
|
|
||||||
version = "0.14.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"fnv",
|
|
||||||
"lazy_static",
|
|
||||||
"memchr",
|
|
||||||
"parking_lot 0.12.3",
|
|
||||||
"protobuf",
|
|
||||||
"thiserror 2.0.12",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "prost"
|
name = "prost"
|
||||||
version = "0.13.5"
|
version = "0.13.5"
|
||||||
@@ -7248,6 +7251,7 @@ dependencies = [
|
|||||||
"mime",
|
"mime",
|
||||||
"mime_guess",
|
"mime_guess",
|
||||||
"netif",
|
"netif",
|
||||||
|
"opentelemetry",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"policy",
|
"policy",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
@@ -7332,18 +7336,19 @@ dependencies = [
|
|||||||
"chrono",
|
"chrono",
|
||||||
"config",
|
"config",
|
||||||
"local-ip-address",
|
"local-ip-address",
|
||||||
|
"nvml-wrapper",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-appender-tracing",
|
"opentelemetry-appender-tracing",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry-prometheus",
|
|
||||||
"opentelemetry-semantic-conventions",
|
"opentelemetry-semantic-conventions",
|
||||||
"opentelemetry-stdout",
|
"opentelemetry-stdout",
|
||||||
"opentelemetry_sdk",
|
"opentelemetry_sdk",
|
||||||
"prometheus",
|
|
||||||
"rdkafka",
|
"rdkafka",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"smallvec",
|
||||||
|
"sysinfo",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -8274,6 +8279,19 @@ dependencies = [
|
|||||||
"syn 2.0.100",
|
"syn 2.0.100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sysinfo"
|
||||||
|
version = "0.34.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a4b93974b3d3aeaa036504b8eefd4c039dced109171c1ae973f1dc63b2c7e4b2"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"memchr",
|
||||||
|
"ntapi",
|
||||||
|
"objc2-core-foundation",
|
||||||
|
"windows 0.57.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "system-configuration"
|
name = "system-configuration"
|
||||||
version = "0.6.1"
|
version = "0.6.1"
|
||||||
@@ -8342,7 +8360,7 @@ dependencies = [
|
|||||||
"tao-macros",
|
"tao-macros",
|
||||||
"unicode-segmentation",
|
"unicode-segmentation",
|
||||||
"url",
|
"url",
|
||||||
"windows",
|
"windows 0.58.0",
|
||||||
"windows-core 0.58.0",
|
"windows-core 0.58.0",
|
||||||
"windows-version",
|
"windows-version",
|
||||||
"x11-dl",
|
"x11-dl",
|
||||||
@@ -9483,7 +9501,7 @@ checksum = "6f61ff3d9d0ee4efcb461b14eb3acfda2702d10dc329f339303fc3e57215ae2c"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"webview2-com-macros",
|
"webview2-com-macros",
|
||||||
"webview2-com-sys",
|
"webview2-com-sys",
|
||||||
"windows",
|
"windows 0.58.0",
|
||||||
"windows-core 0.58.0",
|
"windows-core 0.58.0",
|
||||||
"windows-implement 0.58.0",
|
"windows-implement 0.58.0",
|
||||||
"windows-interface 0.58.0",
|
"windows-interface 0.58.0",
|
||||||
@@ -9507,7 +9525,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "a3a3e2eeb58f82361c93f9777014668eb3d07e7d174ee4c819575a9208011886"
|
checksum = "a3a3e2eeb58f82361c93f9777014668eb3d07e7d174ee4c819575a9208011886"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"windows",
|
"windows 0.58.0",
|
||||||
"windows-core 0.58.0",
|
"windows-core 0.58.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -9554,6 +9572,16 @@ version = "0.4.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows"
|
||||||
|
version = "0.57.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
|
||||||
|
dependencies = [
|
||||||
|
"windows-core 0.57.0",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows"
|
name = "windows"
|
||||||
version = "0.58.0"
|
version = "0.58.0"
|
||||||
@@ -9564,6 +9592,18 @@ dependencies = [
|
|||||||
"windows-targets 0.52.6",
|
"windows-targets 0.52.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-core"
|
||||||
|
version = "0.57.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d"
|
||||||
|
dependencies = [
|
||||||
|
"windows-implement 0.57.0",
|
||||||
|
"windows-interface 0.57.0",
|
||||||
|
"windows-result 0.1.2",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-core"
|
name = "windows-core"
|
||||||
version = "0.58.0"
|
version = "0.58.0"
|
||||||
@@ -9590,6 +9630,17 @@ dependencies = [
|
|||||||
"windows-strings 0.4.0",
|
"windows-strings 0.4.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-implement"
|
||||||
|
version = "0.57.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.100",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-implement"
|
name = "windows-implement"
|
||||||
version = "0.58.0"
|
version = "0.58.0"
|
||||||
@@ -9612,6 +9663,17 @@ dependencies = [
|
|||||||
"syn 2.0.100",
|
"syn 2.0.100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-interface"
|
||||||
|
version = "0.57.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.100",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-interface"
|
name = "windows-interface"
|
||||||
version = "0.58.0"
|
version = "0.58.0"
|
||||||
@@ -9651,6 +9713,15 @@ dependencies = [
|
|||||||
"windows-targets 0.53.0",
|
"windows-targets 0.53.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-result"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-result"
|
name = "windows-result"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
@@ -10020,6 +10091,18 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wrapcenum-derive"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a76ff259533532054cfbaefb115c613203c73707017459206380f03b3b3f266e"
|
||||||
|
dependencies = [
|
||||||
|
"darling",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.100",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "write16"
|
name = "write16"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
@@ -10066,7 +10149,7 @@ dependencies = [
|
|||||||
"webkit2gtk",
|
"webkit2gtk",
|
||||||
"webkit2gtk-sys",
|
"webkit2gtk-sys",
|
||||||
"webview2-com",
|
"webview2-com",
|
||||||
"windows",
|
"windows 0.58.0",
|
||||||
"windows-core 0.58.0",
|
"windows-core 0.58.0",
|
||||||
"windows-version",
|
"windows-version",
|
||||||
"x11-dl",
|
"x11-dl",
|
||||||
|
|||||||
+24
-22
@@ -1,21 +1,21 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = [
|
members = [
|
||||||
"madmin", # Management dashboard and admin API interface
|
"madmin", # Management dashboard and admin API interface
|
||||||
"rustfs", # Core file system implementation
|
"rustfs", # Core file system implementation
|
||||||
"ecstore", # Erasure coding storage implementation
|
"ecstore", # Erasure coding storage implementation
|
||||||
"e2e_test", # End-to-end test suite
|
"e2e_test", # End-to-end test suite
|
||||||
"common/common", # Shared utilities and data structures
|
"common/common", # Shared utilities and data structures
|
||||||
"common/lock", # Distributed locking implementation
|
"common/lock", # Distributed locking implementation
|
||||||
"common/protos", # Protocol buffer definitions
|
"common/protos", # Protocol buffer definitions
|
||||||
"common/workers", # Worker thread pools and task scheduling
|
"common/workers", # Worker thread pools and task scheduling
|
||||||
"iam", # Identity and Access Management
|
"iam", # Identity and Access Management
|
||||||
"crypto", # Cryptography and security features
|
"crypto", # Cryptography and security features
|
||||||
"cli/rustfs-gui", # Graphical user interface client
|
"cli/rustfs-gui", # Graphical user interface client
|
||||||
"crates/obs", # Observability utilities
|
"crates/obs", # Observability utilities
|
||||||
"crates/event-notifier", # Event notification system
|
"crates/event-notifier", # Event notification system
|
||||||
"s3select/api", # S3 Select API interface
|
"s3select/api", # S3 Select API interface
|
||||||
"s3select/query", # S3 Select query engine
|
"s3select/query", # S3 Select query engine
|
||||||
"appauth", # Application authentication and authorization
|
"appauth", # Application authentication and authorization
|
||||||
]
|
]
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
|
|
||||||
@@ -93,6 +93,7 @@ md-5 = "0.10.6"
|
|||||||
mime = "0.3.17"
|
mime = "0.3.17"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
netif = "0.1.6"
|
netif = "0.1.6"
|
||||||
|
nvml-wrapper = "0.10.0"
|
||||||
object_store = "0.11.2"
|
object_store = "0.11.2"
|
||||||
opentelemetry = { version = "0.29.1" }
|
opentelemetry = { version = "0.29.1" }
|
||||||
opentelemetry-appender-tracing = { version = "0.29.1", features = [
|
opentelemetry-appender-tracing = { version = "0.29.1", features = [
|
||||||
@@ -102,10 +103,7 @@ opentelemetry-appender-tracing = { version = "0.29.1", features = [
|
|||||||
opentelemetry_sdk = { version = "0.29.0" }
|
opentelemetry_sdk = { version = "0.29.0" }
|
||||||
opentelemetry-stdout = { version = "0.29.0" }
|
opentelemetry-stdout = { version = "0.29.0" }
|
||||||
opentelemetry-otlp = { version = "0.29.0" }
|
opentelemetry-otlp = { version = "0.29.0" }
|
||||||
opentelemetry-prometheus = { version = "0.29.1" }
|
opentelemetry-semantic-conventions = { version = "0.29.0", features = ["semconv_experimental"] }
|
||||||
opentelemetry-semantic-conventions = { version = "0.29.0", features = [
|
|
||||||
"semconv_experimental",
|
|
||||||
] }
|
|
||||||
parking_lot = "0.12.3"
|
parking_lot = "0.12.3"
|
||||||
pin-project-lite = "0.2.16"
|
pin-project-lite = "0.2.16"
|
||||||
prometheus = "0.14.0"
|
prometheus = "0.14.0"
|
||||||
@@ -143,10 +141,11 @@ serde = { version = "1.0.219", features = ["derive"] }
|
|||||||
serde_json = "1.0.140"
|
serde_json = "1.0.140"
|
||||||
serde_urlencoded = "0.7.1"
|
serde_urlencoded = "0.7.1"
|
||||||
serde_with = "3.12.0"
|
serde_with = "3.12.0"
|
||||||
smallvec = { version = "1.15.0", features = ["serde"] }
|
|
||||||
strum = { version = "0.27.1", features = ["derive"] }
|
|
||||||
sha2 = "0.10.8"
|
sha2 = "0.10.8"
|
||||||
|
smallvec = { version = "1.15.0", features = ["serde"] }
|
||||||
snafu = "0.8.5"
|
snafu = "0.8.5"
|
||||||
|
strum = { version = "0.27.1", features = ["derive"] }
|
||||||
|
sysinfo = "0.34.2"
|
||||||
tempfile = "3.19.1"
|
tempfile = "3.19.1"
|
||||||
test-case = "3.3.1"
|
test-case = "3.3.1"
|
||||||
thiserror = "2.0.12"
|
thiserror = "2.0.12"
|
||||||
@@ -192,7 +191,10 @@ inherits = "dev"
|
|||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
opt-level = 3
|
opt-level = 3
|
||||||
lto = "thin"
|
lto = "fat"
|
||||||
|
codegen-units = 1
|
||||||
|
panic = "abort" # Optional, remove the panic expansion code
|
||||||
|
strip = true # strip symbol information to reduce binary size
|
||||||
|
|
||||||
[profile.production]
|
[profile.production]
|
||||||
inherits = "release"
|
inherits = "release"
|
||||||
|
|||||||
@@ -149,7 +149,6 @@ impl NotifierConfig {
|
|||||||
)
|
)
|
||||||
.build()
|
.build()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
println!("Loaded config: {:?}", app_config);
|
|
||||||
match app_config.try_deserialize::<NotifierConfig>() {
|
match app_config.try_deserialize::<NotifierConfig>() {
|
||||||
Ok(app_config) => {
|
Ok(app_config) => {
|
||||||
println!("Parsed AppConfig: {:?} \n", app_config);
|
println!("Parsed AppConfig: {:?} \n", app_config);
|
||||||
|
|||||||
@@ -11,32 +11,35 @@ workspace = true
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["file"]
|
default = ["file"]
|
||||||
|
file = []
|
||||||
|
gpu = ["dep:nvml-wrapper"]
|
||||||
kafka = ["dep:rdkafka"]
|
kafka = ["dep:rdkafka"]
|
||||||
webhook = ["dep:reqwest"]
|
webhook = ["dep:reqwest"]
|
||||||
file = []
|
full = ["file", "gpu", "kafka", "webhook"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
async-trait = { workspace = true }
|
async-trait = { workspace = true }
|
||||||
chrono = { workspace = true }
|
chrono = { workspace = true }
|
||||||
config = { workspace = true }
|
config = { workspace = true }
|
||||||
|
nvml-wrapper = { workspace = true, optional = true }
|
||||||
opentelemetry = { workspace = true }
|
opentelemetry = { workspace = true }
|
||||||
opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
|
opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] }
|
||||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||||
opentelemetry-stdout = { workspace = true }
|
opentelemetry-stdout = { workspace = true }
|
||||||
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
|
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "gzip-tonic"] }
|
||||||
opentelemetry-prometheus = { workspace = true }
|
|
||||||
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
||||||
prometheus = { workspace = true }
|
|
||||||
serde = { workspace = true }
|
serde = { workspace = true }
|
||||||
|
smallvec = { workspace = true, features = ["serde"] }
|
||||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||||
tracing-core = { workspace = true }
|
tracing-core = { workspace = true }
|
||||||
tracing-error = { workspace = true }
|
tracing-error = { workspace = true }
|
||||||
tracing-opentelemetry = { workspace = true }
|
tracing-opentelemetry = { workspace = true }
|
||||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||||
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
||||||
reqwest = { workspace = true, optional = true, default-features = false }
|
reqwest = { workspace = true, optional = true, default-features = false }
|
||||||
serde_json = { workspace = true }
|
serde_json = { workspace = true }
|
||||||
|
sysinfo = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
local-ip-address = { workspace = true }
|
local-ip-address = { workspace = true }
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ batch_timeout_ms = 1000 # Default is 100ms if not specified
|
|||||||
|
|
||||||
[sinks.file]
|
[sinks.file]
|
||||||
enabled = true
|
enabled = true
|
||||||
path = "logs/app.log"
|
path = "deploy/logs/app.log"
|
||||||
batch_size = 100
|
batch_size = 100
|
||||||
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
|
batch_timeout_ms = 1000 # Default is 8192 bytes if not specified
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
use opentelemetry::global;
|
use opentelemetry::global;
|
||||||
use rustfs_obs::{get_logger, init_obs, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
use rustfs_obs::{get_logger, init_obs, init_process_observer, load_config, log_info, BaseLogEntry, ServerLogEntry};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, SystemTime};
|
||||||
use tracing::{info, instrument};
|
use tracing::{error, info, instrument};
|
||||||
use tracing_core::Level;
|
use tracing_core::Level;
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
@@ -38,6 +38,11 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
|
|||||||
&[opentelemetry::KeyValue::new("operation", "run")],
|
&[opentelemetry::KeyValue::new("operation", "run")],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
match init_process_observer(meter).await {
|
||||||
|
Ok(_) => info!("Process observer initialized successfully"),
|
||||||
|
Err(e) => error!("Failed to initialize process observer: {:?}", e),
|
||||||
|
}
|
||||||
|
|
||||||
let base_entry = BaseLogEntry::new()
|
let base_entry = BaseLogEntry::new()
|
||||||
.message(Some("run logger api_handler info".to_string()))
|
.message(Some("run logger api_handler info".to_string()))
|
||||||
.request_id(Some("request_id".to_string()))
|
.request_id(Some("request_id".to_string()))
|
||||||
|
|||||||
@@ -178,7 +178,6 @@ pub struct AppConfig {
|
|||||||
pub logger: Option<LoggerConfig>,
|
pub logger: Option<LoggerConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// 为 AppConfig 实现 Default
|
|
||||||
impl AppConfig {
|
impl AppConfig {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -189,6 +188,7 @@ impl AppConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// implement default for AppConfig
|
||||||
impl Default for AppConfig {
|
impl Default for AppConfig {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self::new()
|
Self::new()
|
||||||
|
|||||||
@@ -16,11 +16,27 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
|
|||||||
|
|
||||||
/// Error type for global guard operations
|
/// Error type for global guard operations
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
pub enum GuardError {
|
pub enum GlobalError {
|
||||||
#[error("Failed to set global guard: {0}")]
|
#[error("Failed to set global guard: {0}")]
|
||||||
SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
|
SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
|
||||||
#[error("Global guard not initialized")]
|
#[error("Global guard not initialized")]
|
||||||
NotInitialized,
|
NotInitialized,
|
||||||
|
#[error("Global system metrics err: {0}")]
|
||||||
|
MetricsError(String),
|
||||||
|
#[error("Failed to get current PID: {0}")]
|
||||||
|
PidError(String),
|
||||||
|
#[error("Process with PID {0} not found")]
|
||||||
|
ProcessNotFound(u32),
|
||||||
|
#[error("Failed to get physical core count")]
|
||||||
|
CoreCountError,
|
||||||
|
#[error("GPU initialization failed: {0}")]
|
||||||
|
GpuInitError(String),
|
||||||
|
#[error("GPU device not found: {0}")]
|
||||||
|
GpuDeviceError(String),
|
||||||
|
#[error("Failed to send log: {0}")]
|
||||||
|
SendFailed(&'static str),
|
||||||
|
#[error("Operation timed out: {0}")]
|
||||||
|
Timeout(&'static str),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set the global guard for OpenTelemetry
|
/// Set the global guard for OpenTelemetry
|
||||||
@@ -43,9 +59,9 @@ pub enum GuardError {
|
|||||||
/// Ok(())
|
/// Ok(())
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
|
pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
|
||||||
info!("Initializing global OpenTelemetry guard");
|
info!("Initializing global OpenTelemetry guard");
|
||||||
GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GuardError::SetError)
|
GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the global guard for OpenTelemetry
|
/// Get the global guard for OpenTelemetry
|
||||||
@@ -65,8 +81,8 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GuardError> {
|
|||||||
/// Ok(())
|
/// Ok(())
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GuardError> {
|
pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
|
||||||
GLOBAL_GUARD.get().cloned().ok_or(GuardError::NotInitialized)
|
GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Try to get the global guard for OpenTelemetry
|
/// Try to get the global guard for OpenTelemetry
|
||||||
@@ -84,6 +100,6 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_get_uninitialized_guard() {
|
async fn test_get_uninitialized_guard() {
|
||||||
let result = get_global_guard();
|
let result = get_global_guard();
|
||||||
assert!(matches!(result, Err(GuardError::NotInitialized)));
|
assert!(matches!(result, Err(GlobalError::NotInitialized)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+50
-30
@@ -1,23 +1,24 @@
|
|||||||
/// # obs
|
//! # RustFS Observability
|
||||||
///
|
//!
|
||||||
/// `obs` is a logging and observability library for Rust.
|
//! provides tools for system and service monitoring
|
||||||
/// It provides a simple and easy-to-use interface for logging and observability.
|
//!
|
||||||
/// It is built on top of the `log` crate and `opentelemetry` crate.
|
//! ## feature mark
|
||||||
///
|
//!
|
||||||
/// ## Features
|
//! - `file`: enable file logging enabled by default
|
||||||
/// - Structured logging
|
//! - `gpu`: gpu monitoring function
|
||||||
/// - Distributed tracing
|
//! - `kafka`: enable kafka metric output
|
||||||
/// - Metrics collection
|
//! - `webhook`: enable webhook notifications
|
||||||
/// - Log processing worker
|
//! - `full`: includes all functions
|
||||||
/// - Multiple sinks
|
//!
|
||||||
/// - Configuration-based setup
|
//! to enable gpu monitoring add in cargo toml
|
||||||
/// - Telemetry guard
|
//!
|
||||||
/// - Global logger
|
//! ```toml
|
||||||
/// - Log levels
|
//! # using gpu monitoring
|
||||||
/// - Log entry types
|
//! rustfs-obs = { version = "0.1.0", features = ["gpu"] }
|
||||||
/// - Log record
|
//!
|
||||||
/// - Object version
|
//! # use all functions
|
||||||
/// - Local IP address
|
//! rustfs-obs = { version = "0.1.0", features = ["full"] }
|
||||||
|
//! ```
|
||||||
///
|
///
|
||||||
/// ## Usage
|
/// ## Usage
|
||||||
///
|
///
|
||||||
@@ -32,28 +33,34 @@ mod entry;
|
|||||||
mod global;
|
mod global;
|
||||||
mod logger;
|
mod logger;
|
||||||
mod sink;
|
mod sink;
|
||||||
|
mod system;
|
||||||
mod telemetry;
|
mod telemetry;
|
||||||
mod utils;
|
mod utils;
|
||||||
mod worker;
|
mod worker;
|
||||||
|
|
||||||
|
use crate::logger::InitLogStatus;
|
||||||
pub use config::load_config;
|
pub use config::load_config;
|
||||||
pub use config::{AppConfig, OtelConfig};
|
#[cfg(feature = "file")]
|
||||||
|
pub use config::FileSinkConfig;
|
||||||
|
#[cfg(feature = "kafka")]
|
||||||
|
pub use config::KafkaSinkConfig;
|
||||||
|
#[cfg(feature = "webhook")]
|
||||||
|
pub use config::WebhookSinkConfig;
|
||||||
|
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
|
||||||
pub use entry::args::Args;
|
pub use entry::args::Args;
|
||||||
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
||||||
pub use entry::base::BaseLogEntry;
|
pub use entry::base::BaseLogEntry;
|
||||||
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
||||||
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
||||||
pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GuardError};
|
pub use global::{get_global_guard, set_global_guard, try_get_global_guard, GlobalError};
|
||||||
pub use logger::{ensure_logger_initialized, log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
pub use logger::Logger;
|
||||||
pub use logger::{get_global_logger, init_global_logger, locked_logger, start_logger};
|
pub use logger::{get_global_logger, init_global_logger, start_logger};
|
||||||
pub use logger::{log_init_state, InitLogStatus};
|
pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||||
pub use logger::{LogError, Logger};
|
|
||||||
pub use sink::Sink;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
pub use system::{init_process_observer, init_process_observer_for_pid};
|
||||||
pub use telemetry::init_telemetry;
|
pub use telemetry::init_telemetry;
|
||||||
pub use telemetry::{get_global_registry, metrics};
|
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
pub use utils::{get_local_ip, get_local_ip_with_default};
|
use tracing::{error, info};
|
||||||
pub use worker::start_worker;
|
|
||||||
|
|
||||||
/// Initialize the observability module
|
/// Initialize the observability module
|
||||||
///
|
///
|
||||||
@@ -74,6 +81,19 @@ pub async fn init_obs(config: AppConfig) -> (Arc<Mutex<Logger>>, telemetry::Otel
|
|||||||
let guard = init_telemetry(&config.observability);
|
let guard = init_telemetry(&config.observability);
|
||||||
let sinks = sink::create_sinks(&config).await;
|
let sinks = sink::create_sinks(&config).await;
|
||||||
let logger = init_global_logger(&config, sinks).await;
|
let logger = init_global_logger(&config, sinks).await;
|
||||||
|
let obs_config = config.observability.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let result = InitLogStatus::init_start_log(&obs_config).await;
|
||||||
|
match result {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Logger initialized successfully");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to initialize logger: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
(logger, guard)
|
(logger, guard)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+21
-81
@@ -1,5 +1,6 @@
|
|||||||
use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
|
use crate::global::{ENVIRONMENT, SERVICE_NAME, SERVICE_VERSION};
|
||||||
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, OtelConfig, ServerLogEntry, Sink, UnifiedLogEntry};
|
use crate::sink::Sink;
|
||||||
|
use crate::{AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||||
@@ -43,26 +44,26 @@ impl Logger {
|
|||||||
|
|
||||||
/// Log a server entry
|
/// Log a server entry
|
||||||
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
||||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), LogError> {
|
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
|
||||||
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log an audit entry
|
/// Log an audit entry
|
||||||
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
||||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), LogError> {
|
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
|
||||||
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log a console entry
|
/// Log a console entry
|
||||||
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
||||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), LogError> {
|
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
|
||||||
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Asynchronous logging of unified log entries
|
/// Asynchronous logging of unified log entries
|
||||||
#[tracing::instrument(skip(self), fields(log_source = "logger"))]
|
#[tracing::instrument(skip(self), fields(log_source = "logger"))]
|
||||||
#[tracing::instrument(level = "error", skip_all)]
|
#[tracing::instrument(level = "error", skip_all)]
|
||||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), LogError> {
|
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
|
||||||
// Extract information for tracing based on entry type
|
// Extract information for tracing based on entry type
|
||||||
match &entry {
|
match &entry {
|
||||||
UnifiedLogEntry::Server(server) => {
|
UnifiedLogEntry::Server(server) => {
|
||||||
@@ -123,11 +124,11 @@ impl Logger {
|
|||||||
tracing::warn!("Log queue full, applying backpressure");
|
tracing::warn!("Log queue full, applying backpressure");
|
||||||
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
||||||
Ok(Ok(_)) => Ok(()),
|
Ok(Ok(_)) => Ok(()),
|
||||||
Ok(Err(_)) => Err(LogError::SendFailed("Channel closed")),
|
Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
|
||||||
Err(_) => Err(LogError::Timeout("Queue backpressure timeout")),
|
Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(LogError::SendFailed("Logger channel closed")),
|
Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,7 +161,7 @@ impl Logger {
|
|||||||
request_id: Option<String>,
|
request_id: Option<String>,
|
||||||
user_id: Option<String>,
|
user_id: Option<String>,
|
||||||
fields: Vec<(String, String)>,
|
fields: Vec<(String, String)>,
|
||||||
) -> Result<(), LogError> {
|
) -> Result<(), GlobalError> {
|
||||||
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
||||||
|
|
||||||
let server_entry = ServerLogEntry::new(level, source.to_string())
|
let server_entry = ServerLogEntry::new(level, source.to_string())
|
||||||
@@ -190,7 +191,7 @@ impl Logger {
|
|||||||
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), LogError> {
|
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
|
||||||
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,31 +209,12 @@ impl Logger {
|
|||||||
/// let _ = logger.shutdown().await;
|
/// let _ = logger.shutdown().await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn shutdown(self) -> Result<(), LogError> {
|
pub async fn shutdown(self) -> Result<(), GlobalError> {
|
||||||
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log error type
|
|
||||||
/// This enum defines the error types that can occur when logging.
|
|
||||||
/// It is used to provide more detailed error information.
|
|
||||||
/// # Example
|
|
||||||
/// ```
|
|
||||||
/// use rustfs_obs::LogError;
|
|
||||||
/// use thiserror::Error;
|
|
||||||
///
|
|
||||||
/// LogError::SendFailed("Failed to send log");
|
|
||||||
/// LogError::Timeout("Operation timed out");
|
|
||||||
/// ```
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
|
||||||
pub enum LogError {
|
|
||||||
#[error("Failed to send log: {0}")]
|
|
||||||
SendFailed(&'static str),
|
|
||||||
#[error("Operation timed out: {0}")]
|
|
||||||
Timeout(&'static str),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Start the log module
|
/// Start the log module
|
||||||
/// This function starts the log module.
|
/// This function starts the log module.
|
||||||
/// It initializes the logger and starts the worker to process logs.
|
/// It initializes the logger and starts the worker to process logs.
|
||||||
@@ -297,48 +279,6 @@ pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
|
|||||||
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the global logger instance with a lock
|
|
||||||
/// This function returns a reference to the global logger instance with a lock.
|
|
||||||
/// It is used to ensure that the logger is thread-safe.
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// A reference to the global logger instance with a lock
|
|
||||||
///
|
|
||||||
/// # Example
|
|
||||||
/// ```
|
|
||||||
/// use rustfs_obs::locked_logger;
|
|
||||||
///
|
|
||||||
/// async fn example() {
|
|
||||||
/// let logger = locked_logger().await;
|
|
||||||
/// }
|
|
||||||
/// ```
|
|
||||||
pub async fn locked_logger() -> tokio::sync::MutexGuard<'static, Logger> {
|
|
||||||
get_global_logger().lock().await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Initialize with default empty logger if needed (optional)
|
|
||||||
/// This function initializes the logger with a default empty logger if needed.
|
|
||||||
/// It is used to ensure that the logger is initialized before logging.
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// A reference to the global logger instance
|
|
||||||
///
|
|
||||||
/// # Example
|
|
||||||
/// ```
|
|
||||||
/// use rustfs_obs::ensure_logger_initialized;
|
|
||||||
///
|
|
||||||
/// let logger = ensure_logger_initialized();
|
|
||||||
/// ```
|
|
||||||
pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
|
||||||
if GLOBAL_LOGGER.get().is_none() {
|
|
||||||
let config = AppConfig::default();
|
|
||||||
let sinks = vec![];
|
|
||||||
let logger = Arc::new(Mutex::new(start_logger(&config, sinks)));
|
|
||||||
let _ = GLOBAL_LOGGER.set(logger);
|
|
||||||
}
|
|
||||||
GLOBAL_LOGGER.get().unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Log information
|
/// Log information
|
||||||
/// This function logs information messages.
|
/// This function logs information messages.
|
||||||
///
|
///
|
||||||
@@ -357,7 +297,7 @@ pub fn ensure_logger_initialized() -> &'static Arc<Mutex<Logger>> {
|
|||||||
/// let _ = log_info("This is an information message", "example").await;
|
/// let _ = log_info("This is an information message", "example").await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||||
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -375,7 +315,7 @@ pub async fn log_info(message: &str, source: &str) -> Result<(), LogError> {
|
|||||||
/// async fn example() {
|
/// async fn example() {
|
||||||
/// let _ = log_error("This is an error message", "example").await;
|
/// let _ = log_error("This is an error message", "example").await;
|
||||||
/// }
|
/// }
|
||||||
pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||||
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -395,7 +335,7 @@ pub async fn log_error(message: &str, source: &str) -> Result<(), LogError> {
|
|||||||
/// let _ = log_warn("This is a warning message", "example").await;
|
/// let _ = log_warn("This is a warning message", "example").await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||||
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -415,7 +355,7 @@ pub async fn log_warn(message: &str, source: &str) -> Result<(), LogError> {
|
|||||||
/// let _ = log_debug("This is a debug message", "example").await;
|
/// let _ = log_debug("This is a debug message", "example").await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||||
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,7 +376,7 @@ pub async fn log_debug(message: &str, source: &str) -> Result<(), LogError> {
|
|||||||
/// let _ = log_trace("This is a trace message", "example").await;
|
/// let _ = log_trace("This is a trace message", "example").await;
|
||||||
/// }
|
/// }
|
||||||
/// ```
|
/// ```
|
||||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), LogError> {
|
pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||||
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -467,7 +407,7 @@ pub async fn log_with_context(
|
|||||||
request_id: Option<String>,
|
request_id: Option<String>,
|
||||||
user_id: Option<String>,
|
user_id: Option<String>,
|
||||||
fields: Vec<(String, String)>,
|
fields: Vec<(String, String)>,
|
||||||
) -> Result<(), LogError> {
|
) -> Result<(), GlobalError> {
|
||||||
get_global_logger()
|
get_global_logger()
|
||||||
.lock()
|
.lock()
|
||||||
.await
|
.await
|
||||||
@@ -477,7 +417,7 @@ pub async fn log_with_context(
|
|||||||
|
|
||||||
/// Log initialization status
|
/// Log initialization status
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct InitLogStatus {
|
pub(crate) struct InitLogStatus {
|
||||||
pub timestamp: SystemTime,
|
pub timestamp: SystemTime,
|
||||||
pub service_name: String,
|
pub service_name: String,
|
||||||
pub version: String,
|
pub version: String,
|
||||||
@@ -508,14 +448,14 @@ impl InitLogStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), LogError> {
|
pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
|
||||||
let status = Self::new_config(config);
|
let status = Self::new_config(config);
|
||||||
log_init_state(Some(status)).await
|
log_init_state(Some(status)).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Log initialization details during system startup
|
/// Log initialization details during system startup
|
||||||
pub async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), LogError> {
|
async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
|
||||||
let status = status.unwrap_or_default();
|
let status = status.unwrap_or_default();
|
||||||
|
|
||||||
let base_entry = BaseLogEntry::new()
|
let base_entry = BaseLogEntry::new()
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
use crate::GlobalError;
|
||||||
|
use opentelemetry::KeyValue;
|
||||||
|
use sysinfo::{Pid, System};
|
||||||
|
|
||||||
|
pub const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid");
|
||||||
|
pub const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name");
|
||||||
|
pub const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path");
|
||||||
|
pub const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command");
|
||||||
|
|
||||||
|
/// Struct to hold process attributes
|
||||||
|
pub struct ProcessAttributes {
|
||||||
|
pub attributes: Vec<KeyValue>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProcessAttributes {
|
||||||
|
/// Creates a new instance of `ProcessAttributes` for the given PID.
|
||||||
|
pub fn new(pid: Pid, system: &mut System) -> Result<Self, GlobalError> {
|
||||||
|
system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true);
|
||||||
|
let process = system
|
||||||
|
.process(pid)
|
||||||
|
.ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?;
|
||||||
|
|
||||||
|
let attributes = vec![
|
||||||
|
KeyValue::new(PROCESS_PID, pid.as_u32() as i64),
|
||||||
|
KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()),
|
||||||
|
KeyValue::new(
|
||||||
|
PROCESS_EXECUTABLE_PATH,
|
||||||
|
process
|
||||||
|
.exe()
|
||||||
|
.map(|path| path.to_string_lossy().into_owned())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
),
|
||||||
|
KeyValue::new(
|
||||||
|
PROCESS_COMMAND,
|
||||||
|
process
|
||||||
|
.cmd()
|
||||||
|
.iter()
|
||||||
|
.fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
Ok(ProcessAttributes { attributes })
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
use crate::system::attributes::ProcessAttributes;
|
||||||
|
use crate::system::gpu::GpuCollector;
|
||||||
|
use crate::system::metrics::{Metrics, DIRECTION, INTERFACE, STATUS};
|
||||||
|
use crate::GlobalError;
|
||||||
|
use opentelemetry::KeyValue;
|
||||||
|
use std::time::SystemTime;
|
||||||
|
use sysinfo::{Networks, Pid, ProcessStatus, System};
|
||||||
|
use tokio::time::{sleep, Duration};
|
||||||
|
|
||||||
|
/// Collector is responsible for collecting system metrics and attributes.
|
||||||
|
/// It uses the sysinfo crate to gather information about the system and processes.
|
||||||
|
/// It also uses OpenTelemetry to record metrics.
|
||||||
|
pub struct Collector {
|
||||||
|
metrics: Metrics,
|
||||||
|
attributes: ProcessAttributes,
|
||||||
|
gpu_collector: GpuCollector,
|
||||||
|
pid: Pid,
|
||||||
|
system: System,
|
||||||
|
networks: Networks,
|
||||||
|
core_count: usize,
|
||||||
|
interval_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Collector {
|
||||||
|
pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result<Self, GlobalError> {
|
||||||
|
let mut system = System::new_all();
|
||||||
|
let attributes = ProcessAttributes::new(pid, &mut system)?;
|
||||||
|
let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?;
|
||||||
|
let metrics = Metrics::new(&meter);
|
||||||
|
let gpu_collector = GpuCollector::new(pid)?;
|
||||||
|
let networks = Networks::new_with_refreshed_list();
|
||||||
|
|
||||||
|
Ok(Collector {
|
||||||
|
metrics,
|
||||||
|
attributes,
|
||||||
|
gpu_collector,
|
||||||
|
pid,
|
||||||
|
system,
|
||||||
|
networks,
|
||||||
|
core_count,
|
||||||
|
interval_ms,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run(&mut self) -> Result<(), GlobalError> {
|
||||||
|
loop {
|
||||||
|
self.collect()?;
|
||||||
|
tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now());
|
||||||
|
sleep(Duration::from_millis(self.interval_ms)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect(&mut self) -> Result<(), GlobalError> {
|
||||||
|
self.system
|
||||||
|
.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true);
|
||||||
|
|
||||||
|
// refresh the network interface list and statistics
|
||||||
|
self.networks.refresh(false);
|
||||||
|
|
||||||
|
let process = self
|
||||||
|
.system
|
||||||
|
.process(self.pid)
|
||||||
|
.ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?;
|
||||||
|
|
||||||
|
// CPU metrics
|
||||||
|
let cpu_usage = process.cpu_usage();
|
||||||
|
self.metrics.cpu_usage.record(cpu_usage as f64, &[]);
|
||||||
|
self.metrics
|
||||||
|
.cpu_utilization
|
||||||
|
.record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes);
|
||||||
|
|
||||||
|
// Memory metrics
|
||||||
|
self.metrics
|
||||||
|
.memory_usage
|
||||||
|
.record(process.memory() as i64, &self.attributes.attributes);
|
||||||
|
self.metrics
|
||||||
|
.memory_virtual
|
||||||
|
.record(process.virtual_memory() as i64, &self.attributes.attributes);
|
||||||
|
|
||||||
|
// Disk I/O metrics
|
||||||
|
let disk_io = process.disk_usage();
|
||||||
|
self.metrics.disk_io.record(
|
||||||
|
disk_io.read_bytes as i64,
|
||||||
|
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(),
|
||||||
|
);
|
||||||
|
self.metrics.disk_io.record(
|
||||||
|
disk_io.written_bytes as i64,
|
||||||
|
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Network I/O indicators (corresponding to /system/network/internode)
|
||||||
|
let mut total_received: i64 = 0;
|
||||||
|
let mut total_transmitted: i64 = 0;
|
||||||
|
|
||||||
|
// statistics by interface
|
||||||
|
for (interface_name, data) in self.networks.iter() {
|
||||||
|
total_received += data.total_received() as i64;
|
||||||
|
total_transmitted += data.total_transmitted() as i64;
|
||||||
|
|
||||||
|
let received = data.received() as i64;
|
||||||
|
let transmitted = data.transmitted() as i64;
|
||||||
|
self.metrics.network_io_per_interface.record(
|
||||||
|
received,
|
||||||
|
&[
|
||||||
|
&self.attributes.attributes[..],
|
||||||
|
&[
|
||||||
|
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||||
|
KeyValue::new(DIRECTION, "received"),
|
||||||
|
],
|
||||||
|
]
|
||||||
|
.concat(),
|
||||||
|
);
|
||||||
|
self.metrics.network_io_per_interface.record(
|
||||||
|
transmitted,
|
||||||
|
&[
|
||||||
|
&self.attributes.attributes[..],
|
||||||
|
&[
|
||||||
|
KeyValue::new(INTERFACE, interface_name.to_string()),
|
||||||
|
KeyValue::new(DIRECTION, "transmitted"),
|
||||||
|
],
|
||||||
|
]
|
||||||
|
.concat(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// global statistics
|
||||||
|
self.metrics.network_io.record(
|
||||||
|
total_received,
|
||||||
|
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(),
|
||||||
|
);
|
||||||
|
self.metrics.network_io.record(
|
||||||
|
total_transmitted,
|
||||||
|
&[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Process status indicator (corresponding to /system/process)
|
||||||
|
let status_value = match process.status() {
|
||||||
|
ProcessStatus::Run => 0,
|
||||||
|
ProcessStatus::Sleep => 1,
|
||||||
|
ProcessStatus::Zombie => 2,
|
||||||
|
_ => 3, // other status
|
||||||
|
};
|
||||||
|
self.metrics.process_status.record(
|
||||||
|
status_value,
|
||||||
|
&[
|
||||||
|
&self.attributes.attributes[..],
|
||||||
|
&[KeyValue::new(STATUS, format!("{:?}", process.status()))],
|
||||||
|
]
|
||||||
|
.concat(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// GPU Metrics (Optional) Non-MacOS
|
||||||
|
self.gpu_collector.collect(&self.metrics, &self.attributes)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use crate::system::attributes::ProcessAttributes;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use crate::system::metrics::Metrics;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use crate::GlobalError;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use nvml_wrapper::enums::device::UsedGpuMemory;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use nvml_wrapper::Nvml;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use sysinfo::Pid;
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
|
/// `GpuCollector` is responsible for collecting GPU memory usage metrics.
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
pub struct GpuCollector {
|
||||||
|
nvml: Nvml,
|
||||||
|
pid: Pid,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
impl GpuCollector {
|
||||||
|
pub fn new(pid: Pid) -> Result<Self, GlobalError> {
|
||||||
|
let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?;
|
||||||
|
Ok(GpuCollector { nvml, pid })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> {
|
||||||
|
if let Ok(device) = self.nvml.device_by_index(0) {
|
||||||
|
if let Ok(gpu_stats) = device.running_compute_processes() {
|
||||||
|
for stat in gpu_stats.iter() {
|
||||||
|
if stat.pid == self.pid.as_u32() {
|
||||||
|
let memory_used = match stat.used_gpu_memory {
|
||||||
|
UsedGpuMemory::Used(bytes) => bytes,
|
||||||
|
UsedGpuMemory::Unavailable => 0,
|
||||||
|
};
|
||||||
|
metrics.gpu_memory_usage.record(memory_used, &attributes.attributes);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
warn!("Could not get GPU stats, recording 0 for GPU memory usage");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Err(GlobalError::GpuDeviceError("No GPU device found".to_string()));
|
||||||
|
}
|
||||||
|
metrics.gpu_memory_usage.record(0, &attributes.attributes);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "gpu"))]
|
||||||
|
pub struct GpuCollector;
|
||||||
|
|
||||||
|
#[cfg(not(feature = "gpu"))]
|
||||||
|
impl GpuCollector {
|
||||||
|
pub fn new(_pid: sysinfo::Pid) -> Result<Self, crate::GlobalError> {
|
||||||
|
Ok(GpuCollector)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn collect(
|
||||||
|
&self,
|
||||||
|
_metrics: &crate::system::metrics::Metrics,
|
||||||
|
_attributes: &crate::system::attributes::ProcessAttributes,
|
||||||
|
) -> Result<(), crate::GlobalError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
pub const PROCESS_CPU_USAGE: &str = "process.cpu.usage";
|
||||||
|
pub const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization";
|
||||||
|
pub const PROCESS_MEMORY_USAGE: &str = "process.memory.usage";
|
||||||
|
pub const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual";
|
||||||
|
pub const PROCESS_DISK_IO: &str = "process.disk.io";
|
||||||
|
pub const PROCESS_NETWORK_IO: &str = "process.network.io";
|
||||||
|
pub const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface";
|
||||||
|
pub const PROCESS_STATUS: &str = "process.status";
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage";
|
||||||
|
pub const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction");
|
||||||
|
pub const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status");
|
||||||
|
pub const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface");
|
||||||
|
|
||||||
|
/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring.
|
||||||
|
/// It contains various metrics such as CPU usage, memory usage,
|
||||||
|
/// disk I/O, network I/O, and process status.
|
||||||
|
///
|
||||||
|
/// The `Metrics` struct is designed to be used with OpenTelemetry's
|
||||||
|
/// metrics API to record and export these metrics.
|
||||||
|
///
|
||||||
|
/// The `new` method initializes the metrics using the provided
|
||||||
|
/// `opentelemetry::metrics::Meter`.
|
||||||
|
pub struct Metrics {
|
||||||
|
pub cpu_usage: opentelemetry::metrics::Gauge<f64>,
|
||||||
|
pub cpu_utilization: opentelemetry::metrics::Gauge<f64>,
|
||||||
|
pub memory_usage: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
pub memory_virtual: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
pub disk_io: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
pub network_io: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
pub network_io_per_interface: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
pub process_status: opentelemetry::metrics::Gauge<i64>,
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
pub gpu_memory_usage: opentelemetry::metrics::Gauge<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Metrics {
|
||||||
|
pub fn new(meter: &opentelemetry::metrics::Meter) -> Self {
|
||||||
|
let cpu_usage = meter
|
||||||
|
.f64_gauge(PROCESS_CPU_USAGE)
|
||||||
|
.with_description("The percentage of CPU in use.")
|
||||||
|
.with_unit("percent")
|
||||||
|
.build();
|
||||||
|
let cpu_utilization = meter
|
||||||
|
.f64_gauge(PROCESS_CPU_UTILIZATION)
|
||||||
|
.with_description("The amount of CPU in use.")
|
||||||
|
.with_unit("percent")
|
||||||
|
.build();
|
||||||
|
let memory_usage = meter
|
||||||
|
.i64_gauge(PROCESS_MEMORY_USAGE)
|
||||||
|
.with_description("The amount of physical memory in use.")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
let memory_virtual = meter
|
||||||
|
.i64_gauge(PROCESS_MEMORY_VIRTUAL)
|
||||||
|
.with_description("The amount of committed virtual memory.")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
let disk_io = meter
|
||||||
|
.i64_gauge(PROCESS_DISK_IO)
|
||||||
|
.with_description("Disk bytes transferred.")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
let network_io = meter
|
||||||
|
.i64_gauge(PROCESS_NETWORK_IO)
|
||||||
|
.with_description("Network bytes transferred.")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
let network_io_per_interface = meter
|
||||||
|
.i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE)
|
||||||
|
.with_description("Network bytes transferred (per interface).")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
let process_status = meter
|
||||||
|
.i64_gauge(PROCESS_STATUS)
|
||||||
|
.with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
let gpu_memory_usage = meter
|
||||||
|
.u64_gauge(PROCESS_GPU_MEMORY_USAGE)
|
||||||
|
.with_description("The amount of physical GPU memory in use.")
|
||||||
|
.with_unit("byte")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
Metrics {
|
||||||
|
cpu_usage,
|
||||||
|
cpu_utilization,
|
||||||
|
memory_usage,
|
||||||
|
memory_virtual,
|
||||||
|
disk_io,
|
||||||
|
network_io,
|
||||||
|
network_io_per_interface,
|
||||||
|
process_status,
|
||||||
|
#[cfg(feature = "gpu")]
|
||||||
|
gpu_memory_usage,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
use crate::GlobalError;
|
||||||
|
|
||||||
|
pub(crate) mod attributes;
|
||||||
|
mod collector;
|
||||||
|
pub(crate) mod gpu;
|
||||||
|
pub(crate) mod metrics;
|
||||||
|
|
||||||
|
/// Initialize the indicator collector for the current process
|
||||||
|
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||||
|
/// It will run indefinitely until the process is terminated.
|
||||||
|
pub async fn init_process_observer(meter: opentelemetry::metrics::Meter) -> Result<(), GlobalError> {
|
||||||
|
let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?;
|
||||||
|
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||||
|
collector.run().await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initialize the metric collector for the specified PID process
|
||||||
|
/// This function will create a new `Collector` instance and start collecting metrics.
|
||||||
|
/// It will run indefinitely until the process is terminated.
|
||||||
|
pub async fn init_process_observer_for_pid(meter: opentelemetry::metrics::Meter, pid: u32) -> Result<(), GlobalError> {
|
||||||
|
let pid = sysinfo::Pid::from_u32(pid);
|
||||||
|
let mut collector = collector::Collector::new(pid, meter, 30000)?;
|
||||||
|
collector.run().await
|
||||||
|
}
|
||||||
+159
-202
@@ -1,5 +1,6 @@
|
|||||||
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
|
use crate::global::{ENVIRONMENT, LOGGER_LEVEL, METER_INTERVAL, SAMPLE_RATIO, SERVICE_NAME, SERVICE_VERSION, USE_STDOUT};
|
||||||
use crate::{get_local_ip_with_default, OtelConfig};
|
use crate::utils::get_local_ip_with_default;
|
||||||
|
use crate::OtelConfig;
|
||||||
use opentelemetry::trace::TracerProvider;
|
use opentelemetry::trace::TracerProvider;
|
||||||
use opentelemetry::{global, KeyValue};
|
use opentelemetry::{global, KeyValue};
|
||||||
use opentelemetry_appender_tracing::layer;
|
use opentelemetry_appender_tracing::layer;
|
||||||
@@ -14,11 +15,10 @@ use opentelemetry_semantic_conventions::{
|
|||||||
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
|
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
|
||||||
SCHEMA_URL,
|
SCHEMA_URL,
|
||||||
};
|
};
|
||||||
use prometheus::{Encoder, Registry, TextEncoder};
|
use smallvec::SmallVec;
|
||||||
|
use std::borrow::Cow;
|
||||||
use std::io::IsTerminal;
|
use std::io::IsTerminal;
|
||||||
use std::sync::Arc;
|
use tracing::info;
|
||||||
use tokio::sync::{Mutex, OnceCell};
|
|
||||||
use tracing::{info, warn};
|
|
||||||
use tracing_error::ErrorLayer;
|
use tracing_error::ErrorLayer;
|
||||||
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
||||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
|
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
|
||||||
@@ -67,66 +67,20 @@ impl Drop for OtelGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Global registry for Prometheus metrics
|
|
||||||
static GLOBAL_REGISTRY: OnceCell<Arc<Mutex<Registry>>> = OnceCell::const_new();
|
|
||||||
|
|
||||||
/// Get the global registry instance
|
|
||||||
/// This function returns a reference to the global registry instance.
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// A reference to the global registry instance
|
|
||||||
///
|
|
||||||
/// # Example
|
|
||||||
/// ```
|
|
||||||
/// use rustfs_obs::get_global_registry;
|
|
||||||
///
|
|
||||||
/// let registry = get_global_registry();
|
|
||||||
/// ```
|
|
||||||
pub fn get_global_registry() -> Arc<Mutex<Registry>> {
|
|
||||||
GLOBAL_REGISTRY.get().unwrap().clone()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Prometheus metric endpoints
|
|
||||||
/// This function returns a string containing the Prometheus metrics.
|
|
||||||
/// The metrics are collected from the global registry.
|
|
||||||
/// The function is used to expose the metrics via an HTTP endpoint.
|
|
||||||
///
|
|
||||||
/// # Returns
|
|
||||||
/// A string containing the Prometheus metrics
|
|
||||||
///
|
|
||||||
/// # Example
|
|
||||||
/// ```
|
|
||||||
/// use rustfs_obs::metrics;
|
|
||||||
///
|
|
||||||
/// async fn main() {
|
|
||||||
/// let metrics = metrics().await;
|
|
||||||
/// println!("{}", metrics);
|
|
||||||
/// }
|
|
||||||
/// ```
|
|
||||||
pub async fn metrics() -> String {
|
|
||||||
let encoder = TextEncoder::new();
|
|
||||||
// Get a reference to the registry for reading metrics
|
|
||||||
let registry = get_global_registry().lock().await.to_owned();
|
|
||||||
let metric_families = registry.gather();
|
|
||||||
if metric_families.is_empty() {
|
|
||||||
warn!("No metrics available in Prometheus registry");
|
|
||||||
} else {
|
|
||||||
info!("Metrics collected: {} families", metric_families.len());
|
|
||||||
}
|
|
||||||
let mut buffer = Vec::new();
|
|
||||||
encoder.encode(&metric_families, &mut buffer).unwrap();
|
|
||||||
String::from_utf8(buffer).unwrap_or_else(|_| "Error encoding metrics".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// create OpenTelemetry Resource
|
/// create OpenTelemetry Resource
|
||||||
fn resource(config: &OtelConfig) -> Resource {
|
fn resource(config: &OtelConfig) -> Resource {
|
||||||
let config = config.clone();
|
|
||||||
Resource::builder()
|
Resource::builder()
|
||||||
.with_service_name(config.service_name.unwrap_or(SERVICE_NAME.to_string()))
|
.with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(SERVICE_NAME)).to_string())
|
||||||
.with_schema_url(
|
.with_schema_url(
|
||||||
[
|
[
|
||||||
KeyValue::new(OTEL_SERVICE_VERSION, config.service_version.unwrap_or(SERVICE_VERSION.to_string())),
|
KeyValue::new(
|
||||||
KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, config.environment.unwrap_or(ENVIRONMENT.to_string())),
|
OTEL_SERVICE_VERSION,
|
||||||
|
Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
|
||||||
|
),
|
||||||
|
KeyValue::new(
|
||||||
|
DEPLOYMENT_ENVIRONMENT_NAME,
|
||||||
|
Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
|
||||||
|
),
|
||||||
KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
|
KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
|
||||||
],
|
],
|
||||||
SCHEMA_URL,
|
SCHEMA_URL,
|
||||||
@@ -141,167 +95,169 @@ fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout:
|
|||||||
.build()
|
.build()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Initialize Meter Provider
|
|
||||||
fn init_meter_provider(config: &OtelConfig) -> SdkMeterProvider {
|
|
||||||
let mut builder = MeterProviderBuilder::default().with_resource(resource(config));
|
|
||||||
// If endpoint is empty, use stdout output
|
|
||||||
if config.endpoint.is_empty() {
|
|
||||||
builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
|
|
||||||
} else {
|
|
||||||
// If endpoint is not empty, use otlp output
|
|
||||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
|
||||||
.with_tonic()
|
|
||||||
.with_endpoint(&config.endpoint)
|
|
||||||
.with_temporality(opentelemetry_sdk::metrics::Temporality::default())
|
|
||||||
.build()
|
|
||||||
.unwrap();
|
|
||||||
builder = builder.with_reader(
|
|
||||||
PeriodicReader::builder(exporter)
|
|
||||||
.with_interval(std::time::Duration::from_secs(config.meter_interval.unwrap_or(METER_INTERVAL)))
|
|
||||||
.build(),
|
|
||||||
);
|
|
||||||
// If use_stdout is true, output to stdout at the same time
|
|
||||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
|
||||||
builder = builder.with_reader(create_periodic_reader(config.meter_interval.unwrap_or(METER_INTERVAL)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let registry = Registry::new();
|
|
||||||
// Set global registry
|
|
||||||
GLOBAL_REGISTRY.set(Arc::new(Mutex::new(registry.clone()))).unwrap();
|
|
||||||
// Create Prometheus exporter
|
|
||||||
let prometheus_exporter = opentelemetry_prometheus::exporter().with_registry(registry).build().unwrap();
|
|
||||||
// Build meter provider
|
|
||||||
let meter_provider = builder.with_reader(prometheus_exporter).build();
|
|
||||||
global::set_meter_provider(meter_provider.clone());
|
|
||||||
meter_provider
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Initialize Tracer Provider
|
|
||||||
fn init_tracer_provider(config: &OtelConfig) -> SdkTracerProvider {
|
|
||||||
let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
|
|
||||||
let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
|
|
||||||
Sampler::TraceIdRatioBased(sample_ratio)
|
|
||||||
} else {
|
|
||||||
Sampler::AlwaysOn
|
|
||||||
};
|
|
||||||
let builder = SdkTracerProvider::builder()
|
|
||||||
.with_sampler(sampler)
|
|
||||||
.with_id_generator(RandomIdGenerator::default())
|
|
||||||
.with_resource(resource(config));
|
|
||||||
|
|
||||||
let tracer_provider = if config.endpoint.is_empty() {
|
|
||||||
builder
|
|
||||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
|
||||||
.build()
|
|
||||||
} else {
|
|
||||||
let exporter = opentelemetry_otlp::SpanExporter::builder()
|
|
||||||
.with_tonic()
|
|
||||||
.with_endpoint(&config.endpoint)
|
|
||||||
.build()
|
|
||||||
.unwrap();
|
|
||||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
|
||||||
builder
|
|
||||||
.with_batch_exporter(exporter)
|
|
||||||
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
|
||||||
} else {
|
|
||||||
builder.with_batch_exporter(exporter)
|
|
||||||
}
|
|
||||||
.build()
|
|
||||||
};
|
|
||||||
|
|
||||||
global::set_tracer_provider(tracer_provider.clone());
|
|
||||||
tracer_provider
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Initialize Telemetry
|
/// Initialize Telemetry
|
||||||
pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
||||||
let tracer_provider = init_tracer_provider(config);
|
// avoid repeated access to configuration fields
|
||||||
let meter_provider = init_meter_provider(config);
|
let endpoint = &config.endpoint;
|
||||||
|
let use_stdout = config.use_stdout.unwrap_or(USE_STDOUT);
|
||||||
|
let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
|
||||||
|
let logger_level = config.logger_level.as_deref().unwrap_or(LOGGER_LEVEL);
|
||||||
|
let service_name = config.service_name.as_deref().unwrap_or(SERVICE_NAME);
|
||||||
|
|
||||||
// Initialize logger provider based on configuration
|
// Pre-create resource objects to avoid repeated construction
|
||||||
let logger_provider = {
|
let res = resource(config);
|
||||||
let mut builder = SdkLoggerProvider::builder().with_resource(resource(config));
|
|
||||||
|
|
||||||
if config.endpoint.is_empty() {
|
// initialize tracer provider
|
||||||
// Use stdout exporter when no endpoint is configured
|
let tracer_provider = {
|
||||||
builder = builder.with_simple_exporter(opentelemetry_stdout::LogExporter::default());
|
let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
|
||||||
|
let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 {
|
||||||
|
Sampler::TraceIdRatioBased(sample_ratio)
|
||||||
|
} else {
|
||||||
|
Sampler::AlwaysOn
|
||||||
|
};
|
||||||
|
|
||||||
|
let builder = SdkTracerProvider::builder()
|
||||||
|
.with_sampler(sampler)
|
||||||
|
.with_id_generator(RandomIdGenerator::default())
|
||||||
|
.with_resource(res.clone());
|
||||||
|
|
||||||
|
let tracer_provider = if endpoint.is_empty() {
|
||||||
|
builder
|
||||||
|
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||||
|
.build()
|
||||||
|
} else {
|
||||||
|
let exporter = opentelemetry_otlp::SpanExporter::builder()
|
||||||
|
.with_tonic()
|
||||||
|
.with_endpoint(endpoint)
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let builder = if use_stdout {
|
||||||
|
builder
|
||||||
|
.with_batch_exporter(exporter)
|
||||||
|
.with_batch_exporter(opentelemetry_stdout::SpanExporter::default())
|
||||||
|
} else {
|
||||||
|
builder.with_batch_exporter(exporter)
|
||||||
|
};
|
||||||
|
|
||||||
|
builder.build()
|
||||||
|
};
|
||||||
|
|
||||||
|
global::set_tracer_provider(tracer_provider.clone());
|
||||||
|
tracer_provider
|
||||||
|
};
|
||||||
|
|
||||||
|
// initialize meter provider
|
||||||
|
let meter_provider = {
|
||||||
|
let mut builder = MeterProviderBuilder::default().with_resource(res.clone());
|
||||||
|
|
||||||
|
if endpoint.is_empty() {
|
||||||
|
builder = builder.with_reader(create_periodic_reader(meter_interval));
|
||||||
|
} else {
|
||||||
|
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||||
|
.with_tonic()
|
||||||
|
.with_endpoint(endpoint)
|
||||||
|
.with_temporality(opentelemetry_sdk::metrics::Temporality::default())
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
builder = builder.with_reader(
|
||||||
|
PeriodicReader::builder(exporter)
|
||||||
|
.with_interval(std::time::Duration::from_secs(meter_interval))
|
||||||
|
.build(),
|
||||||
|
);
|
||||||
|
|
||||||
|
if use_stdout {
|
||||||
|
builder = builder.with_reader(create_periodic_reader(meter_interval));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let meter_provider = builder.build();
|
||||||
|
global::set_meter_provider(meter_provider.clone());
|
||||||
|
meter_provider
|
||||||
|
};
|
||||||
|
|
||||||
|
// initialize logger provider
|
||||||
|
let logger_provider = {
|
||||||
|
let mut builder = SdkLoggerProvider::builder().with_resource(res);
|
||||||
|
|
||||||
|
if endpoint.is_empty() {
|
||||||
|
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
||||||
} else {
|
} else {
|
||||||
// Configure OTLP exporter when endpoint is provided
|
|
||||||
let exporter = opentelemetry_otlp::LogExporter::builder()
|
let exporter = opentelemetry_otlp::LogExporter::builder()
|
||||||
.with_tonic()
|
.with_tonic()
|
||||||
.with_endpoint(&config.endpoint)
|
.with_endpoint(endpoint)
|
||||||
.build()
|
.build()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
builder = builder.with_batch_exporter(exporter);
|
builder = builder.with_batch_exporter(exporter);
|
||||||
|
|
||||||
// Add stdout exporter if requested
|
if use_stdout {
|
||||||
if config.use_stdout.unwrap_or(USE_STDOUT) {
|
|
||||||
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.build()
|
builder.build()
|
||||||
};
|
};
|
||||||
let config = config.clone();
|
|
||||||
let logger_level = config.logger_level.unwrap_or(LOGGER_LEVEL.to_string());
|
// configuring tracing
|
||||||
let logger_level = logger_level.as_str();
|
{
|
||||||
// Setup OpenTelemetryTracingBridge layer
|
// optimize filter configuration
|
||||||
let otel_layer = {
|
let otel_layer = {
|
||||||
// Filter to prevent infinite telemetry loops
|
let filter_otel = match logger_level {
|
||||||
// This blocks events from OpenTelemetry and its dependent libraries (tonic, reqwest, etc.)
|
"trace" | "debug" => {
|
||||||
// from being sent back to OpenTelemetry itself
|
info!("OpenTelemetry tracing initialized with level: {}", logger_level);
|
||||||
let filter_otel = match logger_level {
|
EnvFilter::new(logger_level)
|
||||||
"trace" | "debug" => {
|
|
||||||
info!("OpenTelemetry tracing initialized with level: {}", logger_level);
|
|
||||||
let mut filter = EnvFilter::new(logger_level);
|
|
||||||
for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
|
|
||||||
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
|
||||||
}
|
}
|
||||||
filter
|
_ => {
|
||||||
}
|
let mut filter = EnvFilter::new(logger_level);
|
||||||
_ => {
|
|
||||||
let mut filter = EnvFilter::new(logger_level);
|
// use smallvec to avoid heap allocation
|
||||||
for directive in ["hyper", "tonic", "h2", "reqwest", "tower"] {
|
let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
|
||||||
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
|
||||||
|
for directive in directives {
|
||||||
|
filter = filter.add_directive(format!("{}=off", directive).parse().unwrap());
|
||||||
|
}
|
||||||
|
filter
|
||||||
}
|
}
|
||||||
filter
|
};
|
||||||
}
|
layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
|
||||||
};
|
};
|
||||||
layer::OpenTelemetryTracingBridge::new(&logger_provider).with_filter(filter_otel)
|
|
||||||
};
|
|
||||||
|
|
||||||
let tracer = tracer_provider.tracer(config.service_name.unwrap_or(SERVICE_NAME.to_string()));
|
let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string());
|
||||||
let registry = tracing_subscriber::registry()
|
|
||||||
.with(switch_level(logger_level))
|
|
||||||
.with(OpenTelemetryLayer::new(tracer))
|
|
||||||
.with(MetricsLayer::new(meter_provider.clone()))
|
|
||||||
.with(otel_layer)
|
|
||||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
|
|
||||||
|
|
||||||
// Configure formatting layer
|
// Configure registry to avoid repeated calls to filter methods
|
||||||
let enable_color = std::io::stdout().is_terminal();
|
let level_filter = switch_level(logger_level);
|
||||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
let registry = tracing_subscriber::registry()
|
||||||
.with_ansi(enable_color)
|
.with(level_filter)
|
||||||
.with_thread_names(true)
|
.with(OpenTelemetryLayer::new(tracer))
|
||||||
.with_file(true)
|
.with(MetricsLayer::new(meter_provider.clone()))
|
||||||
.with_line_number(true);
|
.with(otel_layer)
|
||||||
|
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(logger_level)));
|
||||||
|
|
||||||
// Creating a formatting layer with explicit type to avoid type mismatches
|
// configure the formatting layer
|
||||||
let fmt_layer = fmt_layer.with_filter(
|
let enable_color = std::io::stdout().is_terminal();
|
||||||
EnvFilter::new(logger_level).add_directive(
|
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||||
format!("opentelemetry={}", if config.endpoint.is_empty() { logger_level } else { "off" })
|
.with_ansi(enable_color)
|
||||||
.parse()
|
.with_thread_names(true)
|
||||||
.unwrap(),
|
.with_file(true)
|
||||||
),
|
.with_line_number(true)
|
||||||
);
|
.with_filter(
|
||||||
|
EnvFilter::new(logger_level).add_directive(
|
||||||
|
format!("opentelemetry={}", if endpoint.is_empty() { logger_level } else { "off" })
|
||||||
|
.parse()
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
registry.with(ErrorLayer::default()).with(fmt_layer).init();
|
registry.with(ErrorLayer::default()).with(fmt_layer).init();
|
||||||
if !config.endpoint.is_empty() {
|
|
||||||
info!(
|
if !endpoint.is_empty() {
|
||||||
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
|
info!(
|
||||||
config.endpoint, logger_level
|
"OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {}",
|
||||||
);
|
endpoint, logger_level
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
OtelGuard {
|
OtelGuard {
|
||||||
@@ -313,12 +269,13 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
|||||||
|
|
||||||
/// Switch log level
|
/// Switch log level
|
||||||
fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
|
fn switch_level(logger_level: &str) -> tracing_subscriber::filter::LevelFilter {
|
||||||
|
use tracing_subscriber::filter::LevelFilter;
|
||||||
match logger_level {
|
match logger_level {
|
||||||
"error" => tracing_subscriber::filter::LevelFilter::ERROR,
|
"error" => LevelFilter::ERROR,
|
||||||
"warn" => tracing_subscriber::filter::LevelFilter::WARN,
|
"warn" => LevelFilter::WARN,
|
||||||
"info" => tracing_subscriber::filter::LevelFilter::INFO,
|
"info" => LevelFilter::INFO,
|
||||||
"debug" => tracing_subscriber::filter::LevelFilter::DEBUG,
|
"debug" => LevelFilter::DEBUG,
|
||||||
"trace" => tracing_subscriber::filter::LevelFilter::TRACE,
|
"trace" => LevelFilter::TRACE,
|
||||||
_ => tracing_subscriber::filter::LevelFilter::OFF,
|
_ => LevelFilter::OFF,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-1
@@ -46,6 +46,7 @@ local-ip-address = { workspace = true }
|
|||||||
matchit = { workspace = true }
|
matchit = { workspace = true }
|
||||||
mime.workspace = true
|
mime.workspace = true
|
||||||
mime_guess = { workspace = true }
|
mime_guess = { workspace = true }
|
||||||
|
opentelemetry = { workspace = true }
|
||||||
pin-project-lite.workspace = true
|
pin-project-lite.workspace = true
|
||||||
protos.workspace = true
|
protos.workspace = true
|
||||||
query = { workspace = true }
|
query = { workspace = true }
|
||||||
@@ -76,7 +77,7 @@ tokio-stream.workspace = true
|
|||||||
tonic = { workspace = true }
|
tonic = { workspace = true }
|
||||||
tower.workspace = true
|
tower.workspace = true
|
||||||
transform-stream.workspace = true
|
transform-stream.workspace = true
|
||||||
tower-http = { workspace = true, features = ["trace", "compression-full", "cors"] }
|
tower-http = { workspace = true, features = ["trace", "compression-deflate", "compression-gzip", "cors"] }
|
||||||
uuid = { workspace = true }
|
uuid = { workspace = true }
|
||||||
|
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
[target.'cfg(target_os = "linux")'.dependencies]
|
||||||
|
|||||||
@@ -271,7 +271,7 @@ pub async fn start_static_file_server(
|
|||||||
.route("/config.json", get(config_handler))
|
.route("/config.json", get(config_handler))
|
||||||
.fallback_service(get(static_handler))
|
.fallback_service(get(static_handler))
|
||||||
.layer(cors)
|
.layer(cors)
|
||||||
.layer(tower_http::compression::CompressionLayer::new())
|
.layer(tower_http::compression::CompressionLayer::new().gzip(true).deflate(true))
|
||||||
.layer(TraceLayer::new_for_http());
|
.layer(TraceLayer::new_for_http());
|
||||||
let local_addr: SocketAddr = addrs.parse().expect("Failed to parse socket address");
|
let local_addr: SocketAddr = addrs.parse().expect("Failed to parse socket address");
|
||||||
info!("WebUI: http://{}:{} http://127.0.0.1:{}", local_ip, local_addr.port(), local_addr.port());
|
info!("WebUI: http://{}:{} http://127.0.0.1:{}", local_ip, local_addr.port(), local_addr.port());
|
||||||
|
|||||||
+89
-17
@@ -14,6 +14,7 @@ use crate::console::{init_console_cfg, CONSOLE_CONFIG};
|
|||||||
// Ensure the correct path for parse_license is imported
|
// Ensure the correct path for parse_license is imported
|
||||||
use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT};
|
use crate::server::{wait_for_shutdown, ServiceState, ServiceStateManager, ShutdownSignal, SHUTDOWN_TIMEOUT};
|
||||||
use crate::utils::error;
|
use crate::utils::error;
|
||||||
|
use bytes::Bytes;
|
||||||
use chrono::Datelike;
|
use chrono::Datelike;
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use common::{
|
use common::{
|
||||||
@@ -37,6 +38,7 @@ use ecstore::{
|
|||||||
};
|
};
|
||||||
use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys};
|
use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys};
|
||||||
use grpc::make_server;
|
use grpc::make_server;
|
||||||
|
use http::{HeaderMap, Request as HttpRequest, Response};
|
||||||
use hyper_util::server::graceful::GracefulShutdown;
|
use hyper_util::server::graceful::GracefulShutdown;
|
||||||
use hyper_util::{
|
use hyper_util::{
|
||||||
rt::{TokioExecutor, TokioIo},
|
rt::{TokioExecutor, TokioIo},
|
||||||
@@ -47,17 +49,20 @@ use iam::init_iam_sys;
|
|||||||
use license::init_license;
|
use license::init_license;
|
||||||
use protos::proto_gen::node_service::node_service_server::NodeServiceServer;
|
use protos::proto_gen::node_service::node_service_server::NodeServiceServer;
|
||||||
use rustfs_event_notifier::NotifierConfig;
|
use rustfs_event_notifier::NotifierConfig;
|
||||||
use rustfs_obs::{init_obs, load_config, set_global_guard, InitLogStatus};
|
use rustfs_obs::{init_obs, init_process_observer, load_config, set_global_guard};
|
||||||
use rustls::ServerConfig;
|
use rustls::ServerConfig;
|
||||||
use s3s::{host::MultiDomain, service::S3ServiceBuilder};
|
use s3s::{host::MultiDomain, service::S3ServiceBuilder};
|
||||||
use service::hybrid;
|
use service::hybrid;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::signal::unix::{signal, SignalKind};
|
use tokio::signal::unix::{signal, SignalKind};
|
||||||
use tokio_rustls::TlsAcceptor;
|
use tokio_rustls::TlsAcceptor;
|
||||||
use tonic::{metadata::MetadataValue, Request, Status};
|
use tonic::{metadata::MetadataValue, Request, Status};
|
||||||
use tower_http::cors::CorsLayer;
|
use tower_http::cors::CorsLayer;
|
||||||
|
use tower_http::trace::TraceLayer;
|
||||||
|
use tracing::Span;
|
||||||
use tracing::{debug, error, info, info_span, warn};
|
use tracing::{debug, error, info, info_span, warn};
|
||||||
|
|
||||||
#[cfg(all(target_os = "linux", target_env = "gnu"))]
|
#[cfg(all(target_os = "linux", target_env = "gnu"))]
|
||||||
@@ -103,9 +108,6 @@ async fn main() -> Result<()> {
|
|||||||
// Store in global storage
|
// Store in global storage
|
||||||
set_global_guard(guard)?;
|
set_global_guard(guard)?;
|
||||||
|
|
||||||
// Log initialization status
|
|
||||||
InitLogStatus::init_start_log(&config.observability).await?;
|
|
||||||
|
|
||||||
// Initialize event notifier
|
// Initialize event notifier
|
||||||
let notifier_config = opt.clone().event_config;
|
let notifier_config = opt.clone().event_config;
|
||||||
if notifier_config.is_some() {
|
if notifier_config.is_some() {
|
||||||
@@ -247,6 +249,20 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
b.build()
|
b.build()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
tokio::spawn(async move {
|
||||||
|
// Record the PID-related metrics of the current process
|
||||||
|
let meter = opentelemetry::global::meter("system");
|
||||||
|
let obs_result = init_process_observer(meter).await;
|
||||||
|
match obs_result {
|
||||||
|
Ok(_) => {
|
||||||
|
info!("Process observer initialized successfully");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("Failed to initialize process observer: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let rpc_service = NodeServiceServer::with_interceptor(make_server(), check_auth);
|
let rpc_service = NodeServiceServer::with_interceptor(make_server(), check_auth);
|
||||||
|
|
||||||
let tls_path = opt.tls_path.clone().unwrap_or_default();
|
let tls_path = opt.tls_path.clone().unwrap_or_default();
|
||||||
@@ -300,18 +316,53 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
let mut sigint_inner = sigint_inner;
|
let mut sigint_inner = sigint_inner;
|
||||||
let hybrid_service = TowerToHyperService::new(
|
let hybrid_service = TowerToHyperService::new(
|
||||||
tower::ServiceBuilder::new()
|
tower::ServiceBuilder::new()
|
||||||
|
.layer(
|
||||||
|
TraceLayer::new_for_http()
|
||||||
|
.make_span_with(|request: &HttpRequest<_>| {
|
||||||
|
let span = tracing::debug_span!("http-request",
|
||||||
|
status_code = tracing::field::Empty,
|
||||||
|
method = %request.method(),
|
||||||
|
uri = %request.uri(),
|
||||||
|
version = ?request.version(),
|
||||||
|
);
|
||||||
|
for (header_name, header_value) in request.headers() {
|
||||||
|
if header_name == "user-agent" || header_name == "content-type" || header_name == "content-length"
|
||||||
|
{
|
||||||
|
span.record(header_name.as_str(), header_value.to_str().unwrap_or("invalid"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
span
|
||||||
|
})
|
||||||
|
.on_request(|request: &HttpRequest<_>, _span: &Span| {
|
||||||
|
debug!("started method: {}, url path: {}", request.method(), request.uri().path())
|
||||||
|
})
|
||||||
|
.on_response(|response: &Response<_>, latency: Duration, _span: &Span| {
|
||||||
|
_span.record("status_code", tracing::field::display(response.status()));
|
||||||
|
debug!("response generated in {:?}", latency)
|
||||||
|
})
|
||||||
|
.on_body_chunk(|chunk: &Bytes, latency: Duration, _span: &Span| {
|
||||||
|
debug!("sending {} bytes in {:?}", chunk.len(), latency)
|
||||||
|
})
|
||||||
|
.on_eos(|_trailers: Option<&HeaderMap>, stream_duration: Duration, _span: &Span| {
|
||||||
|
debug!("stream closed after {:?}", stream_duration)
|
||||||
|
})
|
||||||
|
.on_failure(|_error, latency: Duration, _span: &Span| {
|
||||||
|
debug!("request error: {:?} in {:?}", _error, latency)
|
||||||
|
}),
|
||||||
|
)
|
||||||
.layer(CorsLayer::permissive())
|
.layer(CorsLayer::permissive())
|
||||||
.service(hybrid(s3_service, rpc_service)),
|
.service(hybrid(s3_service, rpc_service)),
|
||||||
);
|
);
|
||||||
|
|
||||||
let http_server = ConnBuilder::new(TokioExecutor::new());
|
let http_server = Arc::new(ConnBuilder::new(TokioExecutor::new()));
|
||||||
let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
|
let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
|
||||||
let graceful = GracefulShutdown::new();
|
let graceful = Arc::new(GracefulShutdown::new());
|
||||||
debug!("graceful initiated");
|
debug!("graceful initiated");
|
||||||
|
|
||||||
// 服务准备就绪
|
// 服务准备就绪
|
||||||
worker_state_manager.update(ServiceState::Ready);
|
worker_state_manager.update(ServiceState::Ready);
|
||||||
|
let value = hybrid_service.clone();
|
||||||
loop {
|
loop {
|
||||||
debug!("waiting for SIGINT or SIGTERM has_tls_certs: {}", has_tls_certs);
|
debug!("waiting for SIGINT or SIGTERM has_tls_certs: {}", has_tls_certs);
|
||||||
// Wait for a connection
|
// Wait for a connection
|
||||||
@@ -366,12 +417,17 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let conn = http_server.serve_connection(TokioIo::new(tls_socket), hybrid_service.clone());
|
|
||||||
let conn = graceful.watch(conn.into_owned());
|
let http_server_clone = http_server.clone();
|
||||||
|
let value_clone = value.clone();
|
||||||
|
let graceful_clone = graceful.clone();
|
||||||
|
|
||||||
tokio::task::spawn_blocking(move || {
|
tokio::task::spawn_blocking(move || {
|
||||||
tokio::runtime::Runtime::new()
|
tokio::runtime::Runtime::new()
|
||||||
.expect("Failed to create runtime")
|
.expect("Failed to create runtime")
|
||||||
.block_on(async move {
|
.block_on(async move {
|
||||||
|
let conn = http_server_clone.serve_connection(TokioIo::new(tls_socket), value_clone);
|
||||||
|
let conn = graceful_clone.watch(conn);
|
||||||
if let Err(err) = conn.await {
|
if let Err(err) = conn.await {
|
||||||
error!("Https Connection error: {}", err);
|
error!("Https Connection error: {}", err);
|
||||||
}
|
}
|
||||||
@@ -380,9 +436,13 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
debug!("TLS handshake success");
|
debug!("TLS handshake success");
|
||||||
} else {
|
} else {
|
||||||
debug!("Http handshake start");
|
debug!("Http handshake start");
|
||||||
let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone());
|
|
||||||
let conn = graceful.watch(conn.into_owned());
|
let http_server_clone = http_server.clone();
|
||||||
|
let value_clone = value.clone();
|
||||||
|
let graceful_clone = graceful.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let conn = http_server_clone.serve_connection(TokioIo::new(socket), value_clone);
|
||||||
|
let conn = graceful_clone.watch(conn);
|
||||||
if let Err(err) = conn.await {
|
if let Err(err) = conn.await {
|
||||||
error!("Http Connection error: {}", err);
|
error!("Http Connection error: {}", err);
|
||||||
}
|
}
|
||||||
@@ -391,12 +451,24 @@ async fn run(opt: config::Opt) -> Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
worker_state_manager.update(ServiceState::Stopping);
|
worker_state_manager.update(ServiceState::Stopping);
|
||||||
tokio::select! {
|
match Arc::try_unwrap(graceful) {
|
||||||
() = graceful.shutdown() => {
|
Ok(g) => {
|
||||||
debug!("Gracefully shutdown!");
|
// Successfully obtaining unique ownership, you can call shutdown
|
||||||
},
|
tokio::select! {
|
||||||
() = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
|
() = g.shutdown() => {
|
||||||
debug!("Waited 10 seconds for graceful shutdown, aborting...");
|
debug!("Gracefully shutdown!");
|
||||||
|
},
|
||||||
|
() = tokio::time::sleep(Duration::from_secs(10)) => {
|
||||||
|
debug!("Waited 10 seconds for graceful shutdown, aborting...");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(arc_graceful) => {
|
||||||
|
// There are other references that cannot be obtained for unique ownership
|
||||||
|
error!("Cannot perform graceful shutdown, other references exist err: {:?}", arc_graceful);
|
||||||
|
// In this case, we can only wait for the timeout
|
||||||
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||||
|
debug!("Timeout reached, forcing shutdown");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
worker_state_manager.update(ServiceState::Stopped);
|
worker_state_manager.update(ServiceState::Stopped);
|
||||||
|
|||||||
@@ -166,7 +166,8 @@ impl SimpleQueryDispatcher {
|
|||||||
if let Some(delimiter) = csv.field_delimiter.as_ref() {
|
if let Some(delimiter) = csv.field_delimiter.as_ref() {
|
||||||
file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default());
|
file_format = file_format.with_delimiter(delimiter.as_bytes().first().copied().unwrap_or_default());
|
||||||
}
|
}
|
||||||
if csv.file_header_info.is_some() {}
|
// TODO waiting for processing @junxiang Mu
|
||||||
|
// if csv.file_header_info.is_some() {}
|
||||||
match csv.file_header_info.as_ref() {
|
match csv.file_header_info.as_ref() {
|
||||||
Some(info) => {
|
Some(info) => {
|
||||||
if *info == *NONE {
|
if *info == *NONE {
|
||||||
|
|||||||
+2
-2
@@ -40,13 +40,13 @@ export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml"
|
|||||||
|
|
||||||
# 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值
|
# 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值
|
||||||
export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
|
export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317
|
||||||
export RUSTFS__OBSERVABILITY__USE_STDOUT=true
|
export RUSTFS__OBSERVABILITY__USE_STDOUT=false
|
||||||
export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
|
export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0
|
||||||
export RUSTFS__OBSERVABILITY__METER_INTERVAL=30
|
export RUSTFS__OBSERVABILITY__METER_INTERVAL=30
|
||||||
export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
|
export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs
|
||||||
export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
|
export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0
|
||||||
export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
|
export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop
|
||||||
export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=info
|
export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug
|
||||||
export RUSTFS__SINKS__FILE__ENABLED=true
|
export RUSTFS__SINKS__FILE__ENABLED=true
|
||||||
export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log"
|
export RUSTFS__SINKS__FILE__PATH="./deploy/logs/rustfs.log"
|
||||||
export RUSTFS__SINKS__WEBHOOK__ENABLED=false
|
export RUSTFS__SINKS__WEBHOOK__ENABLED=false
|
||||||
|
|||||||
Reference in New Issue
Block a user