From 3c28f0a0baa9e84788d6995349544826a132b6c9 Mon Sep 17 00:00:00 2001 From: heihutu Date: Fri, 20 Mar 2026 18:52:33 +0800 Subject: [PATCH] feat(metrics): migrate system monitoring from rustfs-obs to rustfs-metrics (#2242) Co-authored-by: houseme --- .../grafana/dashboards/rustfs.json | 30 +-- Cargo.lock | 20 +- Cargo.toml | 2 - crates/appauth/Cargo.toml | 3 + crates/audit/Cargo.toml | 4 + crates/checksums/Cargo.toml | 3 + crates/common/Cargo.toml | 3 + crates/config/Cargo.toml | 3 + crates/crypto/Cargo.toml | 3 + crates/ecstore/Cargo.toml | 4 +- crates/filemeta/Cargo.toml | 3 + crates/heal/Cargo.toml | 3 + crates/iam/Cargo.toml | 3 + crates/lock/Cargo.toml | 3 + crates/madmin/Cargo.toml | 3 + crates/mcp/Cargo.toml | 3 +- crates/metrics/Cargo.toml | 7 + crates/metrics/src/collectors/global.rs | 217 +++++++++++++++--- crates/metrics/src/collectors/mod.rs | 20 +- crates/metrics/src/collectors/system_cpu.rs | 90 +++++++- crates/metrics/src/collectors/system_drive.rs | 43 ++++ crates/metrics/src/collectors/system_gpu.rs | 199 ++++++++++++++++ .../metrics/src/collectors/system_memory.rs | 73 +++++- .../metrics/src/collectors/system_network.rs | 73 +++++- .../metrics/src/collectors/system_process.rs | 196 +++++++++++++++- .../src/metrics_type/entry/metric_name.rs | 25 ++ .../src/metrics_type/entry/subsystem.rs | 4 + crates/metrics/src/metrics_type/mod.rs | 1 + crates/metrics/src/metrics_type/system_gpu.rs | 35 +++ .../src/metrics_type/system_process.rs | 64 ++++++ crates/notify/Cargo.toml | 3 + crates/obs/Cargo.toml | 10 +- crates/obs/examples/server.rs | 9 +- crates/obs/src/global.rs | 17 +- crates/obs/src/lib.rs | 33 +-- crates/obs/src/system/attributes.rs | 58 ----- crates/obs/src/system/collector.rs | 173 -------------- crates/obs/src/system/gpu.rs | 57 ----- crates/obs/src/system/metrics.rs | 116 ---------- crates/obs/src/system/mod.rs | 51 ---- crates/policy/Cargo.toml | 3 + crates/protocols/Cargo.toml | 2 - crates/protos/Cargo.toml | 4 + crates/s3-common/Cargo.toml | 3 + crates/s3select-api/Cargo.toml | 5 +- crates/s3select-query/Cargo.toml | 3 + crates/scanner/Cargo.toml | 3 + crates/signer/Cargo.toml | 5 +- crates/trusted-proxies/Cargo.toml | 3 + crates/utils/Cargo.toml | 3 +- crates/workers/Cargo.toml | 3 + crates/zip/Cargo.toml | 3 + rustfs/Cargo.toml | 3 +- rustfs/src/app/bucket_usecase.rs | 3 +- rustfs/src/storage/concurrency.rs | 2 +- scripts/run.sh | 14 +- 56 files changed, 1116 insertions(+), 613 deletions(-) create mode 100644 crates/metrics/src/collectors/system_gpu.rs create mode 100644 crates/metrics/src/metrics_type/system_gpu.rs delete mode 100644 crates/obs/src/system/attributes.rs delete mode 100644 crates/obs/src/system/collector.rs delete mode 100644 crates/obs/src/system/gpu.rs delete mode 100644 crates/obs/src/system/metrics.rs delete mode 100644 crates/obs/src/system/mod.rs diff --git a/.docker/observability/grafana/dashboards/rustfs.json b/.docker/observability/grafana/dashboards/rustfs.json index 31e8fcc4d..b281a81e3 100644 --- a/.docker/observability/grafana/dashboards/rustfs.json +++ b/.docker/observability/grafana/dashboards/rustfs.json @@ -91,7 +91,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rustfs_process_uptime_seconds{job=~\"$job\"}", + "expr": "gauge_rustfs_process_uptime_seconds{job=~\"$job\"}", "legendFormat": "__auto", "range": true, "refId": "A" @@ -223,7 +223,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})", + "expr": "sum(gauge_rustfs_cluster_buckets_total{job=~\"$job\"})", "legendFormat": "__auto", "range": true, "refId": "A" @@ -289,7 +289,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})", + "expr": "sum(gauge_rustfs_cluster_objects_total{job=~\"$job\"})", "legendFormat": "__auto", "range": true, "refId": "A" @@ -427,7 +427,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})", + "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"})", "legendFormat": "Used", "range": true, "refId": "A" @@ -438,7 +438,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "expr": "sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", "hide": false, "legendFormat": "Total", "range": true, @@ -450,7 +450,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", + "expr": "sum(gauge_rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(gauge_rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})", "hide": false, "instant": false, "legendFormat": "Percent", @@ -1971,7 +1971,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})", + "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"})", "legendFormat": "{{drive}} (bytes)", "range": true, "refId": "A" @@ -1982,7 +1982,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (drive) (rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})", + "expr": "sum by (drive) (gauge_rustfs_node_disk_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(gauge_rustfs_node_disk_total_bytes{job=~\"$job\", drive=~\"$drive\"})", "hide": false, "instant": false, "legendFormat": "{{drive}} (percent)", @@ -2473,7 +2473,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (process_cpu_usage_percent{job=~\"$job\"})", + "expr": "sum by (job) (gauge_rustfs_process_cpu_usage{job=~\"$job\"})", "legendFormat": "{{job}}", "range": true, "refId": "A" @@ -2572,7 +2572,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (rustfs_process_memory_bytes{job=~\"$job\"})", + "expr": "sum by (job) (gauge_rustfs_process_resident_memory_bytes{job=~\"$job\"})", "legendFormat": "{{job}}", "range": true, "refId": "B" @@ -2670,7 +2670,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"received\"}[5m]))", + "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"received\"}[5m]))", "legendFormat": "RX - {{job}}", "range": true, "refId": "C" @@ -2681,7 +2681,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job) (rate(process_network_io_byte{job=~\"$job\", direction=\"transmitted\"}[5m]))", + "expr": "sum by (job) (rate(gauge_rustfs_process_network_io{job=~\"$job\", direction=\"transmitted\"}[5m]))", "legendFormat": "TX - {{job}}", "range": true, "refId": "D" @@ -3339,7 +3339,7 @@ "index": 0, "text": "INACTIVE" }, - "to": 1e-9 + "to": 1e-09 }, "type": "range" } @@ -4048,5 +4048,5 @@ "timezone": "browser", "title": "RustFS", "uid": "rustfs-s3", - "version": 11 -} + "version": 12 +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 5c9dc6f9c..46a6d29a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1227,8 +1227,6 @@ dependencies = [ "cfg-if", "constant_time_eq", "cpufeatures 0.2.17", - "memmap2 0.9.10", - "rayon-core", ] [[package]] @@ -7600,7 +7598,6 @@ dependencies = [ "rustls", "s3s", "serde", - "serde_bytes", "serde_json", "serde_urlencoded", "serial_test", @@ -7812,9 +7809,11 @@ name = "rustfs-metrics" version = "0.0.5" dependencies = [ "metrics", + "nvml-wrapper", "rustfs-ecstore", "rustfs-utils", "sysinfo", + "thiserror 2.0.18", "tokio", "tokio-util", "tracing", @@ -7863,7 +7862,6 @@ dependencies = [ "jiff", "metrics", "num_cpus", - "nvml-wrapper", "opentelemetry", "opentelemetry-appender-tracing", "opentelemetry-otlp", @@ -7874,8 +7872,6 @@ dependencies = [ "rustfs-config", "rustfs-utils", "serde", - "serde_json", - "sysinfo", "temp-env", "tempfile", "thiserror 2.0.18", @@ -7932,7 +7928,6 @@ dependencies = [ "hex", "hmac 0.13.0-rc.5", "http 1.4.0", - "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -8156,7 +8151,6 @@ version = "0.0.5" dependencies = [ "base64-simd", "blake2 0.11.0-rc.5", - "blake3", "brotli", "bytes", "convert_case 0.11.0", @@ -8609,16 +8603,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index bc40d3c43..14cd14624 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,7 +148,6 @@ rmcp = { version = "1.2.0" } rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } -serde_bytes = "0.11" serde_json = { version = "1.0.149", features = ["raw_value"] } serde_urlencoded = "0.7.1" schemars = "1.2.1" @@ -157,7 +156,6 @@ schemars = "1.2.1" aes-gcm = { version = "0.11.0-rc.3", features = ["rand_core"] } argon2 = { version = "0.6.0-rc.7" } blake2 = "0.11.0-rc.5" -blake3 = { version = "1.8.3", features = ["rayon", "mmap"] } chacha20poly1305 = { version = "0.11.0-rc.3" } crc-fast = "1.9.0" hmac = { version = "0.13.0-rc.5" } diff --git a/crates/appauth/Cargo.toml b/crates/appauth/Cargo.toml index ffbbbde8a..ee6c44b9e 100644 --- a/crates/appauth/Cargo.toml +++ b/crates/appauth/Cargo.toml @@ -33,3 +33,6 @@ rand.workspace = true [lints] workspace = true + +[lib] +doctest = false diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml index 137fcc6a7..ec904e1d6 100644 --- a/crates/audit/Cargo.toml +++ b/crates/audit/Cargo.toml @@ -47,3 +47,7 @@ rumqttc = { workspace = true } [lints] workspace = true + +[lib] +test = false +doctest = false diff --git a/crates/checksums/Cargo.toml b/crates/checksums/Cargo.toml index 14840a9e1..8d6fe9dc1 100644 --- a/crates/checksums/Cargo.toml +++ b/crates/checksums/Cargo.toml @@ -36,3 +36,6 @@ sha2 = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } + +[lib] +doctest = false diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 17ad6baf4..4886ce906 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -41,3 +41,6 @@ rmp-serde = { workspace = true } async-trait = { workspace = true } s3s = { workspace = true } tracing = { workspace = true } + +[lib] +doctest = false diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 3b2dcf9cb..faa664826 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -37,3 +37,6 @@ constants = ["dep:const-str"] notify = ["dep:const-str", "constants"] observability = ["constants"] opa = ["constants"] + +[lib] +doctest = false diff --git a/crates/crypto/Cargo.toml b/crates/crypto/Cargo.toml index b5e47cf58..fad498dca 100644 --- a/crates/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -56,3 +56,6 @@ crypto = [ "dep:rand", "dep:sha2", ] + +[lib] +doctest = false diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 71b862780..2bfa426b7 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -74,7 +74,6 @@ regex = { workspace = true } path-absolutize = { workspace = true } rmp.workspace = true rmp-serde.workspace = true -serde_bytes.workspace = true tokio-util = { workspace = true, features = ["io", "compat"] } base64 = { workspace = true } hmac = { workspace = true } @@ -134,3 +133,6 @@ harness = false [[bench]] name = "comparison_benchmark" harness = false + +[lib] +doctest = false diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index d80db2c38..b2d9feff5 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -51,3 +51,6 @@ harness = false [lints] workspace = true + +[lib] +doctest = false diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 423deead3..4be3271dd 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -50,3 +50,6 @@ tracing-subscriber = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } http = { workspace = true } + +[lib] +doctest = false diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index bc19644a9..d3958a2a6 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -58,3 +58,6 @@ url = { workspace = true } [dev-dependencies] pollster.workspace = true + +[lib] +doctest = false diff --git a/crates/lock/Cargo.toml b/crates/lock/Cargo.toml index e12b0d2bd..6e1e9eb05 100644 --- a/crates/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -43,3 +43,6 @@ parking_lot.workspace = true smallvec.workspace = true smartstring.workspace = true crossbeam-queue = { workspace = true } + +[lib] +doctest = false diff --git a/crates/madmin/Cargo.toml b/crates/madmin/Cargo.toml index ca346ebc7..c8c772b27 100644 --- a/crates/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -35,3 +35,6 @@ hyper.workspace = true serde.workspace = true serde_json.workspace = true time.workspace = true + +[lib] +doctest = false diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml index 06d080c03..9846b03d9 100644 --- a/crates/mcp/Cargo.toml +++ b/crates/mcp/Cargo.toml @@ -57,5 +57,6 @@ tracing-subscriber.workspace = true # File handling and MIME type detection mime_guess = { workspace = true } -[dev-dependencies] +[lib] +doctest = false # Testing framework and utilities diff --git a/crates/metrics/Cargo.toml b/crates/metrics/Cargo.toml index 7f3a29728..114ed6c66 100644 --- a/crates/metrics/Cargo.toml +++ b/crates/metrics/Cargo.toml @@ -25,6 +25,11 @@ keywords.workspace = true categories.workspace = true authors.workspace = true +[features] +default = [] +gpu = ["dep:nvml-wrapper"] +full = ["gpu"] + [dependencies] rustfs-ecstore = { workspace = true } rustfs-utils = { workspace = true } @@ -33,6 +38,8 @@ sysinfo = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "rt", "time", "macros"] } tokio-util = { workspace = true } tracing = { workspace = true } +thiserror = { workspace = true } +nvml-wrapper = { workspace = true, optional = true } [lints] workspace = true diff --git a/crates/metrics/src/collectors/global.rs b/crates/metrics/src/collectors/global.rs index 444b19835..ee7fe5239 100644 --- a/crates/metrics/src/collectors/global.rs +++ b/crates/metrics/src/collectors/global.rs @@ -16,13 +16,31 @@ //! //! This module provides the entry point for initializing all metrics collectors. //! The actual statistics collection functions are in `stats_collector.rs`. +//! +//! System monitoring collectors (migrated from `rustfs-obs::system`): +//! - Process CPU metrics +//! - Process memory metrics +//! - Process disk I/O metrics +//! - Process network I/O metrics use crate::collectors::stats_collector::{ collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats, collect_disk_stats, collect_process_stats, }; use crate::collectors::{ - collect_bucket_metrics, collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, + // System monitoring collectors (migrated from rustfs-obs::system) + ProcessCpuStats, + ProcessDiskStats, + ProcessMemoryStats, + ProcessNetworkStats, + collect_bucket_metrics, + collect_bucket_replication_bandwidth_metrics, + collect_cluster_metrics, + collect_node_metrics, + collect_process_cpu_metrics, + collect_process_disk_metrics, + collect_process_memory_metrics, + collect_process_network_metrics, collect_resource_metrics, }; use crate::constants::{ @@ -33,10 +51,19 @@ use crate::constants::{ }; use crate::format::report_metrics; use rustfs_utils::get_env_opt_u64; +use std::borrow::Cow; use std::time::Duration; +use sysinfo::{Pid, System}; use tokio_util::sync::CancellationToken; use tracing::warn; +/// Default interval for system monitoring metrics (15 seconds) +const DEFAULT_SYSTEM_METRICS_INTERVAL: Duration = Duration::from_secs(15); +/// Environment variable for system monitoring interval +const ENV_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_METRICS_SYSTEM_INTERVAL_SEC"; +/// Legacy environment variable for system monitoring interval +const LEGACY_SYSTEM_METRICS_INTERVAL: &str = "RUSTFS_OBS_METRICS_SYSTEM_INTERVAL_MS"; + /// Initialize all metrics collectors. /// /// This function spawns background tasks that periodically collect metrics @@ -69,48 +96,36 @@ pub fn init_metrics_collectors(token: CancellationToken) { const LEGACY_RESOURCE_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL"; const LEGACY_DEFAULT_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL"; - fn parse_interval(msc: &str, legacy_msc: &str) -> Option { - get_env_opt_u64(msc) - .or_else(|| get_env_opt_u64(legacy_msc)) - .filter(|&v| v > 0) - } - - // Read intervals from environment or use defaults, ensuring zero is ignored. - let cluster_interval = parse_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL) - .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) - .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) - .filter(|&v| v > 0) - .map(Duration::from_secs) - .unwrap_or(DEFAULT_CLUSTER_METRICS_INTERVAL); - - let bucket_interval = parse_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL) - .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) - .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) - .filter(|&v| v > 0) - .map(Duration::from_secs) - .unwrap_or(DEFAULT_BUCKET_METRICS_INTERVAL); - - let bucket_replication_bandwidth_interval = - parse_interval(ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, LEGACY_REPLICATION_BANDWIDTH_INTERVAL) + /// Parse metrics interval from environment variables with fallback to default. + /// + /// Priority: primary_env > legacy_env > default_env > legacy_default > default_value + fn parse_metrics_interval(primary_env: &str, legacy_env: &str, default_interval: Duration) -> Duration { + get_env_opt_u64(primary_env) + .or_else(|| get_env_opt_u64(legacy_env)) .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) .filter(|&v| v > 0) .map(Duration::from_secs) - .unwrap_or(DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL); + .unwrap_or(default_interval) + } - let node_interval = parse_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL) - .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) - .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) - .filter(|&v| v > 0) - .map(Duration::from_secs) - .unwrap_or(DEFAULT_NODE_METRICS_INTERVAL); + // Read intervals from environment or use defaults + let cluster_interval = + parse_metrics_interval(ENV_CLUSTER_METRICS_INTERVAL, LEGACY_CLUSTER_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL); - let resource_interval = parse_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL) - .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) - .or_else(|| get_env_opt_u64(LEGACY_DEFAULT_INTERVAL)) - .filter(|&v| v > 0) - .map(Duration::from_secs) - .unwrap_or(DEFAULT_RESOURCE_METRICS_INTERVAL); + let bucket_interval = + parse_metrics_interval(ENV_BUCKET_METRICS_INTERVAL, LEGACY_BUCKET_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL); + + let bucket_replication_bandwidth_interval = parse_metrics_interval( + ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, + LEGACY_REPLICATION_BANDWIDTH_INTERVAL, + DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, + ); + + let node_interval = parse_metrics_interval(ENV_NODE_METRICS_INTERVAL, LEGACY_NODE_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL); + + let resource_interval = + parse_metrics_interval(ENV_RESOURCE_METRICS_INTERVAL, LEGACY_RESOURCE_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL); // Spawn task for cluster metrics let token_clone = token.clone(); @@ -207,4 +222,132 @@ pub fn init_metrics_collectors(token: CancellationToken) { } } }); + + // Spawn task for system monitoring metrics (migrated from rustfs-obs::system) + let system_interval = get_env_opt_u64(ENV_SYSTEM_METRICS_INTERVAL) + .or_else(|| get_env_opt_u64(LEGACY_SYSTEM_METRICS_INTERVAL).map(|ms| ms / 1000)) // Convert ms to seconds + .or_else(|| get_env_opt_u64(ENV_DEFAULT_METRICS_INTERVAL)) + .filter(|&v| v > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_SYSTEM_METRICS_INTERVAL); + + let token_clone = token.clone(); + tokio::spawn(async move { + // Get current process PID + let pid = match sysinfo::get_current_pid() { + Ok(p) => p, + Err(e) => { + warn!("Failed to get current PID for system monitoring: {}", e); + return; + } + }; + + let mut interval = tokio::time::interval(system_interval); + loop { + tokio::select! { + _ = interval.tick() => { + // Collect system monitoring metrics + let metrics = collect_system_monitoring_metrics(pid); + report_metrics(&metrics); + } + _ = token_clone.cancelled() => { + warn!("System monitoring metrics collection cancelled."); + return; + } + } + } + }); +} + +/// Collect all system monitoring metrics for a process. +/// +/// This function collects CPU, memory, disk I/O, and network I/O metrics +/// for the specified process PID. +/// +/// # Arguments +/// * `pid` - The process ID to monitor +/// +/// # Returns +/// A vector of Prometheus metrics for the process. +fn collect_system_monitoring_metrics(pid: Pid) -> Vec { + let mut metrics = Vec::new(); + let mut system = System::new(); + + // Refresh process information + system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true); + + if let Some(process) = system.process(pid) { + // Create labels with process attributes + let labels: Vec<(&'static str, Cow<'static, str>)> = vec![ + ("process_pid", Cow::Owned(pid.as_u32().to_string())), + ("process_executable_name", Cow::Owned(process.name().to_string_lossy().to_string())), + ]; + + // Collect CPU metrics + let cpu_stats = ProcessCpuStats { + usage: process.cpu_usage() as f64, + utilization: process.cpu_usage() as f64, // Same as usage for single process + }; + metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(&labels))); + + // Collect memory metrics + let memory_stats = ProcessMemoryStats { + resident: process.memory(), + virtual_mem: process.virtual_memory(), + }; + metrics.extend(collect_process_memory_metrics(&memory_stats, Some(&labels))); + + // Collect disk I/O metrics + let disk_usage = process.disk_usage(); + let disk_stats = ProcessDiskStats { + read_bytes: disk_usage.read_bytes, + written_bytes: disk_usage.written_bytes, + }; + metrics.extend(collect_process_disk_metrics(&disk_stats, Some(&labels))); + + // Collect network I/O metrics + // Note: sysinfo 0.38.x provides network info via Networks new type + // We use Networks::new_with_refreshed_list() to get network interfaces + let networks = sysinfo::Networks::new_with_refreshed_list(); + let mut total_received = 0u64; + let mut total_transmitted = 0u64; + let mut per_interface = Vec::new(); + + for (interface_name, data) in networks.iter() { + let received = data.received(); + let transmitted = data.transmitted(); + total_received += received; + total_transmitted += transmitted; + per_interface.push((interface_name.to_string(), received, transmitted)); + } + + let network_stats = ProcessNetworkStats { + total_received, + total_transmitted, + per_interface, + }; + metrics.extend(collect_process_network_metrics(&network_stats, Some(&labels))); + + // Collect GPU metrics (if gpu feature is enabled) + #[cfg(feature = "gpu")] + { + use crate::collectors::{GpuCollector, collect_gpu_metrics}; + + match GpuCollector::new(pid) { + Ok(collector) => match collector.collect() { + Ok(gpu_stats) => { + metrics.extend(collect_gpu_metrics(&gpu_stats, &labels)); + } + Err(e) => { + warn!("GPU metrics collection failed: {}", e); + } + }, + Err(e) => { + warn!("GPU collector initialization failed: {}", e); + } + } + } + } + + metrics } diff --git a/crates/metrics/src/collectors/mod.rs b/crates/metrics/src/collectors/mod.rs index bc64c9728..d3972819b 100644 --- a/crates/metrics/src/collectors/mod.rs +++ b/crates/metrics/src/collectors/mod.rs @@ -82,6 +82,8 @@ mod scanner; mod stats_collector; mod system_cpu; mod system_drive; +#[cfg(feature = "gpu")] +mod system_gpu; mod system_memory; mod system_network; mod system_process; @@ -104,8 +106,16 @@ pub use replication::{ReplicationStats, collect_replication_metrics}; pub use request::{ApiRequestStats, collect_request_metrics}; pub use resource::{ResourceStats, collect_resource_metrics}; pub use scanner::{ScannerStats, collect_scanner_metrics}; -pub use system_cpu::{CpuStats, collect_cpu_metrics}; -pub use system_drive::{DriveCountStats, DriveDetailedStats, collect_drive_count_metrics, collect_drive_detailed_metrics}; -pub use system_memory::{MemoryStats, collect_memory_metrics}; -pub use system_network::{NetworkStats, collect_network_metrics}; -pub use system_process::{ProcessStats, collect_process_metrics}; +pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics}; +pub use system_drive::{ + DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics, + collect_process_disk_metrics, +}; +#[cfg(feature = "gpu")] +pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics}; +pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics}; +pub use system_network::{NetworkStats, ProcessNetworkStats, collect_network_metrics, collect_process_network_metrics}; +pub use system_process::{ + ProcessAttributeError, ProcessAttributes, ProcessStats, ProcessStatusType, collect_process_attributes, + collect_process_metrics, +}; diff --git a/crates/metrics/src/collectors/system_cpu.rs b/crates/metrics/src/collectors/system_cpu.rs index 3d3676de4..71de1f224 100644 --- a/crates/metrics/src/collectors/system_cpu.rs +++ b/crates/metrics/src/collectors/system_cpu.rs @@ -16,16 +16,18 @@ //! System CPU metrics collector. //! -//! Collects CPU-related metrics including load average, idle time, -//! I/O wait, and CPU usage percentages. +//! Collects CPU metrics including load average, CPU time distribution, +//! and process-level CPU usage. //! -//! This collector reuses the metric descriptors defined in `metrics_type::system_cpu` -//! to avoid duplication of metric names, types, and help text. +//! This module provides both system-level and process-level CPU metrics, +//! with process-level metrics migrated from `rustfs-obs::system`. use crate::format::PrometheusMetric; use crate::metrics_type::system_cpu::*; +use crate::metrics_type::system_process::{PROCESS_CPU_USAGE_MD, PROCESS_CPU_UTILIZATION_MD}; +use std::borrow::Cow; -/// CPU statistics for a node. +/// System CPU statistics. #[derive(Debug, Clone, Default)] pub struct CpuStats { /// Average CPU idle time (percentage, 0-100) @@ -46,6 +48,17 @@ pub struct CpuStats { pub user: f64, } +/// Process CPU statistics. +/// +/// Contains CPU usage metrics for a specific process. +#[derive(Debug, Clone, Default)] +pub struct ProcessCpuStats { + /// CPU usage percentage (0-100) + pub usage: f64, + /// CPU utilization percentage (considering multiple cores, can exceed 100) + pub utilization: f64, +} + /// Collects CPU metrics from the given stats. /// /// Uses the metric descriptors from `metrics_type::system_cpu` module. @@ -63,6 +76,29 @@ pub fn collect_cpu_metrics(stats: &CpuStats) -> Vec { ] } +/// Collects process CPU metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::system_process` module. +/// Returns a vector of Prometheus metrics for process CPU statistics. +/// +/// # Arguments +/// +/// * `stats` - Process CPU statistics +/// * `labels` - Optional additional labels (e.g., process attributes) +pub fn collect_process_cpu_metrics( + stats: &ProcessCpuStats, + labels: Option<&[(&'static str, Cow<'static, str>)]>, +) -> Vec { + let mut usage_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_USAGE_MD, stats.usage); + let mut utilization_metric = PrometheusMetric::from_descriptor(&PROCESS_CPU_UTILIZATION_MD, stats.utilization); + + if let Some(l) = labels { + usage_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + utilization_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + } + + vec![usage_metric, utilization_metric] +} #[cfg(test)] mod tests { use super::*; @@ -101,4 +137,48 @@ mod tests { assert!(metric.labels.is_empty()); } } + + #[test] + fn test_collect_process_cpu_metrics() { + let stats = ProcessCpuStats { + usage: 45.5, + utilization: 182.0, // 4 cores at ~45% each + }; + + let metrics = collect_process_cpu_metrics(&stats, None); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 2); + + // Verify usage metric + let usage_metric = metrics.iter().find(|m| m.name.contains("cpu_usage")); + assert!(usage_metric.is_some()); + assert_eq!(usage_metric.map(|m| m.value), Some(45.5)); + + // Verify utilization metric + let util_metric = metrics.iter().find(|m| m.name.contains("cpu_utilization")); + assert!(util_metric.is_some()); + assert_eq!(util_metric.map(|m| m.value), Some(182.0)); + } + + #[test] + fn test_collect_process_cpu_metrics_with_labels() { + let stats = ProcessCpuStats { + usage: 25.0, + utilization: 100.0, + }; + + let labels = vec![ + ("process_pid", Cow::Borrowed("12345")), + ("process_executable_name", Cow::Borrowed("rustfs")), + ]; + + let metrics = collect_process_cpu_metrics(&stats, Some(&labels)); + assert_eq!(metrics.len(), 2); + + // All metrics should have the labels + for metric in &metrics { + assert_eq!(metric.labels.len(), 2); + } + } } diff --git a/crates/metrics/src/collectors/system_drive.rs b/crates/metrics/src/collectors/system_drive.rs index 4d819cdfc..be4f71950 100644 --- a/crates/metrics/src/collectors/system_drive.rs +++ b/crates/metrics/src/collectors/system_drive.rs @@ -18,9 +18,14 @@ //! //! Collects detailed drive/disk metrics including capacity, I/O statistics, //! error counts, and health status. +//! +//! This module provides both system-level and process-level disk metrics, +//! with process-level metrics migrated from `rustfs-obs::system`. use crate::format::PrometheusMetric; use crate::metrics_type::system_drive::*; +use crate::metrics_type::system_process::PROCESS_DISK_IO_MD; +use std::borrow::Cow; /// Detailed drive statistics for a single drive. #[derive(Debug, Clone, Default)] @@ -163,6 +168,44 @@ pub fn collect_drive_count_metrics(stats: &DriveCountStats) -> Vec)]>, +) -> Vec { + let mut read_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.read_bytes as f64); + let mut write_metric = PrometheusMetric::from_descriptor(&PROCESS_DISK_IO_MD, stats.written_bytes as f64); + + read_metric.labels.push(("direction", Cow::Borrowed("read"))); + write_metric.labels.push(("direction", Cow::Borrowed("write"))); + + if let Some(l) = labels { + read_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + write_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + } + + vec![read_metric, write_metric] +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/metrics/src/collectors/system_gpu.rs b/crates/metrics/src/collectors/system_gpu.rs new file mode 100644 index 000000000..eca9a786b --- /dev/null +++ b/crates/metrics/src/collectors/system_gpu.rs @@ -0,0 +1,199 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! System GPU metrics collector. +//! +//! Collects GPU memory usage metrics using NVML library. +//! This module is only available when `gpu` feature is enabled. +//! +//! # Example +//! +//! ```ignore +//! use rustfs_metrics::collectors::{GpuCollector, collect_gpu_metrics}; +//! use sysinfo::Pid; +//! +//! let pid = sysinfo::get_current_pid().unwrap(); +//! let collector = GpuCollector::new(pid)?; +//! let stats = collector.collect()?; +//! let metrics = collect_gpu_metrics(&stats, &labels); +//! ``` + +use crate::format::PrometheusMetric; + +use crate::metrics_type::system_gpu::PROCESS_GPU_MEMORY_USAGE_MD; + +use nvml_wrapper::Nvml; + +use nvml_wrapper::enums::device::UsedGpuMemory; + +use sysinfo::Pid; + +use std::borrow::Cow; + +use thiserror::Error; + +use tracing::warn; + +/// GPU statistics. +/// +/// Contains GPU memory usage metrics for the monitored process. + +#[derive(Debug, Clone, Default)] +pub struct GpuStats { + /// GPU memory usage in bytes + pub memory_usage: u64, +} + +/// GPU collector error types. + +#[derive(Debug, Error)] +pub enum GpuError { + /// GPU initialization failed + #[error("GPU initialization failed: {0}")] + InitError(String), + + /// GPU device access error + #[error("GPU device error: {0}")] + DeviceError(String), + + /// Process not found in GPU process list + #[error("Process not found in GPU process list")] + ProcessNotFound, +} + +/// GPU metrics collector. +/// +/// Collects GPU memory usage metrics for a specific process using NVML. +pub struct GpuCollector { + /// NVML instance for GPU access + nvml: Nvml, + /// Process ID to monitor + pid: Pid, +} + +impl GpuCollector { + /// Creates a new GPU collector for the specified process. + /// + /// # Arguments + /// + /// * `pid` - The process ID to monitor + /// + /// # Errors + /// + /// Returns an error if NVML initialization fails. + /// + /// # Example + /// + /// ```ignore + /// use rustfs_metrics::collectors::GpuCollector; + /// use sysinfo::Pid; + /// + /// let pid = sysinfo::get_current_pid().unwrap(); + /// let collector = GpuCollector::new(pid)?; + /// ``` + pub fn new(pid: Pid) -> Result { + let nvml = Nvml::init().map_err(|e| GpuError::InitError(e.to_string()))?; + Ok(GpuCollector { nvml, pid }) + } + + /// Collects GPU metrics for the monitored process. + /// + /// Returns GPU memory usage statistics for the process. + /// + /// # Errors + /// + /// Returns an error if GPU device access fails. + /// + /// # Example + /// + /// ```ignore + /// let stats = collector.collect()?; + /// println!("GPU memory usage: {} bytes", stats.memory_usage); + /// ``` + pub fn collect(&self) -> Result { + if let Ok(device) = self.nvml.device_by_index(0) { + if let Ok(gpu_stats) = device.running_compute_processes() { + for stat in gpu_stats.iter() { + if stat.pid == self.pid.as_u32() { + let memory_used = match stat.used_gpu_memory { + UsedGpuMemory::Used(bytes) => bytes, + UsedGpuMemory::Unavailable => 0, + }; + return Ok(GpuStats { + memory_usage: memory_used, + }); + } + } + } else { + warn!("Could not get GPU stats, recording 0 for GPU memory usage"); + } + } else { + return Err(GpuError::DeviceError("No GPU device found".to_string())); + } + + // Process not found in GPU process list, return 0 usage + Ok(GpuStats { memory_usage: 0 }) + } +} + +/// Converts GPU stats to Prometheus metrics. +/// +/// # Arguments +/// +/// * `stats` - GPU statistics to convert +/// * `labels` - Metric labels (typically from ProcessAttributes) +/// +/// # Returns +/// +/// A vector of Prometheus metrics. +/// +/// # Example +/// +/// ```ignore +/// use rustfs_metrics::collectors::{GpuStats, collect_gpu_metrics}; +/// +/// let stats = GpuStats { memory_usage: 1024 }; +/// let labels = vec![("process_pid", Cow::Borrowed("1234"))]; +/// let metrics = collect_gpu_metrics(&stats, &labels); +/// ``` +pub fn collect_gpu_metrics(stats: &GpuStats, labels: &[(&'static str, Cow<'static, str>)]) -> Vec { + let mut metric = PrometheusMetric::from_descriptor(&PROCESS_GPU_MEMORY_USAGE_MD, stats.memory_usage as f64); + metric.labels.extend(labels.iter().map(|(k, v)| (*k, v.clone()))); + vec![metric] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gpu_stats_default() { + let stats = GpuStats::default(); + assert_eq!(stats.memory_usage, 0); + } + + #[test] + fn test_gpu_error_display() { + let err = GpuError::InitError("test error".to_string()); + assert!(err.to_string().contains("test error")); + + let err = GpuError::DeviceError("device error".to_string()); + assert!(err.to_string().contains("device error")); + + let err = GpuError::ProcessNotFound; + assert!(err.to_string().contains("Process not found")); + } +} diff --git a/crates/metrics/src/collectors/system_memory.rs b/crates/metrics/src/collectors/system_memory.rs index 0622bde90..ff8855ffe 100644 --- a/crates/metrics/src/collectors/system_memory.rs +++ b/crates/metrics/src/collectors/system_memory.rs @@ -19,13 +19,15 @@ //! Collects memory-related metrics including total, used, free, //! buffers, cache, shared, and available memory. //! -//! This collector reuses the metric descriptors defined in `metrics_type::system_memory` -//! to avoid duplication of metric names, types, and help text. +//! This module provides both system-level and process-level memory metrics, +//! with process-level metrics migrated from `rustfs-obs::system`. use crate::format::PrometheusMetric; use crate::metrics_type::system_memory::*; +use crate::metrics_type::system_process::{PROCESS_RESIDENT_MEMORY_BYTES_MD, PROCESS_VIRTUAL_MEMORY_BYTES_MD}; +use std::borrow::Cow; -/// Memory statistics for a node. +/// System memory statistics. #[derive(Debug, Clone, Default)] pub struct MemoryStats { /// Total memory in bytes @@ -46,6 +48,17 @@ pub struct MemoryStats { pub available: u64, } +/// Process memory statistics. +/// +/// Contains memory usage metrics for a specific process. +#[derive(Debug, Clone, Default)] +pub struct ProcessMemoryStats { + /// Resident memory size in bytes + pub resident: u64, + /// Virtual memory size in bytes + pub virtual_mem: u64, +} + /// Collects memory metrics from the given stats. /// /// Uses the metric descriptors from `metrics_type::system_memory` module. @@ -63,6 +76,30 @@ pub fn collect_memory_metrics(stats: &MemoryStats) -> Vec { ] } +/// Collects process memory metrics from the given stats. +/// +/// Uses the metric descriptors from `metrics_type::system_process` module. +/// Returns a vector of Prometheus metrics for process memory statistics. +/// +/// # Arguments +/// +/// * `stats` - Process memory statistics +/// * `labels` - Optional additional labels (e.g., process attributes) +pub fn collect_process_memory_metrics( + stats: &ProcessMemoryStats, + labels: Option<&[(&'static str, Cow<'static, str>)]>, +) -> Vec { + let mut resident_metric = PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident as f64); + let mut virtual_metric = PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_mem as f64); + + if let Some(l) = labels { + resident_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + virtual_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + } + + vec![resident_metric, virtual_metric] +} + #[cfg(test)] mod tests { use super::*; @@ -99,4 +136,34 @@ mod tests { assert!(metric.labels.is_empty()); } } + + #[test] + fn test_collect_process_memory_metrics() { + let stats = ProcessMemoryStats { + resident: 512 * 1024 * 1024, // 512 MB + virtual_mem: 2 * 1024 * 1024 * 1024, // 2 GB + }; + + let metrics = collect_process_memory_metrics(&stats, None); + report_metrics(&metrics); + + assert_eq!(metrics.len(), 2); + } + + #[test] + fn test_collect_process_memory_metrics_with_labels() { + let stats = ProcessMemoryStats { + resident: 256 * 1024 * 1024, + virtual_mem: 1024 * 1024 * 1024, + }; + + let labels = vec![("process_pid", Cow::Borrowed("12345"))]; + + let metrics = collect_process_memory_metrics(&stats, Some(&labels)); + assert_eq!(metrics.len(), 2); + + for metric in &metrics { + assert_eq!(metric.labels.len(), 1); + } + } } diff --git a/crates/metrics/src/collectors/system_network.rs b/crates/metrics/src/collectors/system_network.rs index 07f916d1c..bebac46c9 100644 --- a/crates/metrics/src/collectors/system_network.rs +++ b/crates/metrics/src/collectors/system_network.rs @@ -19,11 +19,13 @@ //! Collects internode network metrics including errors, dial times, //! and bytes sent/received. //! -//! This collector reuses the metric descriptors defined in `metrics_type::system_network` -//! to avoid duplication of metric names, types, and help text. +//! This module provides both system-level and process-level network metrics, +//! with process-level metrics migrated from `rustfs-obs::system`. use crate::format::PrometheusMetric; use crate::metrics_type::system_network::*; +use crate::metrics_type::system_process::{PROCESS_NETWORK_IO_MD, PROCESS_NETWORK_IO_PER_INTERFACE_MD}; +use std::borrow::Cow; /// Network statistics for internode communication. #[derive(Debug, Clone, Default)] @@ -40,6 +42,19 @@ pub struct NetworkStats { pub internode_recv_bytes_total: u64, } +/// Process network I/O statistics. +/// +/// Contains network I/O metrics for a specific process. +#[derive(Debug, Clone, Default)] +pub struct ProcessNetworkStats { + /// Total bytes received + pub total_received: u64, + /// Total bytes transmitted + pub total_transmitted: u64, + /// Per-interface statistics: (interface_name, received_bytes, transmitted_bytes) + pub per_interface: Vec<(String, u64, u64)>, +} + /// Collects network metrics from the given stats. /// /// Uses the metric descriptors from `metrics_type::system_network` module. @@ -54,6 +69,60 @@ pub fn collect_network_metrics(stats: &NetworkStats) -> Vec { ] } +/// Collects process network I/O metrics from the given stats. +/// +/// Returns a vector of Prometheus metrics for process network I/O statistics. +/// Each metric includes a `direction` label ("received" or "transmitted"). +/// Per-interface metrics also include an `interface` label. +/// +/// # Arguments +/// +/// * `stats` - Process network I/O statistics +/// * `labels` - Optional additional labels (e.g., process attributes) +pub fn collect_process_network_metrics( + stats: &ProcessNetworkStats, + labels: Option<&[(&'static str, Cow<'static, str>)]>, +) -> Vec { + let mut metrics = Vec::with_capacity(2 + stats.per_interface.len() * 2); + + // Total network I/O + let mut received_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_received as f64); + let mut transmitted_metric = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_MD, stats.total_transmitted as f64); + + received_metric.labels.push(("direction", Cow::Borrowed("received"))); + transmitted_metric.labels.push(("direction", Cow::Borrowed("transmitted"))); + + if let Some(l) = labels { + received_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + transmitted_metric.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + } + + metrics.push(received_metric); + metrics.push(transmitted_metric); + + // Per-interface network I/O + for (interface, received, transmitted) in &stats.per_interface { + let mut iface_received = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *received as f64); + let mut iface_transmitted = PrometheusMetric::from_descriptor(&PROCESS_NETWORK_IO_PER_INTERFACE_MD, *transmitted as f64); + + iface_received.labels.push(("interface", Cow::Owned(interface.clone()))); + iface_received.labels.push(("direction", Cow::Borrowed("received"))); + + iface_transmitted.labels.push(("interface", Cow::Owned(interface.clone()))); + iface_transmitted.labels.push(("direction", Cow::Borrowed("transmitted"))); + + if let Some(l) = labels { + iface_received.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + iface_transmitted.labels.extend(l.iter().map(|(k, v)| (*k, v.clone()))); + } + + metrics.push(iface_received); + metrics.push(iface_transmitted); + } + + metrics +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/metrics/src/collectors/system_process.rs b/crates/metrics/src/collectors/system_process.rs index cc11706b1..dcb9624d2 100644 --- a/crates/metrics/src/collectors/system_process.rs +++ b/crates/metrics/src/collectors/system_process.rs @@ -18,9 +18,130 @@ //! //! Collects process-level metrics including file descriptors, memory, //! syscalls, and runtime statistics. +//! +//! This module also provides process attribute collection for use as +//! metric labels, migrated from `rustfs-obs::system`. use crate::format::PrometheusMetric; use crate::metrics_type::system_process::*; +use std::borrow::Cow; +use sysinfo::{Pid, ProcessStatus, System}; + +/// Process attributes used as metric labels. +/// +/// Contains identifying information about the process being monitored. +#[derive(Debug, Clone)] +pub struct ProcessAttributes { + /// Process ID + pub pid: u32, + /// Executable name (e.g., "rustfs") + pub executable_name: String, + /// Full path to the executable + pub executable_path: String, + /// Full command line with arguments + pub command: String, +} + +impl ProcessAttributes { + /// Creates a new instance by reading from the current process. + /// + /// # Errors + /// + /// Returns an error if the current process PID cannot be determined + /// or if process information cannot be retrieved. + pub fn current() -> Result { + let pid = sysinfo::get_current_pid().map_err(|e| ProcessAttributeError::PidError(e.to_string()))?; + Self::from_pid(pid) + } + + /// Creates a new instance for a specific PID. + /// + /// # Arguments + /// + /// * `pid` - The process ID to query + /// + /// # Errors + /// + /// Returns an error if the process does not exist or information + /// cannot be retrieved. + pub fn from_pid(pid: Pid) -> Result { + let mut system = System::new(); + system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true); + + let process = system + .process(pid) + .ok_or_else(|| ProcessAttributeError::ProcessNotFound(pid.as_u32()))?; + + Ok(ProcessAttributes { + pid: pid.as_u32(), + executable_name: process.name().to_string_lossy().to_string(), + executable_path: process.exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_default(), + command: process + .cmd() + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .collect::>() + .join(" "), + }) + } + + /// Converts attributes to Prometheus metric labels. + pub fn to_labels(&self) -> Vec<(&'static str, Cow<'static, str>)> { + vec![ + ("process_pid", Cow::Owned(self.pid.to_string())), + ("process_executable_name", Cow::Owned(self.executable_name.clone())), + ("process_executable_path", Cow::Owned(self.executable_path.clone())), + ("process_command", Cow::Owned(self.command.clone())), + ] + } +} + +/// Errors that can occur when collecting process attributes. +#[derive(Debug, Clone)] +pub enum ProcessAttributeError { + /// Failed to get current process PID + PidError(String), + /// Process not found + ProcessNotFound(u32), +} + +impl std::fmt::Display for ProcessAttributeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::PidError(e) => write!(f, "Failed to get current PID: {}", e), + Self::ProcessNotFound(pid) => write!(f, "Process not found: {}", pid), + } + } +} + +impl std::error::Error for ProcessAttributeError {} + +/// Process status enumeration. +/// +/// Maps `sysinfo::ProcessStatus` to a simpler representation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ProcessStatusType { + /// Process is currently running + Running = 0, + /// Process is sleeping (waiting for I/O or event) + Sleeping = 1, + /// Process is a zombie (terminated but not reaped) + Zombie = 2, + /// Process is in some other state + #[default] + Other = 3, +} + +impl From for ProcessStatusType { + fn from(status: ProcessStatus) -> Self { + match status { + ProcessStatus::Run => ProcessStatusType::Running, + ProcessStatus::Sleep => ProcessStatusType::Sleeping, + ProcessStatus::Zombie => ProcessStatusType::Zombie, + _ => ProcessStatusType::Other, + } + } +} /// Process statistics for the RustFS server process. #[derive(Debug, Clone, Default)] @@ -59,13 +180,17 @@ pub struct ProcessStats { pub virtual_memory_bytes: u64, /// Maximum virtual memory size in bytes pub virtual_memory_max_bytes: u64, + /// Process status + pub status: ProcessStatusType, + /// Process status value (numeric) + pub status_value: i64, } /// Collects process metrics from the given stats. /// /// Returns a vector of Prometheus metrics for process statistics. pub fn collect_process_metrics(stats: &ProcessStats) -> Vec { - vec![ + let mut metrics = vec![ PrometheusMetric::from_descriptor(&PROCESS_LOCKS_READ_TOTAL_MD, stats.locks_read_total as f64), PrometheusMetric::from_descriptor(&PROCESS_LOCKS_WRITE_TOTAL_MD, stats.locks_write_total as f64), PrometheusMetric::from_descriptor(&PROCESS_CPU_TOTAL_SECONDS_MD, stats.cpu_total_seconds), @@ -83,7 +208,23 @@ pub fn collect_process_metrics(stats: &ProcessStats) -> Vec { PrometheusMetric::from_descriptor(&PROCESS_RESIDENT_MEMORY_BYTES_MD, stats.resident_memory_bytes as f64), PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_BYTES_MD, stats.virtual_memory_bytes as f64), PrometheusMetric::from_descriptor(&PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD, stats.virtual_memory_max_bytes as f64), - ] + ]; + + // Add process status metric + let mut status_metric = PrometheusMetric::from_descriptor(&PROCESS_STATUS_MD, stats.status_value as f64); + status_metric + .labels + .push(("status", Cow::Owned(format!("{:?}", stats.status)))); + metrics.push(status_metric); + + metrics +} + +/// Collects process attributes for the current process. +/// +/// This is a convenience function that wraps `ProcessAttributes::current()`. +pub fn collect_process_attributes() -> Result { + ProcessAttributes::current() } #[cfg(test)] @@ -111,12 +252,15 @@ mod tests { resident_memory_bytes: 1024 * 1024 * 512, virtual_memory_bytes: 1024 * 1024 * 1024, virtual_memory_max_bytes: 1024 * 1024 * 2048, + status: ProcessStatusType::Running, + status_value: 0, }; let metrics = collect_process_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 17); + // 17 original metrics + 1 status metric = 18 + assert_eq!(metrics.len(), 18); // Verify uptime let uptime_name = PROCESS_UPTIME_SECONDS_MD.get_full_metric_name(); @@ -129,6 +273,12 @@ mod tests { let fd_open = metrics.iter().find(|m| m.name == fd_open_name); assert!(fd_open.is_some()); assert_eq!(fd_open.map(|m| m.value), Some(1500.0)); + + // Verify status metric + let status_name = PROCESS_STATUS_MD.get_full_metric_name(); + let status_metric = metrics.iter().find(|m| m.name == status_name); + assert!(status_metric.is_some()); + assert_eq!(status_metric.map(|m| m.value), Some(0.0)); } #[test] @@ -136,10 +286,40 @@ mod tests { let stats = ProcessStats::default(); let metrics = collect_process_metrics(&stats); - assert_eq!(metrics.len(), 17); - for metric in &metrics { - assert_eq!(metric.value, 0.0); - assert!(metric.labels.is_empty()); - } + // 17 original metrics + 1 status metric = 18 + assert_eq!(metrics.len(), 18); + } + + #[test] + fn test_process_attributes_current() { + // This test should succeed as we're querying the current process + let result = collect_process_attributes(); + assert!(result.is_ok()); + + let attrs = result.unwrap(); + assert!(attrs.pid > 0); + assert!(!attrs.executable_name.is_empty()); + } + + #[test] + fn test_process_status_conversion() { + assert_eq!(ProcessStatusType::from(ProcessStatus::Run), ProcessStatusType::Running); + assert_eq!(ProcessStatusType::from(ProcessStatus::Sleep), ProcessStatusType::Sleeping); + assert_eq!(ProcessStatusType::from(ProcessStatus::Zombie), ProcessStatusType::Zombie); + } + + #[test] + fn test_process_attributes_to_labels() { + let attrs = ProcessAttributes { + pid: 12345, + executable_name: "rustfs".to_string(), + executable_path: "/usr/bin/rustfs".to_string(), + command: "rustfs server /data".to_string(), + }; + + let labels = attrs.to_labels(); + assert_eq!(labels.len(), 4); + assert_eq!(labels[0].0, "process_pid"); + assert_eq!(labels[0].1, "12345"); } } diff --git a/crates/metrics/src/metrics_type/entry/metric_name.rs b/crates/metrics/src/metrics_type/entry/metric_name.rs index af98dc774..7f1eeb9bc 100644 --- a/crates/metrics/src/metrics_type/entry/metric_name.rs +++ b/crates/metrics/src/metrics_type/entry/metric_name.rs @@ -350,6 +350,22 @@ pub enum MetricName { ProcessVirtualMemoryBytes, ProcessVirtualMemoryMaxBytes, + // Process-level system monitoring metrics (migrated from rustfs-obs::system) + /// Process CPU usage percentage (0-100) + ProcessCPUUsage, + /// Process CPU utilization percentage (considering multiple cores) + ProcessCPUUtilization, + /// Process disk I/O bytes + ProcessDiskIO, + /// Process network I/O bytes + ProcessNetworkIO, + /// Process network I/O bytes per interface + ProcessNetworkIOPerInterface, + /// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other) + ProcessStatus, + /// Process GPU memory usage in bytes + ProcessGpuMemoryUsage, + // Custom metrics Custom(String), } @@ -672,6 +688,15 @@ impl MetricName { Self::ProcessVirtualMemoryBytes => "virtual_memory_bytes".to_string(), Self::ProcessVirtualMemoryMaxBytes => "virtual_memory_max_bytes".to_string(), + // Process-level system monitoring metrics (migrated from rustfs-obs::system) + Self::ProcessCPUUsage => "cpu_usage".to_string(), + Self::ProcessCPUUtilization => "cpu_utilization".to_string(), + Self::ProcessDiskIO => "disk_io".to_string(), + Self::ProcessNetworkIO => "network_io".to_string(), + Self::ProcessNetworkIOPerInterface => "network_io_per_interface".to_string(), + Self::ProcessGpuMemoryUsage => "gpu_memory_usage".to_string(), + Self::ProcessStatus => "status".to_string(), + Self::Custom(name) => name.clone(), } } diff --git a/crates/metrics/src/metrics_type/entry/subsystem.rs b/crates/metrics/src/metrics_type/entry/subsystem.rs index 30f12f698..e6f0b83c1 100644 --- a/crates/metrics/src/metrics_type/entry/subsystem.rs +++ b/crates/metrics/src/metrics_type/entry/subsystem.rs @@ -32,6 +32,7 @@ pub enum MetricSubsystem { SystemCpu, SystemProcess, + SystemGpu, // debug related subsystems DebugGo, @@ -74,6 +75,7 @@ impl MetricSubsystem { Self::SystemProcess => "/system/process", // debug related subsystems + Self::SystemGpu => "/system/gpu", Self::DebugGo => "/debug/go", // cluster related subsystems @@ -120,6 +122,7 @@ impl MetricSubsystem { "/system/cpu" => Self::SystemCpu, "/system/process" => Self::SystemProcess, + "/system/gpu" => Self::SystemGpu, // Debug related subsystems "/debug/go" => Self::DebugGo, @@ -181,6 +184,7 @@ pub mod subsystems { pub const API_REQUESTS: MetricSubsystem = MetricSubsystem::ApiRequests; pub const BUCKET_API: MetricSubsystem = MetricSubsystem::BucketApi; pub const BUCKET_REPLICATION: MetricSubsystem = MetricSubsystem::BucketReplication; + pub const SYSTEM_GPU: MetricSubsystem = MetricSubsystem::SystemGpu; pub const SYSTEM_NETWORK_INTERNODE: MetricSubsystem = MetricSubsystem::SystemNetworkInternode; pub const SYSTEM_DRIVE: MetricSubsystem = MetricSubsystem::SystemDrive; pub const SYSTEM_MEMORY: MetricSubsystem = MetricSubsystem::SystemMemory; diff --git a/crates/metrics/src/metrics_type/mod.rs b/crates/metrics/src/metrics_type/mod.rs index d3eac431d..603a7b83e 100644 --- a/crates/metrics/src/metrics_type/mod.rs +++ b/crates/metrics/src/metrics_type/mod.rs @@ -33,6 +33,7 @@ pub mod request; pub mod scanner; pub mod system_cpu; pub mod system_drive; +pub mod system_gpu; pub mod system_memory; pub mod system_network; pub mod system_process; diff --git a/crates/metrics/src/metrics_type/system_gpu.rs b/crates/metrics/src/metrics_type/system_gpu.rs new file mode 100644 index 000000000..958e6cace --- /dev/null +++ b/crates/metrics/src/metrics_type/system_gpu.rs @@ -0,0 +1,35 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![allow(dead_code)] + +//! GPU-related metric descriptors. +//! +//! This module defines metric descriptors for GPU monitoring, +//! including GPU memory usage metrics. + +use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems}; +use std::sync::LazyLock; + +/// Process GPU memory usage metric descriptor. +/// +/// Records the amount of physical GPU memory in use by the process. +pub static PROCESS_GPU_MEMORY_USAGE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessGpuMemoryUsage, + "The amount of physical GPU memory in use", + &[], + subsystems::SYSTEM_GPU, + ) +}); diff --git a/crates/metrics/src/metrics_type/system_process.rs b/crates/metrics/src/metrics_type/system_process.rs index 1bb662b4b..c455faac4 100644 --- a/crates/metrics/src/metrics_type/system_process.rs +++ b/crates/metrics/src/metrics_type/system_process.rs @@ -186,3 +186,67 @@ pub static PROCESS_VIRTUAL_MEMORY_MAX_BYTES_MD: LazyLock = Laz subsystems::SYSTEM_PROCESS, ) }); + +// ============================================================================ +// Process-level system monitoring metrics (migrated from rustfs-obs::system) +// ============================================================================ + +/// Process CPU usage percentage (0-100) +pub static PROCESS_CPU_USAGE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessCPUUsage, + "The percentage of CPU in use by the process", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); + +/// Process CPU utilization percentage (considering multiple cores) +pub static PROCESS_CPU_UTILIZATION_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessCPUUtilization, + "The amount of CPU in use by the process (considering multiple cores)", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); + +/// Process disk I/O bytes +pub static PROCESS_DISK_IO_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessDiskIO, + "Disk bytes transferred by the process", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); + +/// Process network I/O bytes +pub static PROCESS_NETWORK_IO_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessNetworkIO, + "Network bytes transferred by the process", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); + +/// Process network I/O bytes per interface +pub static PROCESS_NETWORK_IO_PER_INTERFACE_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessNetworkIOPerInterface, + "Network bytes transferred by the process (per interface)", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); + +/// Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other) +pub static PROCESS_STATUS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ProcessStatus, + "Process status (0: Running, 1: Sleeping, 2: Zombie, 3: Other)", + &[], + subsystems::SYSTEM_PROCESS, + ) +}); diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index 6531cf87d..df929bf60 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -64,3 +64,6 @@ serde_json = { workspace = true } [lints] workspace = true + +[lib] +doctest = false diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 85fe9a652..2f2bd6932 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -28,14 +28,9 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/" [lints] workspace = true -[features] -default = [] -gpu = ["dep:nvml-wrapper"] -full = ["gpu"] - [dependencies] rustfs-config = { workspace = true, features = ["constants", "observability"] } -rustfs-utils = { workspace = true, features = ["ip", "path"] } +rustfs-utils = { workspace = true, features = ["ip"] } flate2 = { workspace = true } glob = { workspace = true } jiff = { workspace = true } @@ -44,7 +39,6 @@ crossbeam-channel = { workspace = true } crossbeam-deque = { workspace = true } crossbeam-utils = { workspace = true } num_cpus = { workspace = true } -nvml-wrapper = { workspace = true, optional = true } opentelemetry = { workspace = true } opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] } opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } @@ -58,7 +52,6 @@ tracing-error = { workspace = true } tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] } tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } -sysinfo = { workspace = true } thiserror = { workspace = true } zstd = { workspace = true, features = ["zstdmt"] } @@ -70,4 +63,3 @@ pyroscope = { workspace = true, features = ["backend-pprof-rs"] } tokio = { workspace = true, features = ["full"] } tempfile = { workspace = true } temp-env = { workspace = true } -serde_json = { workspace = true } diff --git a/crates/obs/examples/server.rs b/crates/obs/examples/server.rs index 2116dea1a..b8ca821ab 100644 --- a/crates/obs/examples/server.rs +++ b/crates/obs/examples/server.rs @@ -13,9 +13,9 @@ // limitations under the License. use opentelemetry::global; -use rustfs_obs::{SystemObserver, init_obs}; +use rustfs_obs::init_obs; use std::time::{Duration, SystemTime}; -use tracing::{Level, error, info, instrument}; +use tracing::{Level, info, instrument}; #[tokio::main] async fn main() { @@ -43,11 +43,6 @@ async fn run(service_name: String) { &[opentelemetry::KeyValue::new("operation", "run")], ); - match SystemObserver::init_process_observer().await { - Ok(_) => info!("Process observer initialized successfully"), - Err(e) => error!("Failed to initialize process observer: {:?}", e), - } - put_object("bucket".to_string(), "object".to_string(), "user".to_string()).await; info!("Logging is completed"); tokio::time::sleep(Duration::from_secs(2)).await; diff --git a/crates/obs/src/global.rs b/crates/obs/src/global.rs index 03b86442f..c71fdade9 100644 --- a/crates/obs/src/global.rs +++ b/crates/obs/src/global.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{AppConfig, GlobalError, OtelConfig, OtelGuard, SystemObserver, telemetry::init_telemetry}; +use crate::{AppConfig, GlobalError, OtelConfig, OtelGuard, telemetry::init_telemetry}; use std::sync::{Arc, Mutex}; use tokio::sync::OnceCell; -use tracing::{error, info, warn}; +use tracing::{info, warn}; /// Global guard for OpenTelemetry tracing static GLOBAL_GUARD: OnceCell>> = OnceCell::const_new(); @@ -116,17 +116,8 @@ pub async fn init_obs(endpoint: Option) -> Result Result { let otel_guard = init_telemetry(config)?; - tokio::spawn(async move { - let obs_result = SystemObserver::init_process_observer().await; - match obs_result { - Ok(_) => { - info!(target: "rustfs::obs::system::metrics", "Process observer initialized successfully"); - } - Err(e) => { - error!(target: "rustfs::obs::system::metrics", "Failed to initialize process observer: {}", e); - } - } - }); + // Note: System monitoring has been migrated to rustfs-metrics + // Use rustfs_metrics::init_metrics_collectors() for system metrics Ok(otel_guard) } diff --git a/crates/obs/src/lib.rs b/crates/obs/src/lib.rs index 08338d525..7d0cdf1c2 100644 --- a/crates/obs/src/lib.rs +++ b/crates/obs/src/lib.rs @@ -16,20 +16,12 @@ //! //! provides tools for system and service monitoring //! -//! ## feature mark -//! - `default`: default monitoring function -//! - `gpu`: gpu monitoring function -//! - `full`: includes all functions +//! ## Features //! -//! to enable gpu monitoring add in cargo toml -//! -//! ```toml -//! # using gpu monitoring -//! rustfs-obs = { version = "0.1.0", features = ["gpu"] } -//! -//! # use all functions -//! rustfs-obs = { version = "0.1.0", features = ["full"] } -//! ``` +//! This crate provides observability tools for RustFS: +//! - Logging with tracing +//! - Metrics collection +//! - Distributed tracing //! //! ## Usage //! @@ -53,16 +45,27 @@ //! # // Guard will be dropped here, flushing telemetry data //! # } //! ``` +//! +//! ## System Monitoring Migration +//! +//! The system monitoring functionality has been migrated to `rustfs-metrics`. +//! Use `rustfs_metrics::init_metrics_collectors()` for system metrics collection. +//! +//! ```ignore +//! use tokio_util::sync::CancellationToken; +//! use rustfs_metrics::init_metrics_collectors; +//! +//! let token = CancellationToken::new(); +//! init_metrics_collectors(token.clone()); +//! ``` mod cleaner; mod config; mod error; mod global; -mod system; mod telemetry; pub use cleaner::*; pub use config::*; pub use error::*; pub use global::*; -pub use system::SystemObserver; pub use telemetry::{OtelGuard, Recorder}; diff --git a/crates/obs/src/system/attributes.rs b/crates/obs/src/system/attributes.rs deleted file mode 100644 index 86c553cec..000000000 --- a/crates/obs/src/system/attributes.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::GlobalError; -use opentelemetry::KeyValue; -use sysinfo::{Pid, System}; - -pub(crate) const PROCESS_PID: opentelemetry::Key = opentelemetry::Key::from_static_str("process.pid"); -pub(crate) const PROCESS_EXECUTABLE_NAME: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.name"); -pub(crate) const PROCESS_EXECUTABLE_PATH: opentelemetry::Key = opentelemetry::Key::from_static_str("process.executable.path"); -pub(crate) const PROCESS_COMMAND: opentelemetry::Key = opentelemetry::Key::from_static_str("process.command"); - -/// Struct to hold process attributes -pub struct ProcessAttributes { - pub attributes: Vec, -} - -impl ProcessAttributes { - /// Creates a new instance of `ProcessAttributes` for the given PID. - pub fn new(pid: Pid, system: &mut System) -> Result { - system.refresh_processes(sysinfo::ProcessesToUpdate::Some(&[pid]), true); - let process = system - .process(pid) - .ok_or_else(|| GlobalError::ProcessNotFound(pid.as_u32()))?; - - let attributes = vec![ - KeyValue::new(PROCESS_PID, pid.as_u32() as i64), - KeyValue::new(PROCESS_EXECUTABLE_NAME, process.name().to_os_string().into_string().unwrap_or_default()), - KeyValue::new( - PROCESS_EXECUTABLE_PATH, - process - .exe() - .map(|path| path.to_string_lossy().into_owned()) - .unwrap_or_default(), - ), - KeyValue::new( - PROCESS_COMMAND, - process - .cmd() - .iter() - .fold(String::new(), |t1, t2| t1 + " " + t2.to_str().unwrap_or_default()), - ), - ]; - - Ok(ProcessAttributes { attributes }) - } -} diff --git a/crates/obs/src/system/collector.rs b/crates/obs/src/system/collector.rs deleted file mode 100644 index 45e84d23e..000000000 --- a/crates/obs/src/system/collector.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::GlobalError; -use crate::system::attributes::ProcessAttributes; -use crate::system::metrics::{DIRECTION, INTERFACE, Metrics, STATUS}; -use opentelemetry::KeyValue; -use std::time::SystemTime; -use sysinfo::{Networks, Pid, ProcessStatus, System}; -use tokio::time::{Duration, sleep}; - -/// Collector is responsible for collecting system metrics and attributes. -/// It uses the sysinfo crate to gather information about the system and processes. -/// It also uses OpenTelemetry to record metrics. -pub struct Collector { - metrics: Metrics, - attributes: ProcessAttributes, - #[cfg(feature = "gpu")] - gpu_collector: crate::system::gpu::GpuCollector, - pid: Pid, - system: System, - networks: Networks, - core_count: usize, - interval_ms: u64, -} - -impl Collector { - pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result { - let mut system = System::new(); - let attributes = ProcessAttributes::new(pid, &mut system)?; - let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?; - let metrics = Metrics::new(&meter); - #[cfg(feature = "gpu")] - let gpu_collector = crate::system::gpu::GpuCollector::new(pid)?; - let networks = Networks::new_with_refreshed_list(); - - Ok(Collector { - metrics, - attributes, - #[cfg(feature = "gpu")] - gpu_collector, - pid, - system, - networks, - core_count, - interval_ms, - }) - } - - pub async fn run(&mut self) -> Result<(), GlobalError> { - loop { - self.collect()?; - tracing::debug!("Collected metrics for PID: {} ,time: {:?}", self.pid, SystemTime::now()); - sleep(Duration::from_millis(self.interval_ms)).await; - } - } - - fn collect(&mut self) -> Result<(), GlobalError> { - self.system - .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), false); - - // refresh the network interface list and statistics - self.networks.refresh(false); - - let process = self - .system - .process(self.pid) - .ok_or_else(|| GlobalError::ProcessNotFound(self.pid.as_u32()))?; - - // CPU metrics - let cpu_usage = process.cpu_usage(); - self.metrics.cpu_usage.record(cpu_usage as f64, &[]); - self.metrics - .cpu_utilization - .record((cpu_usage / self.core_count as f32) as f64, &self.attributes.attributes); - - // Memory metrics - self.metrics - .memory_usage - .record(process.memory() as i64, &self.attributes.attributes); - self.metrics - .memory_virtual - .record(process.virtual_memory() as i64, &self.attributes.attributes); - - // Disk I/O metrics - let disk_io = process.disk_usage(); - self.metrics.disk_io.record( - disk_io.read_bytes as i64, - &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "read")]].concat(), - ); - self.metrics.disk_io.record( - disk_io.written_bytes as i64, - &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "write")]].concat(), - ); - - // Network I/O indicators (corresponding to /system/network/internode) - let mut total_received: i64 = 0; - let mut total_transmitted: i64 = 0; - - // statistics by interface - for (interface_name, data) in self.networks.iter() { - total_received += data.total_received() as i64; - total_transmitted += data.total_transmitted() as i64; - - let received = data.received() as i64; - let transmitted = data.transmitted() as i64; - self.metrics.network_io_per_interface.record( - received, - &[ - &self.attributes.attributes[..], - &[ - KeyValue::new(INTERFACE, interface_name.to_string()), - KeyValue::new(DIRECTION, "received"), - ], - ] - .concat(), - ); - self.metrics.network_io_per_interface.record( - transmitted, - &[ - &self.attributes.attributes[..], - &[ - KeyValue::new(INTERFACE, interface_name.to_string()), - KeyValue::new(DIRECTION, "transmitted"), - ], - ] - .concat(), - ); - } - // global statistics - self.metrics.network_io.record( - total_received, - &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "received")]].concat(), - ); - self.metrics.network_io.record( - total_transmitted, - &[&self.attributes.attributes[..], &[KeyValue::new(DIRECTION, "transmitted")]].concat(), - ); - - // Process status indicator (corresponding to /system/process) - let status_value = match process.status() { - ProcessStatus::Run => 0, - ProcessStatus::Sleep => 1, - ProcessStatus::Zombie => 2, - _ => 3, // other status - }; - self.metrics.process_status.record( - status_value, - &[ - &self.attributes.attributes[..], - &[KeyValue::new(STATUS, format!("{:?}", process.status()))], - ] - .concat(), - ); - - // GPU Metrics (Optional) Non-MacOS - #[cfg(feature = "gpu")] - self.gpu_collector.collect(&self.metrics, &self.attributes)?; - - Ok(()) - } -} diff --git a/crates/obs/src/system/gpu.rs b/crates/obs/src/system/gpu.rs deleted file mode 100644 index 403c30ae8..000000000 --- a/crates/obs/src/system/gpu.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::GlobalError; -use crate::system::attributes::ProcessAttributes; -use crate::system::metrics::Metrics; -use nvml_wrapper::Nvml; -use nvml_wrapper::enums::device::UsedGpuMemory; -use sysinfo::Pid; -use tracing::warn; - -/// `GpuCollector` is responsible for collecting GPU memory usage metrics. -pub struct GpuCollector { - nvml: Nvml, - pid: Pid, -} - -impl GpuCollector { - pub fn new(pid: Pid) -> Result { - let nvml = Nvml::init().map_err(|e| GlobalError::GpuInitError(e.to_string()))?; - Ok(GpuCollector { nvml, pid }) - } - - pub fn collect(&self, metrics: &Metrics, attributes: &ProcessAttributes) -> Result<(), GlobalError> { - if let Ok(device) = self.nvml.device_by_index(0) { - if let Ok(gpu_stats) = device.running_compute_processes() { - for stat in gpu_stats.iter() { - if stat.pid == self.pid.as_u32() { - let memory_used = match stat.used_gpu_memory { - UsedGpuMemory::Used(bytes) => bytes, - UsedGpuMemory::Unavailable => 0, - }; - metrics.gpu_memory_usage.record(memory_used, &attributes.attributes); - return Ok(()); - } - } - } else { - warn!("Could not get GPU stats, recording 0 for GPU memory usage"); - } - } else { - return Err(GlobalError::GpuDeviceError("No GPU device found".to_string())); - } - metrics.gpu_memory_usage.record(0, &attributes.attributes); - Ok(()) - } -} diff --git a/crates/obs/src/system/metrics.rs b/crates/obs/src/system/metrics.rs deleted file mode 100644 index 43883956b..000000000 --- a/crates/obs/src/system/metrics.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use opentelemetry::metrics::{Gauge, Meter}; - -pub(crate) const PROCESS_CPU_USAGE: &str = "process.cpu.usage"; -pub(crate) const PROCESS_CPU_UTILIZATION: &str = "process.cpu.utilization"; -pub(crate) const PROCESS_MEMORY_USAGE: &str = "process.memory.usage"; -pub(crate) const PROCESS_MEMORY_VIRTUAL: &str = "process.memory.virtual"; -pub(crate) const PROCESS_DISK_IO: &str = "process.disk.io"; -pub(crate) const PROCESS_NETWORK_IO: &str = "process.network.io"; -pub(crate) const PROCESS_NETWORK_IO_PER_INTERFACE: &str = "process.network.io.per_interface"; -pub(crate) const PROCESS_STATUS: &str = "process.status"; -#[cfg(feature = "gpu")] -pub const PROCESS_GPU_MEMORY_USAGE: &str = "process.gpu.memory.usage"; -pub(crate) const DIRECTION: opentelemetry::Key = opentelemetry::Key::from_static_str("direction"); -pub(crate) const STATUS: opentelemetry::Key = opentelemetry::Key::from_static_str("status"); -pub(crate) const INTERFACE: opentelemetry::Key = opentelemetry::Key::from_static_str("interface"); - -/// `Metrics` struct holds the OpenTelemetry metrics for process monitoring. -/// It contains various metrics such as CPU usage, memory usage, -/// disk I/O, network I/O, and process status. -/// -/// The `Metrics` struct is designed to be used with OpenTelemetry's -/// metrics API to record and export these metrics. -/// -/// The `new` method initializes the metrics using the provided -/// `opentelemetry::metrics::Meter`. -pub struct Metrics { - pub cpu_usage: Gauge, - pub cpu_utilization: Gauge, - pub memory_usage: Gauge, - pub memory_virtual: Gauge, - pub disk_io: Gauge, - pub network_io: Gauge, - pub network_io_per_interface: Gauge, - pub process_status: Gauge, - #[cfg(feature = "gpu")] - pub gpu_memory_usage: Gauge, -} - -impl Metrics { - pub fn new(meter: &Meter) -> Self { - let cpu_usage = meter - .f64_gauge(PROCESS_CPU_USAGE) - .with_description("The percentage of CPU in use.") - .with_unit("percent") - .build(); - let cpu_utilization = meter - .f64_gauge(PROCESS_CPU_UTILIZATION) - .with_description("The amount of CPU in use.") - .with_unit("percent") - .build(); - let memory_usage = meter - .i64_gauge(PROCESS_MEMORY_USAGE) - .with_description("The amount of physical memory in use.") - .with_unit("byte") - .build(); - let memory_virtual = meter - .i64_gauge(PROCESS_MEMORY_VIRTUAL) - .with_description("The amount of committed virtual memory.") - .with_unit("byte") - .build(); - let disk_io = meter - .i64_gauge(PROCESS_DISK_IO) - .with_description("Disk bytes transferred.") - .with_unit("byte") - .build(); - let network_io = meter - .i64_gauge(PROCESS_NETWORK_IO) - .with_description("Network bytes transferred.") - .with_unit("byte") - .build(); - let network_io_per_interface = meter - .i64_gauge(PROCESS_NETWORK_IO_PER_INTERFACE) - .with_description("Network bytes transferred (per interface).") - .with_unit("byte") - .build(); - - let process_status = meter - .i64_gauge(PROCESS_STATUS) - .with_description("Process status (0: Running, 1: Sleeping, 2: Zombie, etc.)") - .build(); - - #[cfg(feature = "gpu")] - let gpu_memory_usage = meter - .u64_gauge(PROCESS_GPU_MEMORY_USAGE) - .with_description("The amount of physical GPU memory in use.") - .with_unit("byte") - .build(); - - Metrics { - cpu_usage, - cpu_utilization, - memory_usage, - memory_virtual, - disk_io, - network_io, - network_io_per_interface, - process_status, - #[cfg(feature = "gpu")] - gpu_memory_usage, - } - } -} diff --git a/crates/obs/src/system/mod.rs b/crates/obs/src/system/mod.rs deleted file mode 100644 index 4050465ea..000000000 --- a/crates/obs/src/system/mod.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{GlobalError, observability_metric_enabled}; -use opentelemetry::{global::meter, metrics::Meter}; -use sysinfo::Pid; - -mod attributes; -mod collector; -#[cfg(feature = "gpu")] -mod gpu; -mod metrics; - -pub struct SystemObserver {} - -impl SystemObserver { - /// Initialize the indicator collector for the current process - /// This function will create a new `Collector` instance and start collecting metrics. - /// It will run indefinitely until the process is terminated. - pub async fn init_process_observer() -> Result<(), GlobalError> { - if observability_metric_enabled() { - let meter = meter("system"); - let pid = sysinfo::get_current_pid().map_err(|e| GlobalError::PidError(e.to_string()))?; - return SystemObserver::init_process_observer_for_pid(meter, pid).await; - } - Ok(()) - } - - /// Initialize the metric collector for the specified PID process - /// This function will create a new `Collector` instance and start collecting metrics. - /// It will run indefinitely until the process is terminated. - pub async fn init_process_observer_for_pid(meter: Meter, pid: Pid) -> Result<(), GlobalError> { - let interval_ms = rustfs_utils::get_env_u64( - rustfs_config::observability::ENV_OBS_METRICS_SYSTEM_INTERVAL_MS, - rustfs_config::observability::DEFAULT_METRICS_SYSTEM_INTERVAL_MS, - ); - let mut collector = collector::Collector::new(pid, meter, interval_ms)?; - collector.run().await - } -} diff --git a/crates/policy/Cargo.toml b/crates/policy/Cargo.toml index b48f700c4..0239baeb6 100644 --- a/crates/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -54,3 +54,6 @@ pollster.workspace = true pollster.workspace = true test-case.workspace = true temp-env = { workspace = true } + +[lib] +doctest = false diff --git a/crates/protocols/Cargo.toml b/crates/protocols/Cargo.toml index 17cceca5d..23b954181 100644 --- a/crates/protocols/Cargo.toml +++ b/crates/protocols/Cargo.toml @@ -40,7 +40,6 @@ swift = [ "dep:sha2", "dep:uuid", "dep:futures", - "dep:http-body", "dep:http-body-util", "dep:tokio-util", "dep:serde", @@ -99,7 +98,6 @@ percent-encoding = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } uuid = { workspace = true, optional = true } futures = { workspace = true, optional = true } -http-body = { workspace = true, optional = true } http-body-util = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } serde = { workspace = true, optional = true } diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index f0c7bd2a1..58b735a64 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -40,3 +40,7 @@ tonic = { workspace = true, features = ["transport"] } tonic-prost = { workspace = true } tonic-prost-build = { workspace = true } tracing = { workspace = true } + +[lib] +test = false +doctest = false diff --git a/crates/s3-common/Cargo.toml b/crates/s3-common/Cargo.toml index 49a0550a6..6b1ae5811 100644 --- a/crates/s3-common/Cargo.toml +++ b/crates/s3-common/Cargo.toml @@ -31,3 +31,6 @@ workspace = true metrics = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } + +[lib] +doctest = false diff --git a/crates/s3select-api/Cargo.toml b/crates/s3select-api/Cargo.toml index 7c177dcaa..b09c770bd 100644 --- a/crates/s3select-api/Cargo.toml +++ b/crates/s3select-api/Cargo.toml @@ -45,4 +45,7 @@ tokio.workspace = true tokio-util.workspace = true tracing.workspace = true transform-stream.workspace = true -url.workspace = true \ No newline at end of file +url.workspace = true + +[lib] +doctest = false diff --git a/crates/s3select-query/Cargo.toml b/crates/s3select-query/Cargo.toml index 84aa0e0cd..bbc7a176a 100644 --- a/crates/s3select-query/Cargo.toml +++ b/crates/s3select-query/Cargo.toml @@ -37,3 +37,6 @@ s3s.workspace = true snafu = { workspace = true, features = ["backtrace"] } tokio = { workspace = true } tracing = { workspace = true } + +[lib] +doctest = false diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 825245f42..ec97bfa1b 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -56,3 +56,6 @@ tracing-subscriber = { workspace = true } serial_test = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } tokio = { workspace = true, features = ["test-util"] } + +[lib] +doctest = false diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 646ed3534..49291ebe3 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -36,7 +36,8 @@ rustfs-utils = { workspace = true, features = ["full"] } s3s.workspace = true base64-simd.workspace = true -[dev-dependencies] - [lints] workspace = true + +[lib] +doctest = false diff --git a/crates/trusted-proxies/Cargo.toml b/crates/trusted-proxies/Cargo.toml index 87b4199dd..a342bcfd7 100644 --- a/crates/trusted-proxies/Cargo.toml +++ b/crates/trusted-proxies/Cargo.toml @@ -58,3 +58,6 @@ path = "tests/unit/mod.rs" [[test]] name = "integration_tests" path = "tests/integration/mod.rs" + +[lib] +doctest = false diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 50789dd4b..c4f6742d1 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -27,7 +27,6 @@ categories = ["web-programming", "development-tools", "cryptography"] [dependencies] base64-simd = { workspace = true, optional = true } blake2 = { workspace = true, optional = true } -blake3 = { workspace = true, optional = true } brotli = { workspace = true, optional = true } bytes = { workspace = true, optional = true } crc-fast = { workspace = true, optional = true } @@ -89,7 +88,7 @@ notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex"] crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"] -hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"] +hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake2", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:crc-fast"] os = ["dep:rustix", "dep:tempfile", "dep:windows"] # operating system utilities integration = [] # integration test features sys = ["dep:sysinfo"] # system information features diff --git a/crates/workers/Cargo.toml b/crates/workers/Cargo.toml index 97c9aa8e3..70a360ef9 100644 --- a/crates/workers/Cargo.toml +++ b/crates/workers/Cargo.toml @@ -31,3 +31,6 @@ workspace = true [dependencies] tokio = { workspace = true, features = ["sync", "time", "macros"] } tracing.workspace = true + +[lib] +doctest = false diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 072a97fd4..f07da5ba0 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -42,3 +42,6 @@ astral-tokio-tar = { workspace = true } [lints] workspace = true + +[lib] +doctest = false diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 7a40250be..8535927ca 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -33,11 +33,12 @@ path = "src/main.rs" [features] default = ["metrics"] metrics = [] +metrics-gpu = ["metrics", "rustfs-metrics/gpu"] ftps = ["rustfs-protocols/ftps"] swift = ["rustfs-protocols/swift"] webdav = ["rustfs-protocols/webdav"] license = [] -full = ["metrics", "ftps", "swift", "webdav"] +full = ["metrics", "metrics-gpu", "ftps", "swift", "webdav"] [lints] workspace = true diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 3484080e5..07e57d1a7 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -25,6 +25,7 @@ use crate::storage::s3_api::{acl, encryption, replication, tagging}; use crate::storage::*; use futures::StreamExt; use http::StatusCode; +use metrics::counter; use rustfs_config::RUSTFS_REGION; use rustfs_ecstore::bucket::{ lifecycle::bucket_lifecycle_ops::validate_transition_tier, @@ -197,7 +198,7 @@ impl DefaultBucketUsecase { } let output = CreateBucketOutput::default(); - + counter!("rustfs_create_bucket_total").increment(1); let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); result diff --git a/rustfs/src/storage/concurrency.rs b/rustfs/src/storage/concurrency.rs index 054199d77..42ac4cbcb 100644 --- a/rustfs/src/storage/concurrency.rs +++ b/rustfs/src/storage/concurrency.rs @@ -1178,7 +1178,7 @@ impl HotObjectCache { #[cfg(all(feature = "metrics", not(test)))] { use metrics::counter; - counter!("rustfs_object_cache_invalidations").increment(1); + counter!("rustfs_object_cache_invalidations_total").increment(1); } } diff --git a/scripts/run.sh b/scripts/run.sh index a0fdef7cf..0ae46ca0d 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -53,7 +53,7 @@ export RUSTFS_CONSOLE_ADDRESS=":9001" # export RUSTFS_TLS_PATH="./deploy/certs" # Observability related configuration -#export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector address +export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector address # RustFS OR OTEL exporter configuration #export RUSTFS_OBS_TRACE_ENDPOINT=http://localhost:4318/v1/traces # OpenTelemetry Collector trace address http://localhost:4318/v1/traces #export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:14318/v1/traces @@ -61,15 +61,15 @@ export RUSTFS_CONSOLE_ADDRESS=":9001" #export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:9090/api/v1/otlp/v1/metrics #export RUSTFS_OBS_LOG_ENDPOINT=http://loki:3100/otlp/v1/logs # OpenTelemetry Collector logs address http://loki:3100/otlp/v1/logs #export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://loki:3100/otlp/v1/logs -#export RUSTFS_OBS_PROFILING_ENDPOINT=http://localhost:4040 # OpenTelemetry Collector profiling address +export RUSTFS_OBS_PROFILING_ENDPOINT=http://localhost:4040 # OpenTelemetry Collector profiling address #export RUSTFS_OBS_USE_STDOUT=true # Whether to use standard output -#export RUSTFS_OBS_SAMPLE_RATIO=2.0 # Sample ratio, between 0.0-1.0, 0.0 means no sampling, 1.0 means full sampling -#export RUSTFS_OBS_METER_INTERVAL=1 # Sampling interval in seconds -#export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name -#export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version +export RUSTFS_OBS_SAMPLE_RATIO=2.0 # Sample ratio, between 0.0-1.0, 0.0 means no sampling, 1.0 means full sampling +export RUSTFS_OBS_METER_INTERVAL=1 # Sampling interval in seconds +export RUSTFS_OBS_SERVICE_NAME=rustfs # Service name +export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging +#export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep