gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): improve telemetry stack, replication metrics, and Grafana alignment (#2672)

Browse Source 
Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-04-24 21:50:17 +08:00
committed by GitHub
parent 2705e3f53b
commit 13b4500212
19 changed files with 4603 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/global.rs
+12 -12
View File
@@ -24,16 +24,16 @@ static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
pub(crate) static OBSERVABILITY_METRIC_ENABLED: OnceCell<bool> = OnceCell::const_new();
/// Namespaced metrics for cleaner and rolling logging.
pub(crate) const METRIC_LOG_CLEANER_DELETED_FILES_TOTAL: &str = "rustfs.log_cleaner.deleted_files_total";
pub(crate) const METRIC_LOG_CLEANER_FREED_BYTES_TOTAL: &str = "rustfs.log_cleaner.freed_bytes_total";
pub(crate) const METRIC_LOG_CLEANER_COMPRESS_DURATION_SECONDS: &str = "rustfs.log_cleaner.compress_duration_seconds";
pub(crate) const METRIC_LOG_CLEANER_STEAL_SUCCESS_RATE: &str = "rustfs.log_cleaner.steal_success_rate";
pub(crate) const METRIC_LOG_CLEANER_RUNS_TOTAL: &str = "rustfs.log_cleaner.runs_total";
pub(crate) const METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL: &str = "rustfs.log_cleaner.run_failures_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_TOTAL: &str = "rustfs.log_cleaner.rotation_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL: &str = "rustfs.log_cleaner.rotation_failures_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS: &str = "rustfs.log_cleaner.rotation_duration_seconds";
pub(crate) const METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES: &str = "rustfs.log_cleaner.active_file_size_bytes";
pub(crate) const METRIC_LOG_CLEANER_DELETED_FILES_TOTAL: &str = "rustfs_log_cleaner_deleted_files_total";
pub(crate) const METRIC_LOG_CLEANER_FREED_BYTES_TOTAL: &str = "rustfs_log_cleaner_freed_bytes_total";
pub(crate) const METRIC_LOG_CLEANER_COMPRESS_DURATION_SECONDS: &str = "rustfs_log_cleaner_compress_duration_seconds";
pub(crate) const METRIC_LOG_CLEANER_STEAL_SUCCESS_RATE: &str = "rustfs_log_cleaner_steal_success_rate";
pub(crate) const METRIC_LOG_CLEANER_RUNS_TOTAL: &str = "rustfs_log_cleaner_runs_total";
pub(crate) const METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL: &str = "rustfs_log_cleaner_run_failures_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_TOTAL: &str = "rustfs_log_cleaner_rotation_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL: &str = "rustfs_log_cleaner_rotation_failures_total";
pub(crate) const METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS: &str = "rustfs_log_cleaner_rotation_duration_seconds";
pub(crate) const METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES: &str = "rustfs_log_cleaner_active_file_size_bytes";
/// Check whether Observability metric is enabled
pub fn observability_metric_enabled() -> bool {
@@ -198,8 +198,8 @@ mod tests {
for metric in metrics {
assert!(
metric.starts_with("rustfs.log_cleaner."),
"metric '{metric}' should use rustfs.log_cleaner.* namespace"
metric.starts_with("rustfs_log_cleaner_"),
"metric '{metric}' should use rustfs_log_cleaner_* namespace"
);
}
}
crates/obs/src/metrics/collectors/bucket_replication.rs
+243 -28
View File
@@ -12,34 +12,62 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket replication bandwidth metrics collector.
//!
//! Collects bandwidth metrics for bucket replication targets.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
//! to avoid duplication of metric names, types, and help text.
//! Bucket replication metrics collector.
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD,
BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
};
use std::borrow::Cow;
/// Bucket replication bandwidth statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
pub target_arn: String,
pub bandwidth_limit_bytes_per_sec: u64,
pub current_bandwidth_bytes_per_sec: f64,
pub latency_ms: f64,
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
/// Name of the bucket
pub bucket: String,
/// Target ARN for replication
pub target_arn: String,
/// Configured bandwidth limit in bytes per second
pub limit_bytes_per_sec: u64,
/// Current bandwidth in bytes per second (EWMA)
pub current_bandwidth_bytes_per_sec: f64,
}
/// Collects bucket replication bandwidth metrics from the provided statistics.
///
/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
/// Returns a vector of Prometheus metrics for replication bandwidth.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationStats {
pub bucket: String,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
pub last_min_failed_bytes: u64,
pub last_min_failed_count: u64,
pub last_hour_failed_bytes: u64,
pub last_hour_failed_count: u64,
pub sent_bytes: u64,
pub sent_count: u64,
pub proxied_get_requests_total: u64,
pub proxied_get_requests_failures: u64,
pub proxied_head_requests_total: u64,
pub proxied_head_requests_failures: u64,
pub proxied_put_tagging_requests_total: u64,
pub proxied_put_tagging_requests_failures: u64,
pub proxied_get_tagging_requests_total: u64,
pub proxied_get_tagging_requests_failures: u64,
pub proxied_delete_tagging_requests_total: u64,
pub proxied_delete_tagging_requests_failures: u64,
pub targets: Vec<BucketReplicationTargetStats>,
}
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -52,24 +80,206 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
.with_label("bucket", bucket_label.clone())
.with_label("targetArn", target_arn_label.clone()),
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
.with_label("bucket", bucket_label)
.with_label("targetArn", target_arn_label),
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_arn_label),
);
}
metrics
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::new();
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_BYTES_MD, stat.total_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_COUNT_MD, stat.total_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, stat.last_min_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, stat.last_min_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, stat.last_hour_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, stat.last_hour_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_BYTES_MD, stat.sent_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_COUNT_MD, stat.sent_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, stat.proxied_get_requests_total as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
stat.proxied_get_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
stat.proxied_head_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
stat.proxied_head_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_put_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_put_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_get_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_get_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_delete_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_delete_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
for target in &stat.targets {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LATENCY_MS_MD, target.latency_ms)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(OPERATION_L, Cow::Borrowed("object_replication"))
.with_label(RANGE_L, Cow::Borrowed("all"))
.with_label(TARGET_ARN_L, target_label),
);
}
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 0,
targets: vec![BucketReplicationTargetStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
}],
}];
let metrics = collect_bucket_replication_metrics(&stats);
assert_eq!(metrics.len(), 19);
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == sent_name
&& metric.value == 8.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == latency_name
&& metric.value == 15.0
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
}
#[test]
fn test_collect_bucket_replication_metrics_empty() {
let stats: Vec<BucketReplicationStats> = Vec::new();
let metrics = collect_bucket_replication_metrics(&stats);
assert!(metrics.is_empty());
}
#[test]
fn test_collect_bucket_replication_bandwidth_metrics() {
let stats = vec![BucketReplicationBandwidthStats {
@@ -83,24 +293,29 @@ mod tests {
assert_eq!(metrics.len(), 2);
let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
let limit_metric = metrics.iter().find(|m| {
m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
let limit_metric = metrics.iter().find(|metric| {
metric.name == limit_metric_name
&& metric.value == 1_048_576.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
});
assert!(limit_metric.is_some());
assert!(
limit_metric
.and_then(|m| {
m.labels
.and_then(|metric| {
metric
.labels
.iter()
.find(|(k, _)| *k == "targetArn")
.map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
.find(|(key, _)| *key == TARGET_ARN_L)
.map(|(_, value)| value.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
})
.unwrap_or(false)
);
let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
let current_metric = metrics.iter().find(|m| {
m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
let current_metric = metrics.iter().find(|metric| {
metric.name == current_metric_name
&& metric.value == 204_800.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
});
assert!(current_metric.is_some());
}
crates/obs/src/metrics/collectors/mod.rs
+4 -1
View File
@@ -41,7 +41,10 @@ pub mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
};
pub use cluster::{ClusterStats, collect_cluster_metrics};
pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
crates/obs/src/metrics/collectors/replication.rs
+22 -2
View File
@@ -25,6 +25,14 @@ use crate::metrics::schema::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
/// Average number of active replication workers
pub average_active_workers: f64,
/// Average queued bytes since server start
pub average_queued_bytes: i64,
/// Average queued objects since server start
pub average_queued_count: i64,
/// Average data transfer rate in bytes/sec
pub average_data_transfer_rate: f64,
/// Number of active replication workers
pub active_workers: u64,
/// Current data transfer rate in bytes/sec
@@ -50,6 +58,10 @@ pub struct ReplicationStats {
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_MD, stats.average_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD, stats.average_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
@@ -70,6 +82,10 @@ mod tests {
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
average_active_workers: 8.5,
average_queued_bytes: 1024 * 1024 * 40,
average_queued_count: 240,
average_data_transfer_rate: 1024.0 * 1024.0 * 3.0,
active_workers: 10,
current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
last_minute_queued_bytes: 1024 * 1024 * 100, // 100 MB
@@ -84,13 +100,17 @@ mod tests {
let metrics = collect_replication_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 13);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
let active = metrics.iter().find(|m| m.name == active_name);
assert!(active.is_some());
assert_eq!(active.map(|m| m.value), Some(10.0));
let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
assert_eq!(avg_active.map(|m| m.value), Some(8.5));
}
#[test]
@@ -98,7 +118,7 @@ mod tests {
let stats = ReplicationStats::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 13);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());
crates/obs/src/metrics/scheduler.rs
+30 -9
View File
@@ -35,8 +35,14 @@ use crate::metrics::collectors::{
collect_audit_metrics,
collect_bucket_metrics,
collect_bucket_replication_bandwidth_metrics,
collect_bucket_replication_metrics,
collect_cluster_health_metrics,
collect_cluster_metrics,
collect_cpu_metrics,
collect_drive_count_metrics,
collect_drive_detailed_metrics,
collect_host_network_metrics,
collect_memory_metrics,
collect_node_metrics,
collect_notification_metrics,
collect_notification_target_metrics,
@@ -45,6 +51,7 @@ use crate::metrics::collectors::{
collect_process_disk_metrics,
collect_process_memory_metrics,
collect_process_metrics,
collect_replication_metrics,
collect_resource_metrics,
};
use crate::metrics::config::{
@@ -56,14 +63,16 @@ use crate::metrics::config::{
};
use crate::metrics::report::{PrometheusMetric, report_metrics};
use crate::metrics::stats_collector::{
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_stats, collect_cluster_stats,
collect_disk_stats, collect_host_network_stats, collect_process_metric_bundle,
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
collect_bucket_stats, collect_cluster_and_health_stats, collect_disk_and_system_drive_stats, collect_host_network_stats,
collect_process_metric_bundle, collect_replication_stats, collect_system_cpu_and_memory_stats_with,
};
use rustfs_audit::audit_target_metrics;
use rustfs_notify::{notification_metrics_snapshot, notification_target_metrics};
use rustfs_utils::get_env_opt_u64;
use std::borrow::Cow;
use std::time::Duration;
use sysinfo::System;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::warn;
@@ -154,8 +163,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
loop {
tokio::select! {
_ = interval.tick() => {
let stats = collect_cluster_stats().await;
let metrics = collect_cluster_metrics(&stats);
let (stats, cluster_health) = collect_cluster_and_health_stats().await;
let mut metrics = collect_cluster_metrics(&stats);
metrics.extend(collect_cluster_health_metrics(&cluster_health));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -192,8 +202,10 @@ pub fn init_metrics_runtime(token: CancellationToken) {
loop {
tokio::select! {
_ = interval.tick() => {
let stats = collect_disk_stats().await;
let metrics = collect_node_metrics(&stats);
let (disk_stats, drive_stats, drive_counts) = collect_disk_and_system_drive_stats().await;
let mut metrics = collect_node_metrics(&disk_stats);
metrics.extend(collect_drive_detailed_metrics(&drive_stats));
metrics.extend(collect_drive_count_metrics(&drive_counts));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -212,7 +224,11 @@ pub fn init_metrics_runtime(token: CancellationToken) {
tokio::select! {
_ = interval.tick() => {
let stats = collect_bucket_replication_bandwidth_stats();
let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
let mut metrics = collect_bucket_replication_bandwidth_metrics(&stats);
let bucket_replication = collect_bucket_replication_detail_stats().await;
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
let replication = collect_replication_stats().await;
metrics.extend(collect_replication_metrics(&replication));
report_metrics(&metrics);
}
_ = token_clone.cancelled() => {
@@ -297,6 +313,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let token_clone = token;
tokio::spawn(async move {
let labels = current_process_metric_labels();
let mut host_system = System::new_all();
let process_interval = resource_interval.min(system_interval);
let mut interval = tokio::time::interval(process_interval);
let now = Instant::now();
@@ -327,9 +344,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
if now >= next_system_run {
#[cfg(feature = "gpu")]
let mut metrics = collect_system_monitoring_metrics(&bundle, &labels);
let mut metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
#[cfg(not(feature = "gpu"))]
let metrics = collect_system_monitoring_metrics(&bundle, &labels);
let metrics = collect_system_monitoring_metrics(&bundle, &labels, &mut host_system);
#[cfg(feature = "gpu")]
if let Some(pid) = current_pid {
@@ -418,6 +435,7 @@ fn fallback_process_metric_labels(err: ProcessAttributeError) -> Vec<(&'static s
fn collect_system_monitoring_metrics(
bundle: &ProcessMetricBundle,
labels: &[(&'static str, Cow<'static, str>)],
host_system: &mut System,
) -> Vec<PrometheusMetric> {
let cpu_stats = ProcessCpuStats {
usage: bundle.resource.cpu_percent,
@@ -432,8 +450,11 @@ fn collect_system_monitoring_metrics(
written_bytes: bundle.disk_write_bytes,
};
let network_stats = collect_host_network_stats();
let (system_cpu_stats, system_memory_stats) = collect_system_cpu_and_memory_stats_with(host_system);
let mut metrics = Vec::new();
metrics.extend(collect_cpu_metrics(&system_cpu_stats));
metrics.extend(collect_memory_metrics(&system_memory_stats));
metrics.extend(collect_process_cpu_metrics(&cpu_stats, Some(labels)));
metrics.extend(collect_process_memory_metrics(&memory_stats, Some(labels)));
metrics.extend(collect_process_disk_metrics(&disk_stats, Some(labels)));
crates/obs/src/metrics/schema/bucket_replication.rs
+1 -1
View File
@@ -22,7 +22,7 @@ pub const BUCKET_L: &str = "bucket";
/// Replication operation
pub const OPERATION_L: &str = "operation";
/// Replication target ARN
pub const TARGET_ARN_L: &str = "targetArn";
pub const TARGET_ARN_L: &str = "target_arn";
/// Replication range
pub const RANGE_L: &str = "range";
crates/obs/src/metrics/stats_collector.rs
+338 -25
View File
@@ -21,19 +21,41 @@
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::{
BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, HostNetworkStats, ProcessStats, ProcessStatusType,
ResourceStats,
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, ClusterHealthStats,
ClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, HostNetworkStats, MemoryStats, ProcessStats,
ProcessStatusType, ReplicationStats, ResourceStats,
};
use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
use rustfs_ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use rustfs_ecstore::data_usage::load_data_usage_from_backend;
use rustfs_ecstore::global::get_global_bucket_monitor;
use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
use sysinfo::Networks;
use std::time::Duration;
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
const DRIVE_STATE_OK: &str = "ok";
const DRIVE_STATE_ONLINE: &str = "online";
const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
const DRIVE_RUNTIME_STATE_RETURNING: &str = "returning";
fn disk_is_online_for_metrics(state: &str, runtime_state: Option<&str>) -> bool {
let state_is_acceptable = state.eq_ignore_ascii_case(DRIVE_STATE_OK)
|| state.eq_ignore_ascii_case(DRIVE_STATE_ONLINE)
|| state.eq_ignore_ascii_case(DRIVE_STATE_UNFORMATTED);
if let Some(runtime_state) = runtime_state {
let runtime_state_is_acceptable = runtime_state.eq_ignore_ascii_case(DRIVE_STATE_ONLINE)
|| runtime_state.eq_ignore_ascii_case(DRIVE_RUNTIME_STATE_RETURNING);
return runtime_state_is_acceptable && state_is_acceptable;
}
state_is_acceptable
}
#[derive(Debug, Clone, Default)]
pub struct ProcessMetricBundle {
pub resource: ResourceStats,
@@ -42,48 +64,76 @@ pub struct ProcessMetricBundle {
pub disk_write_bytes: u64,
}
/// Collect cluster statistics from the storage layer.
#[instrument]
pub async fn collect_cluster_stats() -> ClusterStats {
/// Collect cluster and cluster-health statistics from a single storage snapshot.
pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthStats) {
let Some(store) = new_object_layer_fn() else {
return ClusterStats::default();
return (ClusterStats::default(), ClusterHealthStats::default());
};
let storage_info = store.storage_info().await;
let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
let usable_capacity = get_total_usable_capacity(&storage_info.disks, &storage_info) as u64;
let free = get_total_usable_capacity_free(&storage_info.disks, &storage_info) as u64;
// Get bucket and object counts from data usage info
// Get bucket and object counts from data usage info.
let (buckets_count, objects_count) = match load_data_usage_from_backend(store.clone()).await {
Ok(data_usage) => (data_usage.buckets_count, data_usage.objects_total_count),
Err(e) => {
warn!("Failed to load data usage from backend: {}", e);
// Fall back to bucket list for buckets_count, objects_count stays 0
// Fall back to bucket list for buckets_count, objects_count stays 0.
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
..Default::default()
})
.await
.unwrap_or_else(|e| {
warn!("Failed to list buckets for cluster metrics: {}", e);
.unwrap_or_else(|err| {
warn!("Failed to list buckets for cluster metrics: {}", err);
Vec::new()
});
(buckets.len() as u64, 0)
}
};
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
let mut online = 0u64;
let mut offline = 0u64;
for disk in &storage_info.disks {
if disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref()) {
online += 1;
} else {
offline += 1;
}
}
(
ClusterStats {
raw_capacity_bytes: raw_capacity,
usable_capacity_bytes: usable_capacity,
used_bytes: used,
free_bytes: free,
objects_count,
buckets_count,
},
ClusterHealthStats {
drives_offline_count: offline,
drives_online_count: online,
drives_count: storage_info.disks.len() as u64,
},
)
}
/// Collect cluster statistics from the storage layer.
#[instrument]
pub async fn collect_cluster_stats() -> ClusterStats {
let (cluster_stats, _) = collect_cluster_and_health_stats().await;
cluster_stats
}
/// Collect cluster health statistics from the storage layer.
pub async fn collect_cluster_health_stats() -> ClusterHealthStats {
let (_, cluster_health_stats) = collect_cluster_and_health_stats().await;
cluster_health_stats
}
/// Collect bucket statistics from the storage layer.
@@ -177,15 +227,207 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
.collect()
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let Some(store) = new_object_layer_fn() else {
/// Collect bucket and target level replication stats from the global replication runtime.
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
let Some(stats) = GLOBAL_REPLICATION_STATS.get() else {
return Vec::new();
};
let storage_info = store.storage_info().await;
let all_bucket_stats = stats.get_all().await;
let mut buckets = Vec::with_capacity(all_bucket_stats.len());
storage_info
for (bucket, bucket_stats) in all_bucket_stats {
let proxy = stats.get_proxy_stats(&bucket).await;
let mut total_failed_bytes = 0u64;
let mut total_failed_count = 0u64;
let mut last_min_failed_bytes = 0u64;
let mut last_min_failed_count = 0u64;
let mut last_hour_failed_bytes = 0u64;
let mut last_hour_failed_count = 0u64;
let mut sent_bytes = 0u64;
let mut sent_count = 0u64;
let mut targets = Vec::with_capacity(bucket_stats.stats.len());
for (target_arn, target_stats) in bucket_stats.stats {
total_failed_bytes += target_stats.fail_stats.size.max(0) as u64;
total_failed_count += target_stats.fail_stats.count.max(0) as u64;
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
last_min_failed_bytes += last_min.size.max(0) as u64;
last_min_failed_count += last_min.count.max(0) as u64;
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
last_hour_failed_bytes += last_hour.size.max(0) as u64;
last_hour_failed_count += last_hour.count.max(0) as u64;
sent_bytes += target_stats.replicated_size.max(0) as u64;
sent_count += target_stats.replicated_count.max(0) as u64;
targets.push(BucketReplicationTargetStats {
target_arn,
bandwidth_limit_bytes_per_sec: target_stats.bandwidth_limit_bytes_per_sec.max(0) as u64,
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
});
}
buckets.push(BucketReplicationStats {
bucket,
total_failed_bytes,
total_failed_count,
last_min_failed_bytes,
last_min_failed_count,
last_hour_failed_bytes,
last_hour_failed_count,
sent_bytes,
sent_count,
proxied_get_requests_total: proxy.get_total.max(0) as u64,
proxied_get_requests_failures: proxy.get_failed.max(0) as u64,
proxied_head_requests_total: proxy.head_total.max(0) as u64,
proxied_head_requests_failures: proxy.head_failed.max(0) as u64,
// Proxy cache currently tracks generic PutObject requests, not tagging-specific APIs.
// Keep tagging counters zero until PutObjectTagging stats are tracked separately.
proxied_put_tagging_requests_total: 0,
proxied_put_tagging_requests_failures: 0,
proxied_get_tagging_requests_total: 0,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 0,
proxied_delete_tagging_requests_failures: 0,
targets,
});
}
buckets
}
/// Collect site-level replication stats from the global replication runtime.
pub async fn collect_replication_stats() -> ReplicationStats {
let Some(stats) = GLOBAL_REPLICATION_STATS.get() else {
return ReplicationStats::default();
};
let site_metrics = stats.get_sr_metrics_for_node().await;
let current_active_workers = u64::try_from(site_metrics.active_workers.curr).unwrap_or(0);
let bandwidth_stats = collect_bucket_replication_bandwidth_stats();
let current_data_transfer_rate = bandwidth_stats
.iter()
.map(|stat| stat.current_bandwidth_bytes_per_sec)
.sum::<f64>();
let all_bucket_stats = stats.get_all().await;
let average_data_transfer_rate = all_bucket_stats
.values()
.flat_map(|bucket| bucket.stats.values())
.map(|stat| stat.xfer_rate_lrg.avg + stat.xfer_rate_sml.avg)
.sum::<f64>();
let max_data_transfer_rate = all_bucket_stats
.values()
.flat_map(|bucket| bucket.stats.values())
.map(|stat| stat.xfer_rate_lrg.peak + stat.xfer_rate_sml.peak)
.sum::<f64>();
let recent_backlog_count = stats
.mrf_stats
.values()
.copied()
.filter(|value| *value > 0)
.sum::<i64>()
.try_into()
.unwrap_or(0);
ReplicationStats {
average_active_workers: site_metrics.active_workers.avg,
average_queued_bytes: site_metrics.queued.avg.bytes,
average_queued_count: site_metrics.queued.avg.count,
average_data_transfer_rate,
active_workers: current_active_workers,
current_data_transfer_rate,
last_minute_queued_bytes: site_metrics.queued.last_minute.bytes.max(0) as u64,
last_minute_queued_count: site_metrics.queued.last_minute.count.max(0) as u64,
max_active_workers: u64::try_from(site_metrics.active_workers.max).unwrap_or(0),
max_queued_bytes: site_metrics.queued.max.bytes.max(0) as u64,
max_queued_count: site_metrics.queued.max.count.max(0) as u64,
max_data_transfer_rate,
recent_backlog_count,
}
}
/// Collect disk statistics from the storage layer.
pub async fn collect_disk_stats() -> Vec<DiskStats> {
let (disk_stats, _, _) = collect_disk_and_system_drive_stats().await;
disk_stats
}
fn build_system_cpu_stats(system: &System) -> CpuStats {
let cpu_usage = system.global_cpu_usage() as f64;
let cpu_count = system.cpus().len().max(1) as f64;
let load_avg = System::load_average().one;
CpuStats {
avg_idle: (100.0 - cpu_usage).max(0.0),
avg_iowait: 0.0,
load_avg,
load_avg_perc: (load_avg / cpu_count) * 100.0,
nice: 0.0,
steal: 0.0,
system: cpu_usage,
user: 0.0,
}
}
fn build_system_memory_stats(system: &System) -> MemoryStats {
let total = system.total_memory();
let used = system.used_memory();
MemoryStats {
total,
used,
used_perc: if total > 0 {
(used as f64 / total as f64) * 100.0
} else {
0.0
},
free: system.free_memory(),
buffers: 0,
cache: 0,
shared: 0,
available: system.available_memory(),
}
}
/// Collect system CPU and memory statistics from a shared sysinfo snapshot.
pub fn collect_system_cpu_and_memory_stats() -> (CpuStats, MemoryStats) {
let mut system = System::new_all();
collect_system_cpu_and_memory_stats_with(&mut system)
}
/// Collect system CPU and memory statistics by refreshing a reusable sysinfo instance.
pub fn collect_system_cpu_and_memory_stats_with(system: &mut System) -> (CpuStats, MemoryStats) {
system.refresh_cpu_all();
system.refresh_memory();
(build_system_cpu_stats(system), build_system_memory_stats(system))
}
/// Collect system CPU statistics from the current host.
pub fn collect_system_cpu_stats() -> CpuStats {
let (cpu_stats, _) = collect_system_cpu_and_memory_stats();
cpu_stats
}
/// Collect system memory statistics from the current host.
pub fn collect_system_memory_stats() -> MemoryStats {
let (_, memory_stats) = collect_system_cpu_and_memory_stats();
memory_stats
}
/// Collect node disk stats and drive stats from a single storage snapshot.
pub async fn collect_disk_and_system_drive_stats() -> (Vec<DiskStats>, Vec<DriveDetailedStats>, DriveCountStats) {
let Some(store) = new_object_layer_fn() else {
return (Vec::new(), Vec::new(), DriveCountStats::default());
};
let storage_info = store.storage_info().await;
let disk_stats = storage_info
.disks
.iter()
.map(|disk| DiskStats {
@@ -195,7 +437,63 @@ pub async fn collect_disk_stats() -> Vec<DiskStats> {
used_bytes: disk.used_space,
free_bytes: disk.available_space,
})
.collect()
.collect();
let mut online_count = 0u64;
let mut offline_count = 0u64;
let drive_stats = storage_info
.disks
.iter()
.map(|disk| {
let is_online = disk_is_online_for_metrics(disk.state.as_str(), disk.runtime_state.as_deref());
if is_online {
online_count += 1;
} else {
offline_count += 1;
}
DriveDetailedStats {
server: disk.endpoint.clone(),
drive: disk.drive_path.clone(),
total_bytes: disk.total_space,
used_bytes: disk.used_space,
free_bytes: disk.available_space,
used_inodes: 0,
free_inodes: 0,
total_inodes: 0,
timeout_errors_total: 0,
io_errors_total: 0,
availability_errors_total: 0,
waiting_io: 0,
api_latency_micros: 0,
health: if is_online { 1 } else { 0 },
reads_per_sec: 0.0,
reads_kb_per_sec: 0.0,
reads_await: 0.0,
writes_per_sec: 0.0,
writes_kb_per_sec: 0.0,
writes_await: 0.0,
perc_util: if disk.total_space > 0 {
(disk.used_space as f64 / disk.total_space as f64) * 100.0
} else {
0.0
},
}
})
.collect();
let drive_count_stats = DriveCountStats {
offline_count,
online_count,
total_count: online_count + offline_count,
};
(disk_stats, drive_stats, drive_count_stats)
}
/// Collect system drive statistics using the storage layer snapshot.
pub async fn collect_system_drive_stats() -> (Vec<DriveDetailedStats>, DriveCountStats) {
let (_, drive_stats, drive_count_stats) = collect_disk_and_system_drive_stats().await;
(drive_stats, drive_count_stats)
}
/// Collect resource and process statistics for the current process in one sysinfo refresh.
@@ -286,3 +584,18 @@ pub fn collect_host_network_stats() -> HostNetworkStats {
per_interface,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn disk_is_online_for_metrics_accepts_online_state_case_insensitive() {
assert!(disk_is_online_for_metrics("OnLiNe", Some("online")));
}
#[test]
fn disk_is_online_for_metrics_rejects_offline_runtime_state() {
assert!(!disk_is_online_for_metrics(DRIVE_STATE_OK, Some("offline")));
}
}