diff --git a/crates/obs/src/metrics/collectors/cluster_usage.rs b/crates/obs/src/metrics/collectors/cluster_usage.rs index a1259402b..85e6cb1d7 100644 --- a/crates/obs/src/metrics/collectors/cluster_usage.rs +++ b/crates/obs/src/metrics/collectors/cluster_usage.rs @@ -21,10 +21,13 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::cluster_usage::*; +use std::time::{Duration, SystemTime}; /// Cluster-wide usage statistics. #[derive(Debug, Clone, Default)] pub struct ClusterUsageStats { + /// Time since the persisted usage snapshot was last updated + pub since_last_update_seconds: u64, /// Total bytes used in the cluster pub total_bytes: u64, /// Total number of objects @@ -33,6 +36,8 @@ pub struct ClusterUsageStats { pub versions_count: u64, /// Total number of delete markers pub delete_markers_count: u64, + /// Total number of buckets in the usage snapshot + pub buckets_count: u64, /// Object size distribution by range pub object_size_distribution: Vec<(String, u64)>, /// Version count distribution by range @@ -64,8 +69,12 @@ pub struct BucketUsageStats { /// /// Returns a vector of Prometheus metrics for cluster usage. pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec { - let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len()); + let mut metrics = Vec::with_capacity(6 + stats.object_size_distribution.len() + stats.versions_distribution.len()); + metrics.push(PrometheusMetric::from_descriptor( + &USAGE_SINCE_LAST_UPDATE_SECONDS_MD, + stats.since_last_update_seconds as f64, + )); metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64)); metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64)); metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64)); @@ -73,6 +82,7 @@ pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec Vec, now: SystemTime) -> u64 { + last_update + .and_then(|updated_at| now.duration_since(updated_at).ok()) + .unwrap_or(Duration::ZERO) + .as_secs() +} + /// Collects per-bucket usage metrics from the given stats. /// /// Returns a vector of Prometheus metrics for bucket usage. @@ -153,10 +170,12 @@ mod tests { #[test] fn test_collect_cluster_usage_metrics() { let stats = ClusterUsageStats { + since_last_update_seconds: 45, total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB objects_count: 10000, versions_count: 15000, delete_markers_count: 500, + buckets_count: 8, object_size_distribution: vec![ ("0-1KB".to_string(), 5000), ("1KB-1MB".to_string(), 3000), @@ -169,12 +188,16 @@ mod tests { let metrics = collect_cluster_usage_metrics(&stats); report_metrics(&metrics); - // 4 base metrics + 4 size distribution + 3 version distribution = 11 - assert_eq!(metrics.len(), 11); + // 6 base metrics + 4 size distribution + 3 version distribution = 13 + assert_eq!(metrics.len(), 13); let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name(); let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name); assert!(total_bytes.is_some()); + + let stale_name = USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(); + let stale = metrics.iter().find(|m| m.name == stale_name && m.value == 45.0); + assert!(stale.is_some()); } #[test] @@ -196,4 +219,23 @@ mod tests { // 5 base metrics + 1 size distribution + 1 version distribution = 7 assert_eq!(metrics.len(), 7); } + + #[test] + fn usage_since_last_update_seconds_reports_elapsed_time() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(120); + let last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(75)); + + assert_eq!(usage_since_last_update_seconds(last_update, now), 45); + } + + #[test] + fn usage_since_last_update_seconds_returns_zero_for_missing_or_future_timestamps() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(120); + + assert_eq!(usage_since_last_update_seconds(None, now), 0); + assert_eq!( + usage_since_last_update_seconds(Some(SystemTime::UNIX_EPOCH + Duration::from_secs(180)), now), + 0 + ); + } } diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 9af8f611d..9782c5fff 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -38,7 +38,7 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{ScannerMetricsReport, global_metrics}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system}; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, sync::Arc, time::SystemTime}; use sysinfo::{Networks, System}; use tracing::{instrument, warn}; @@ -49,6 +49,7 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state"; type ObsStorageInfo = ::StorageInfo; struct ObsDataUsageInfo { + last_update: Option, buckets_count: u64, objects_total_count: u64, versions_total_count: u64, @@ -82,6 +83,7 @@ async fn load_obs_data_usage_from_backend(store: Arc) -> ObsEcstoreRes let data_usage = obs_load_data_usage_from_backend(store).await?; Ok(ObsDataUsageInfo { + last_update: data_usage.last_update, buckets_count: data_usage.buckets_count, objects_total_count: data_usage.objects_total_count, versions_total_count: data_usage.versions_total_count, @@ -895,10 +897,15 @@ pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats, Some(( ClusterUsageStats { + since_last_update_seconds: crate::metrics::collectors::cluster_usage::usage_since_last_update_seconds( + data_usage.last_update, + SystemTime::now(), + ), total_bytes: data_usage.objects_total_size, objects_count: data_usage.objects_total_count, versions_count: data_usage.versions_total_count, delete_markers_count: data_usage.delete_markers_total_count, + buckets_count: data_usage.buckets_count, object_size_distribution: data_usage .buckets_usage .values()