gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(obs): export cluster usage staleness (#4467)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-08 17:44:05 +08:00
committed by GitHub
parent 3430a7ead6
commit 718c051ff3
2 changed files with 53 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/cluster_usage.rs
+45 -3
View File
@@ -21,10 +21,13 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::cluster_usage::*;
use std::time::{Duration, SystemTime};
/// Cluster-wide usage statistics.
#[derive(Debug, Clone, Default)]
pub struct ClusterUsageStats {
/// Time since the persisted usage snapshot was last updated
pub since_last_update_seconds: u64,
/// Total bytes used in the cluster
pub total_bytes: u64,
/// Total number of objects
@@ -33,6 +36,8 @@ pub struct ClusterUsageStats {
pub versions_count: u64,
/// Total number of delete markers
pub delete_markers_count: u64,
/// Total number of buckets in the usage snapshot
pub buckets_count: u64,
/// Object size distribution by range
pub object_size_distribution: Vec<(String, u64)>,
/// Version count distribution by range
@@ -64,8 +69,12 @@ pub struct BucketUsageStats {
///
/// Returns a vector of Prometheus metrics for cluster usage.
pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
let mut metrics = Vec::with_capacity(4 + stats.object_size_distribution.len() + stats.versions_distribution.len());
let mut metrics = Vec::with_capacity(6 + stats.object_size_distribution.len() + stats.versions_distribution.len());
metrics.push(PrometheusMetric::from_descriptor(
&USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
stats.since_last_update_seconds as f64,
));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_TOTAL_BYTES_MD, stats.total_bytes as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_OBJECTS_COUNT_MD, stats.objects_count as f64));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_VERSIONS_COUNT_MD, stats.versions_count as f64));
@@ -73,6 +82,7 @@ pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<Prometheu
&USAGE_DELETE_MARKERS_COUNT_MD,
stats.delete_markers_count as f64,
));
metrics.push(PrometheusMetric::from_descriptor(&USAGE_BUCKETS_COUNT_MD, stats.buckets_count as f64));
// Object size distribution
for (range, count) in &stats.object_size_distribution {
@@ -93,6 +103,13 @@ pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<Prometheu
metrics
}
pub fn usage_since_last_update_seconds(last_update: Option<SystemTime>, now: SystemTime) -> u64 {
last_update
.and_then(|updated_at| now.duration_since(updated_at).ok())
.unwrap_or(Duration::ZERO)
.as_secs()
}
/// Collects per-bucket usage metrics from the given stats.
///
/// Returns a vector of Prometheus metrics for bucket usage.
@@ -153,10 +170,12 @@ mod tests {
#[test]
fn test_collect_cluster_usage_metrics() {
let stats = ClusterUsageStats {
since_last_update_seconds: 45,
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
objects_count: 10000,
versions_count: 15000,
delete_markers_count: 500,
buckets_count: 8,
object_size_distribution: vec![
("0-1KB".to_string(), 5000),
("1KB-1MB".to_string(), 3000),
@@ -169,12 +188,16 @@ mod tests {
let metrics = collect_cluster_usage_metrics(&stats);
report_metrics(&metrics);
// 4 base metrics + 4 size distribution + 3 version distribution = 11
assert_eq!(metrics.len(), 11);
// 6 base metrics + 4 size distribution + 3 version distribution = 13
assert_eq!(metrics.len(), 13);
let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
assert!(total_bytes.is_some());
let stale_name = USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name();
let stale = metrics.iter().find(|m| m.name == stale_name && m.value == 45.0);
assert!(stale.is_some());
}
#[test]
@@ -196,4 +219,23 @@ mod tests {
// 5 base metrics + 1 size distribution + 1 version distribution = 7
assert_eq!(metrics.len(), 7);
}
#[test]
fn usage_since_last_update_seconds_reports_elapsed_time() {
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(120);
let last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(75));
assert_eq!(usage_since_last_update_seconds(last_update, now), 45);
}
#[test]
fn usage_since_last_update_seconds_returns_zero_for_missing_or_future_timestamps() {
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(120);
assert_eq!(usage_since_last_update_seconds(None, now), 0);
assert_eq!(
usage_since_last_update_seconds(Some(SystemTime::UNIX_EPOCH + Duration::from_secs(180)), now),
0
);
}
}
crates/obs/src/metrics/stats_collector.rs
+8 -1
View File
@@ -38,7 +38,7 @@ use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{ScannerMetricsReport, global_metrics};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_io_metrics::{ProcessStatusSnapshot, snapshot_process_resource_and_system};
use std::{collections::HashMap, sync::Arc};
use std::{collections::HashMap, sync::Arc, time::SystemTime};
use sysinfo::{Networks, System};
use tracing::{instrument, warn};
@@ -49,6 +49,7 @@ const EVENT_METRICS_COLLECTOR_STATE: &str = "metrics_collector_state";
type ObsStorageInfo = <ObsStore as StorageAdminApi>::StorageInfo;
struct ObsDataUsageInfo {
last_update: Option<SystemTime>,
buckets_count: u64,
objects_total_count: u64,
versions_total_count: u64,
@@ -82,6 +83,7 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
let data_usage = obs_load_data_usage_from_backend(store).await?;
Ok(ObsDataUsageInfo {
last_update: data_usage.last_update,
buckets_count: data_usage.buckets_count,
objects_total_count: data_usage.objects_total_count,
versions_total_count: data_usage.versions_total_count,
@@ -895,10 +897,15 @@ pub async fn collect_cluster_usage_metric_stats() -> Option<(ClusterUsageStats,
Some((
ClusterUsageStats {
since_last_update_seconds: crate::metrics::collectors::cluster_usage::usage_since_last_update_seconds(
data_usage.last_update,
SystemTime::now(),
),
total_bytes: data_usage.objects_total_size,
objects_count: data_usage.objects_total_count,
versions_count: data_usage.versions_total_count,
delete_markers_count: data_usage.delete_markers_total_count,
buckets_count: data_usage.buckets_count,
object_size_distribution: data_usage
.buckets_usage
.values()