gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-03 18:55:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(obs): improve telemetry stack, replication metrics, and Grafana alignment (#2672)

Browse Source 
Co-authored-by: Filipe Monteiro <a22407332@alunos.ulht.pt>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: weisd <im@weisd.in>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
houseme
2026-04-24 21:50:17 +08:00
committed by GitHub
parent 2705e3f53b
commit 13b4500212
19 changed files with 4603 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/bucket_replication.rs
+243 -28
View File
@@ -12,34 +12,62 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket replication bandwidth metrics collector.
//!
//! Collects bandwidth metrics for bucket replication targets.
//!
//! This collector reuses the metric descriptors defined in `metrics_type::bucket_replication`
//! to avoid duplication of metric names, types, and help text.
//! Bucket replication metrics collector.
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::bucket_replication::{BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD};
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD,
BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
};
use std::borrow::Cow;
/// Bucket replication bandwidth statistics for metrics collection.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
pub target_arn: String,
pub bandwidth_limit_bytes_per_sec: u64,
pub current_bandwidth_bytes_per_sec: f64,
pub latency_ms: f64,
}
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationBandwidthStats {
/// Name of the bucket
pub bucket: String,
/// Target ARN for replication
pub target_arn: String,
/// Configured bandwidth limit in bytes per second
pub limit_bytes_per_sec: u64,
/// Current bandwidth in bytes per second (EWMA)
pub current_bandwidth_bytes_per_sec: f64,
}
/// Collects bucket replication bandwidth metrics from the provided statistics.
///
/// Uses the metric descriptors from `metrics_type::bucket_replication` module.
/// Returns a vector of Prometheus metrics for replication bandwidth.
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationStats {
pub bucket: String,
pub total_failed_bytes: u64,
pub total_failed_count: u64,
pub last_min_failed_bytes: u64,
pub last_min_failed_count: u64,
pub last_hour_failed_bytes: u64,
pub last_hour_failed_count: u64,
pub sent_bytes: u64,
pub sent_count: u64,
pub proxied_get_requests_total: u64,
pub proxied_get_requests_failures: u64,
pub proxied_head_requests_total: u64,
pub proxied_head_requests_failures: u64,
pub proxied_put_tagging_requests_total: u64,
pub proxied_put_tagging_requests_failures: u64,
pub proxied_get_tagging_requests_total: u64,
pub proxied_get_tagging_requests_failures: u64,
pub proxied_delete_tagging_requests_total: u64,
pub proxied_delete_tagging_requests_failures: u64,
pub targets: Vec<BucketReplicationTargetStats>,
}
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -52,24 +80,206 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_LIMIT_MD, stat.limit_bytes_per_sec as f64)
.with_label("bucket", bucket_label.clone())
.with_label("targetArn", target_arn_label.clone()),
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_BANDWIDTH_CURRENT_MD, stat.current_bandwidth_bytes_per_sec)
.with_label("bucket", bucket_label)
.with_label("targetArn", target_arn_label),
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_arn_label),
);
}
metrics
}
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::new();
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_BYTES_MD, stat.total_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_TOTAL_FAILED_COUNT_MD, stat.total_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, stat.last_min_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, stat.last_min_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, stat.last_hour_failed_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, stat.last_hour_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_BYTES_MD, stat.sent_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_SENT_COUNT_MD, stat.sent_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, stat.proxied_get_requests_total as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
stat.proxied_get_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
stat.proxied_head_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
stat.proxied_head_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_put_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_put_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_get_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_get_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
stat.proxied_delete_tagging_requests_total as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
stat.proxied_delete_tagging_requests_failures as f64,
)
.with_label(BUCKET_L, bucket_label.clone()),
);
for target in &stat.targets {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_LATENCY_MS_MD, target.latency_ms)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(OPERATION_L, Cow::Borrowed("object_replication"))
.with_label(RANGE_L, Cow::Borrowed("all"))
.with_label(TARGET_ARN_L, target_label),
);
}
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collect_bucket_replication_metrics() {
let stats = vec![BucketReplicationStats {
bucket: "b1".to_string(),
total_failed_bytes: 64,
total_failed_count: 2,
last_min_failed_bytes: 32,
last_min_failed_count: 1,
last_hour_failed_bytes: 64,
last_hour_failed_count: 2,
sent_bytes: 1024,
sent_count: 8,
proxied_get_requests_total: 5,
proxied_get_requests_failures: 1,
proxied_head_requests_total: 4,
proxied_head_requests_failures: 0,
proxied_put_tagging_requests_total: 3,
proxied_put_tagging_requests_failures: 1,
proxied_get_tagging_requests_total: 2,
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 0,
targets: vec![BucketReplicationTargetStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
current_bandwidth_bytes_per_sec: 1024.0,
latency_ms: 15.0,
}],
}];
let metrics = collect_bucket_replication_metrics(&stats);
assert_eq!(metrics.len(), 19);
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == sent_name
&& metric.value == 8.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == latency_name
&& metric.value == 15.0
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
}));
}
#[test]
fn test_collect_bucket_replication_metrics_empty() {
let stats: Vec<BucketReplicationStats> = Vec::new();
let metrics = collect_bucket_replication_metrics(&stats);
assert!(metrics.is_empty());
}
#[test]
fn test_collect_bucket_replication_bandwidth_metrics() {
let stats = vec![BucketReplicationBandwidthStats {
@@ -83,24 +293,29 @@ mod tests {
assert_eq!(metrics.len(), 2);
let limit_metric_name = BUCKET_REPL_BANDWIDTH_LIMIT_MD.get_full_metric_name();
let limit_metric = metrics.iter().find(|m| {
m.name == limit_metric_name && m.value == 1_048_576.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
let limit_metric = metrics.iter().find(|metric| {
metric.name == limit_metric_name
&& metric.value == 1_048_576.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
});
assert!(limit_metric.is_some());
assert!(
limit_metric
.and_then(|m| {
m.labels
.and_then(|metric| {
metric
.labels
.iter()
.find(|(k, _)| *k == "targetArn")
.map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
.find(|(key, _)| *key == TARGET_ARN_L)
.map(|(_, value)| value.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
})
.unwrap_or(false)
);
let current_metric_name = BUCKET_REPL_BANDWIDTH_CURRENT_MD.get_full_metric_name();
let current_metric = metrics.iter().find(|m| {
m.name == current_metric_name && m.value == 204_800.0 && m.labels.iter().any(|(k, v)| *k == "bucket" && v == "b1")
let current_metric = metrics.iter().find(|metric| {
metric.name == current_metric_name
&& metric.value == 204_800.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
});
assert!(current_metric.is_some());
}
crates/obs/src/metrics/collectors/mod.rs
+4 -1
View File
@@ -41,7 +41,10 @@ pub mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
};
pub use cluster::{ClusterStats, collect_cluster_metrics};
pub use cluster_config::{ClusterConfigStats, collect_cluster_config_metrics};
pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
crates/obs/src/metrics/collectors/replication.rs
+22 -2
View File
@@ -25,6 +25,14 @@ use crate::metrics::schema::replication::*;
/// Replication statistics.
#[derive(Debug, Clone, Default)]
pub struct ReplicationStats {
/// Average number of active replication workers
pub average_active_workers: f64,
/// Average queued bytes since server start
pub average_queued_bytes: i64,
/// Average queued objects since server start
pub average_queued_count: i64,
/// Average data transfer rate in bytes/sec
pub average_data_transfer_rate: f64,
/// Number of active replication workers
pub active_workers: u64,
/// Current data transfer rate in bytes/sec
@@ -50,6 +58,10 @@ pub struct ReplicationStats {
/// Returns a vector of Prometheus metrics for replication statistics.
pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMetric> {
vec![
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_MD, stats.average_active_workers),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_MD, stats.average_queued_bytes as f64),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_MD, stats.average_queued_count as f64),
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD, stats.average_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_MD, stats.active_workers as f64),
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD, stats.current_data_transfer_rate),
PrometheusMetric::from_descriptor(&REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD, stats.last_minute_queued_bytes as f64),
@@ -70,6 +82,10 @@ mod tests {
#[test]
fn test_collect_replication_metrics() {
let stats = ReplicationStats {
average_active_workers: 8.5,
average_queued_bytes: 1024 * 1024 * 40,
average_queued_count: 240,
average_data_transfer_rate: 1024.0 * 1024.0 * 3.0,
active_workers: 10,
current_data_transfer_rate: 1024.0 * 1024.0 * 5.0, // 5 MB/s
last_minute_queued_bytes: 1024 * 1024 * 100, // 100 MB
@@ -84,13 +100,17 @@ mod tests {
let metrics = collect_replication_metrics(&stats);
report_metrics(&metrics);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 13);
// Verify active workers
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
let active = metrics.iter().find(|m| m.name == active_name);
assert!(active.is_some());
assert_eq!(active.map(|m| m.value), Some(10.0));
let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
assert_eq!(avg_active.map(|m| m.value), Some(8.5));
}
#[test]
@@ -98,7 +118,7 @@ mod tests {
let stats = ReplicationStats::default();
let metrics = collect_replication_metrics(&stats);
assert_eq!(metrics.len(), 9);
assert_eq!(metrics.len(), 13);
for metric in &metrics {
assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty());