gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): add resync metrics (#4408)

Browse Source
This commit is contained in:
Zhengchao An
2026-07-08 15:01:28 +08:00
committed by GitHub
parent 1e6207c08e
commit 9a7255540b
7 changed files with 295 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/bucket_replication.rs
+50 -4
View File
@@ -24,12 +24,14 @@ use crate::metrics::schema::bucket_replication::{
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD,
BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
};
use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 20;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
@@ -70,6 +72,11 @@ pub struct BucketReplicationStats {
pub proxied_get_tagging_requests_failures: u64,
pub proxied_delete_tagging_requests_total: u64,
pub proxied_delete_tagging_requests_failures: u64,
pub resync_started_count: u64,
pub resync_completed_count: u64,
pub resync_failed_count: u64,
pub resync_canceled_count: u64,
pub resync_duration_ms: u64,
pub targets: Vec<BucketReplicationTargetStats>,
}
@@ -222,6 +229,26 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, stat.resync_started_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, stat.resync_completed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, stat.resync_failed_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, stat.resync_canceled_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
for target in &stat.targets {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
@@ -266,6 +293,11 @@ mod tests {
proxied_get_tagging_requests_failures: 0,
proxied_delete_tagging_requests_total: 1,
proxied_delete_tagging_requests_failures: 1,
resync_started_count: 2,
resync_completed_count: 1,
resync_failed_count: 1,
resync_canceled_count: 0,
resync_duration_ms: 1500,
targets: vec![BucketReplicationTargetStats {
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
bandwidth_limit_bytes_per_sec: 2048,
@@ -275,7 +307,7 @@ mod tests {
}];
let metrics = collect_bucket_replication_metrics(&stats);
assert_eq!(metrics.len(), 21);
assert_eq!(metrics.len(), 26);
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
@@ -321,6 +353,20 @@ mod tests {
&& metric.value == 1.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let resync_started_name = BUCKET_REPL_RESYNC_STARTED_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == resync_started_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let resync_duration_name = BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == resync_duration_name
&& metric.value == 1500.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
}
#[test]
crates/obs/src/metrics/schema/bucket_replication.rs
+50
View File
@@ -28,6 +28,11 @@ pub const RANGE_L: &str = "range";
const PROXIED_PUT_REQUESTS_TOTAL: &str = "proxied_put_requests_total";
const PROXIED_PUT_REQUESTS_FAILURES: &str = "proxied_put_requests_failures";
const RESYNC_STARTED_TOTAL: &str = "resync_started_total";
const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -191,6 +196,51 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_STARTED_TOTAL),
"Total number of bucket replication resync runs started",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_COMPLETED_TOTAL),
"Total number of bucket replication resync runs completed",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_FAILED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_FAILED_TOTAL),
"Total number of bucket replication resync runs failed",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_CANCELED_TOTAL),
"Total number of bucket replication resync runs canceled",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(RESYNC_DURATION_MS_TOTAL),
"Total elapsed time of finished bucket replication resync runs in milliseconds",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::TotalFailedBytes,
crates/obs/src/metrics/stats_collector.rs
+5
View File
@@ -174,6 +174,11 @@ async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets: stats
.targets
.into_iter()
crates/obs/src/metrics/storage_api.rs
+10
View File
@@ -63,6 +63,11 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) proxied_get_tagging_requests_failures: u64,
pub(crate) proxied_delete_tagging_requests_total: u64,
pub(crate) proxied_delete_tagging_requests_failures: u64,
pub(crate) resync_started_count: u64,
pub(crate) resync_completed_count: u64,
pub(crate) resync_failed_count: u64,
pub(crate) resync_canceled_count: u64,
pub(crate) resync_duration_ms: u64,
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
}
@@ -156,6 +161,11 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
targets,
});
}