gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 07:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose target-scoped durable MRF backlog metrics (#5584)

Browse Source 
* feat(replication): expose target durable mrf backlog

Add target ARN attribution to durable MRF entries and surface target-scoped durable backlog metrics without changing existing bucket-only metric labels.

Keep legacy MRF files bucket-only by defaulting missing targetARNs to an empty list, and expose target snapshots through an additive API so existing DurableMrfBacklogSummary callers remain source-compatible.

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(replication): expose runtime target backlog (#5586)

Track runtime replication backlog by target ARN for regular, large, delete, and MRF admission paths while preserving the existing bucket-level backlog semantics.

Add target-scoped current backlog metrics and merge them with durable target backlog snapshots for observability.

Co-authored-by: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-02 01:31:50 +08:00
committed by GitHub
parent 4c83bff81c
commit fbec33bd29
14 changed files with 1241 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/collectors/bucket_replication.rs
+112 -4
View File
@@ -17,8 +17,10 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD, BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD,
BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD, BUCKET_REPL_MRF_PENDING_COUNT_MD,
@@ -37,6 +39,7 @@ use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
@@ -99,6 +102,16 @@ pub(crate) struct BucketReplicationBacklogStats {
pub(crate) mrf_missed_count: u64,
pub(crate) mrf_flush_failures: u64,
pub(crate) mrf_last_flush_duration_millis: u64,
pub(crate) target_backlogs: Vec<BucketReplicationTargetBacklogStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationTargetBacklogStats {
pub(crate) target_arn: String,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
}
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
@@ -290,7 +303,14 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
let metric_count = stats
.iter()
.map(|stat| {
BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET
+ stat.target_backlogs.len() * BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET
})
.sum();
let mut metrics = Vec::with_capacity(metric_count);
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
@@ -345,6 +365,42 @@ pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicat
)
.with_label(BUCKET_L, bucket_label),
);
for target in &stat.target_backlogs {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD,
target.current_backlog_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
target.current_backlog_bytes as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD,
target.durable_mrf_backlog_count as f64,
)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
target.durable_mrf_backlog_bytes as f64,
)
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_label),
);
}
}
metrics
@@ -469,10 +525,17 @@ mod tests {
mrf_missed_count: 4,
mrf_flush_failures: 5,
mrf_last_flush_duration_millis: 6,
target_backlogs: vec![BucketReplicationTargetBacklogStats {
target_arn: "arn:rustfs:replication:target-a".to_string(),
current_backlog_count: 3,
current_backlog_bytes: 4096,
durable_mrf_backlog_count: 2,
durable_mrf_backlog_bytes: 2048,
}],
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
assert_eq!(metrics.len(), 11);
assert_eq!(metrics.len(), 15);
let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
@@ -509,6 +572,50 @@ mod tests {
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let target_count_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_count_name
&& metric.value == 3.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let target_bytes_name = BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == target_bytes_name
&& metric.value == 4096.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let durable_target_count_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_target_count_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let durable_target_bytes_name = BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_target_bytes_name
&& metric.value == 2048.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
&& metric
.labels
.iter()
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:target-a")
}));
let pending_count_name = BUCKET_REPL_MRF_PENDING_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == pending_count_name
@@ -574,6 +681,7 @@ mod tests {
mrf_missed_count: 4,
mrf_flush_failures: 5,
mrf_last_flush_duration_millis: 6,
target_backlogs: Vec::new(),
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
crates/obs/src/metrics/collectors/mod.rs
+3 -1
View File
@@ -42,7 +42,9 @@ pub mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
pub(crate) use bucket_replication::{
BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
crates/obs/src/metrics/scheduler.rs
+156 -2
View File
@@ -80,8 +80,10 @@ use crate::metrics::runtime_sources::bucket_monitor_available;
use crate::metrics::schema::audit::{AUDIT_FAILED_MESSAGES_MD, AUDIT_TARGET_QUEUE_LENGTH_MD, AUDIT_TOTAL_MESSAGES_MD};
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, BUCKET_REPL_MRF_DROPPED_COUNT_MD, BUCKET_REPL_MRF_FLUSH_FAILURES_MD,
BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD, BUCKET_REPL_MRF_MISSED_COUNT_MD, BUCKET_REPL_MRF_PENDING_BYTES_MD,
BUCKET_REPL_MRF_PENDING_COUNT_MD, TARGET_ARN_L,
};
@@ -633,6 +635,17 @@ fn repl_backlog_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<Bu
stats.iter().map(|s| s.bucket.clone()).collect()
}
fn repl_backlog_target_live_keys(stats: &[BucketReplicationBacklogStats]) -> HashSet<ReplBwKey> {
stats
.iter()
.flat_map(|stat| {
stat.target_backlogs
.iter()
.map(|target| (stat.bucket.clone(), target.target_arn.clone()))
})
.collect()
}
fn update_series_zero_tombstones<T: Clone + Eq + std::hash::Hash>(
has_seen_valid_snapshot: &mut bool,
prev_live_keys: &mut HashSet<T>,
@@ -1060,6 +1073,40 @@ fn collect_repl_backlog_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketK
collect_bucket_replication_backlog_metrics(&stats)
}
fn collect_repl_backlog_target_zero_tombstone_metrics(zero_tombstones: &HashMap<ReplBwKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
}
let mut zero_metrics = Vec::with_capacity(zero_tombstones.len() * 4);
for (bucket, target_arn) in zero_tombstones.keys() {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.clone());
let target_arn_label: Cow<'static, str> = Cow::Owned(target_arn.clone());
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD, 0.0)
.with_label(BUCKET_L, bucket_label.clone())
.with_label(TARGET_ARN_L, target_arn_label.clone()),
);
zero_metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD, 0.0)
.with_label(BUCKET_L, bucket_label)
.with_label(TARGET_ARN_L, target_arn_label),
);
}
zero_metrics
}
fn retire_repl_bw_metric_series(bucket: &str, target_arn: &str) -> usize {
let labels = [
(BUCKET_L, Cow::Owned(bucket.to_string())),
@@ -1089,6 +1136,17 @@ fn retire_repl_backlog_metric_series(bucket: &str) -> usize {
.sum()
}
fn retire_repl_backlog_target_metric_series(bucket: &str, target_arn: &str) -> usize {
let labels = [
(BUCKET_L, Cow::Owned(bucket.to_string())),
(TARGET_ARN_L, Cow::Owned(target_arn.to_string())),
];
retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(), &labels)
}
fn expire_repl_bw_zero_tombstones(monitor_available: bool, zero_tombstones: &mut HashMap<ReplBwKey, u8>) -> Vec<ReplBwKey> {
if !monitor_available {
return Vec::new();
@@ -1105,6 +1163,17 @@ fn expire_repl_backlog_zero_tombstones(monitor_available: bool, zero_tombstones:
expire_series_zero_tombstones(zero_tombstones)
}
fn expire_repl_backlog_target_zero_tombstones(
target_metrics_available: bool,
zero_tombstones: &mut HashMap<ReplBwKey, u8>,
) -> Vec<ReplBwKey> {
if !target_metrics_available {
return Vec::new();
}
expire_series_zero_tombstones(zero_tombstones)
}
/// Initialize all metrics collectors.
///
/// This function spawns background tasks that periodically collect metrics
@@ -1399,6 +1468,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut prev_backlog_live_keys: HashSet<BucketKey> = HashSet::new();
let mut backlog_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut has_seen_valid_backlog_snapshot = false;
let mut prev_backlog_target_live_keys: HashSet<ReplBwKey> = HashSet::new();
let mut backlog_target_zero_tombstones: HashMap<ReplBwKey, u8> = HashMap::new();
let mut has_seen_valid_backlog_target_snapshot = false;
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1429,6 +1501,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
let durable_mrf_available = bucket_replication_backlog.iter().any(|stat| stat.durable_mrf_available);
let backlog_target_metrics_available =
durable_mrf_available || monitor_available || !bucket_replication_backlog.is_empty();
update_repl_backlog_zero_tombstones(
monitor_available,
&mut has_seen_valid_backlog_snapshot,
@@ -1437,9 +1512,19 @@ pub fn init_metrics_runtime(token: CancellationToken) {
repl_backlog_live_keys(&bucket_replication_backlog),
repl_bw_zero_tombstone_cycles,
);
if backlog_target_metrics_available {
update_series_zero_tombstones(
&mut has_seen_valid_backlog_target_snapshot,
&mut prev_backlog_target_live_keys,
&mut backlog_target_zero_tombstones,
repl_backlog_target_live_keys(&bucket_replication_backlog),
repl_bw_zero_tombstone_cycles,
);
}
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
metrics.extend(collect_repl_backlog_zero_tombstone_metrics(&backlog_zero_tombstones));
metrics.extend(collect_repl_backlog_target_zero_tombstone_metrics(&backlog_target_zero_tombstones));
let replication = collect_replication_stats().await;
metrics.extend(collect_replication_metrics(&replication));
report_metrics(&metrics);
@@ -1451,6 +1536,12 @@ pub fn init_metrics_runtime(token: CancellationToken) {
for bucket in expire_repl_backlog_zero_tombstones(monitor_available, &mut backlog_zero_tombstones) {
let _ = retire_repl_backlog_metric_series(&bucket);
}
for (bucket, target_arn) in expire_repl_backlog_target_zero_tombstones(
backlog_target_metrics_available,
&mut backlog_target_zero_tombstones,
) {
let _ = retire_repl_backlog_target_metric_series(&bucket, &target_arn);
}
},
).await;
}
@@ -2259,6 +2350,69 @@ mod tests {
assert_eq!(prev_live_keys, bucket_keys(&["photos"]));
}
#[test]
fn repl_backlog_target_tombstones_zero_removed_targets_then_expire() {
let mut has_seen_valid_snapshot = false;
let mut prev_live_keys = HashSet::new();
let mut zero_tombstones = HashMap::new();
let key = repl_bw_key("photos", "arn:rustfs:replication:target-a");
update_series_zero_tombstones(
&mut has_seen_valid_snapshot,
&mut prev_live_keys,
&mut zero_tombstones,
repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]),
2,
);
assert!(has_seen_valid_snapshot);
assert_eq!(prev_live_keys, repl_bw_keys(&[("photos", "arn:rustfs:replication:target-a")]));
assert!(zero_tombstones.is_empty());
update_series_zero_tombstones(&mut has_seen_valid_snapshot, &mut prev_live_keys, &mut zero_tombstones, HashSet::new(), 2);
assert_eq!(zero_tombstones.get(&key), Some(&2));
let metrics = collect_repl_backlog_target_zero_tombstone_metrics(&zero_tombstones);
assert_eq!(metrics.len(), 4);
let expected_names = HashSet::from([
BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD.get_full_metric_name(),
]);
let mut actual_names = HashSet::new();
for metric in metrics {
actual_names.insert(metric.name.to_string());
assert_eq!(metric.value, 0.0);
let labels = metric
.labels
.into_iter()
.map(|(key, value)| (key, value.to_string()))
.collect::<HashMap<_, _>>();
assert_eq!(labels.get(BUCKET_L).map(String::as_str), Some("photos"));
assert_eq!(labels.get(TARGET_ARN_L).map(String::as_str), Some("arn:rustfs:replication:target-a"));
}
assert_eq!(actual_names, expected_names);
let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
assert!(expired.is_empty());
assert_eq!(zero_tombstones.get(&key), Some(&1));
let expired = expire_repl_backlog_target_zero_tombstones(true, &mut zero_tombstones);
assert_eq!(expired, vec![key]);
assert!(zero_tombstones.is_empty());
}
#[test]
fn repl_backlog_target_tombstones_do_not_advance_when_target_metrics_unavailable() {
let key = repl_bw_key("videos", "arn:rustfs:replication:target-b");
let mut zero_tombstones = HashMap::from([(key.clone(), 1)]);
let expired = expire_repl_backlog_target_zero_tombstones(false, &mut zero_tombstones);
assert!(expired.is_empty());
assert_eq!(zero_tombstones.get(&key), Some(&1));
}
#[test]
fn bucket_tombstones_zero_removed_buckets_then_expire() {
let mut has_seen_valid_snapshot = false;
crates/obs/src/metrics/schema/bucket_replication.rs
+40
View File
@@ -35,9 +35,13 @@ const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
const CURRENT_TARGET_BACKLOG_COUNT: &str = "current_target_backlog_count";
const CURRENT_TARGET_BACKLOG_BYTES: &str = "current_target_backlog_bytes";
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
const DURABLE_MRF_TARGET_BACKLOG_COUNT: &str = "durable_mrf_target_backlog_count";
const DURABLE_MRF_TARGET_BACKLOG_BYTES: &str = "durable_mrf_target_backlog_bytes";
const MRF_PENDING_COUNT: &str = "mrf_pending_count";
const MRF_PENDING_BYTES: &str = "mrf_pending_bytes";
const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
@@ -108,6 +112,24 @@ pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = La
)
});
pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_TARGET_BACKLOG_COUNT),
"Current number of target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_CURRENT_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_TARGET_BACKLOG_BYTES),
"Current bytes in target-scoped operations admitted to in-memory replication worker queues for a bucket and target ARN",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_AVAILABLE),
@@ -135,6 +157,24 @@ pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor>
)
});
pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_TARGET_BACKLOG_COUNT),
"Current number of target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_TARGET_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_TARGET_BACKLOG_BYTES),
"Current bytes in target-scoped operations in the durable MRF backlog file for a bucket and target ARN on this node",
&[BUCKET_L, TARGET_ARN_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_PENDING_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(MRF_PENDING_COUNT),
crates/obs/src/metrics/stats_collector.rs
+16 -5
View File
@@ -21,11 +21,11 @@
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::{
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
ScannerStats,
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetBacklogStats,
BucketReplicationTargetStats, BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats,
ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats,
HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats,
ResourceStats, ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
@@ -212,6 +212,17 @@ async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>,
mrf_missed_count: stats.mrf_missed_count,
mrf_flush_failures: stats.mrf_flush_failures,
mrf_last_flush_duration_millis: stats.mrf_last_flush_duration_millis,
target_backlogs: stats
.target_backlogs
.iter()
.map(|target| BucketReplicationTargetBacklogStats {
target_arn: target.target_arn.clone(),
current_backlog_count: target.current_backlog_count,
current_backlog_bytes: target.current_backlog_bytes,
durable_mrf_backlog_count: target.durable_mrf_backlog_count,
durable_mrf_backlog_bytes: target.durable_mrf_backlog_bytes,
})
.collect(),
});
detail_stats.push(bucket_replication_detail_from_snapshot(stats));
}
crates/obs/src/metrics/storage_api.rs
+161 -50
View File
@@ -18,7 +18,8 @@ use std::time::Duration;
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
use rustfs_ecstore::api::bucket::replication::{
DurableMrfBucketBacklog, MrfBucketBacklogObservability, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
DurableMrfBucketBacklog, DurableMrfTargetBacklog, MrfBucketBacklogObservability, RuntimeReplicationTargetBacklog,
durable_mrf_backlog_summary_snapshot, durable_mrf_target_backlog_snapshot, get_global_replication_stats,
mrf_backlog_observability_snapshot,
};
pub(crate) use rustfs_ecstore::api::capacity::{
@@ -44,6 +45,15 @@ pub(crate) struct ObsBucketReplicationTargetStatsSnapshot {
pub(crate) latency_ms: f64,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ObsBucketReplicationTargetBacklogSnapshot {
pub(crate) target_arn: String,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) bucket: String,
@@ -84,6 +94,7 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) mrf_flush_failures: u64,
pub(crate) mrf_last_flush_duration_millis: u64,
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
pub(crate) target_backlogs: Vec<ObsBucketReplicationTargetBacklogSnapshot>,
}
#[derive(Debug, Clone, Default, PartialEq)]
@@ -122,6 +133,15 @@ struct ObsBucketReplicationProxySnapshot {
proxied_delete_tagging_requests_failures: u64,
}
#[derive(Debug, Clone, Default, PartialEq)]
struct ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: bool,
durable_bucket: DurableMrfBucketBacklog,
runtime_targets: Vec<RuntimeReplicationTargetBacklog>,
durable_targets: Vec<DurableMrfTargetBacklog>,
mrf_observability: MrfBucketBacklogObservability,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub(crate) struct ObsReplicationSiteStatsSnapshot {
pub(crate) average_active_workers: f64,
@@ -157,10 +177,40 @@ fn bucket_replication_stats_snapshot_from_parts(
bucket: String,
runtime: ObsBucketReplicationRuntimeSnapshot,
proxy: ObsBucketReplicationProxySnapshot,
durable_mrf_available: bool,
durable_bucket: DurableMrfBucketBacklog,
mrf_observability: MrfBucketBacklogObservability,
backlog: ObsBucketReplicationBacklogSnapshot,
) -> ObsBucketReplicationStatsSnapshot {
let mut target_backlogs = HashMap::with_capacity(backlog.runtime_targets.len().saturating_add(backlog.durable_targets.len()));
for target in backlog.runtime_targets {
let entry =
target_backlogs
.entry(target.target_arn.clone())
.or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
target_arn: target.target_arn,
current_backlog_count: 0,
current_backlog_bytes: 0,
durable_mrf_backlog_count: 0,
durable_mrf_backlog_bytes: 0,
});
entry.current_backlog_count = target.count;
entry.current_backlog_bytes = target.bytes;
}
for target in backlog.durable_targets {
let entry =
target_backlogs
.entry(target.target_arn.clone())
.or_insert_with(|| ObsBucketReplicationTargetBacklogSnapshot {
target_arn: target.target_arn,
current_backlog_count: 0,
current_backlog_bytes: 0,
durable_mrf_backlog_count: 0,
durable_mrf_backlog_bytes: 0,
});
entry.durable_mrf_backlog_count = target.count;
entry.durable_mrf_backlog_bytes = target.bytes;
}
let mut target_backlogs = target_backlogs.into_values().collect::<Vec<_>>();
target_backlogs.sort_by(|left, right| left.target_arn.cmp(&right.target_arn));
ObsBucketReplicationStatsSnapshot {
bucket,
total_failed_bytes: runtime.total_failed_bytes,
@@ -190,16 +240,17 @@ fn bucket_replication_stats_snapshot_from_parts(
resync_duration_ms: runtime.resync_duration_ms,
current_backlog_count: runtime.current_backlog_count,
current_backlog_bytes: runtime.current_backlog_bytes,
durable_mrf_available,
durable_mrf_backlog_count: durable_bucket.count,
durable_mrf_backlog_bytes: durable_bucket.bytes,
mrf_pending_count: mrf_observability.pending_count,
mrf_pending_bytes: mrf_observability.pending_bytes,
mrf_dropped_count: mrf_observability.dropped_count,
mrf_missed_count: mrf_observability.missed_count,
mrf_flush_failures: mrf_observability.flush_failure_count,
mrf_last_flush_duration_millis: mrf_observability.last_flush_duration_millis,
durable_mrf_available: backlog.durable_mrf_available,
durable_mrf_backlog_count: backlog.durable_bucket.count,
durable_mrf_backlog_bytes: backlog.durable_bucket.bytes,
mrf_pending_count: backlog.mrf_observability.pending_count,
mrf_pending_bytes: backlog.mrf_observability.pending_bytes,
mrf_dropped_count: backlog.mrf_observability.dropped_count,
mrf_missed_count: backlog.mrf_observability.missed_count,
mrf_flush_failures: backlog.mrf_observability.flush_failure_count,
mrf_last_flush_duration_millis: backlog.mrf_observability.last_flush_duration_millis,
targets: runtime.targets,
target_backlogs,
}
}
@@ -222,6 +273,27 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
.into_iter()
.map(|bucket| (bucket.bucket.clone(), bucket))
.collect::<HashMap<String, DurableMrfBucketBacklog>>();
let mut durable_targets_by_bucket: HashMap<String, Vec<DurableMrfTargetBacklog>> = HashMap::new();
let durable_mrf_targets = if replication_storage_available && durable_mrf_available {
durable_mrf_target_backlog_snapshot()
} else {
Vec::new()
};
for target in durable_mrf_targets {
durable_targets_by_bucket
.entry(target.bucket.clone())
.or_default()
.push(target);
}
let mut runtime_targets_by_bucket: HashMap<String, Vec<RuntimeReplicationTargetBacklog>> = HashMap::new();
if let Some(stats) = &stats {
for target in stats.runtime_target_backlog_snapshot() {
runtime_targets_by_bucket
.entry(target.bucket.clone())
.or_default()
.push(target);
}
}
let mrf_observability = if replication_storage_available {
mrf_backlog_observability_snapshot()
} else {
@@ -236,7 +308,8 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
all_bucket_stats
.len()
.saturating_add(durable_buckets.len())
.saturating_add(mrf_observability_buckets.len()),
.saturating_add(mrf_observability_buckets.len())
.saturating_add(runtime_targets_by_bucket.len()),
);
bucket_names.extend(all_bucket_stats.keys().cloned());
bucket_names.extend(
@@ -251,6 +324,16 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
.filter(|bucket| !all_bucket_stats.contains_key(*bucket) && !durable_buckets.contains_key(*bucket))
.cloned(),
);
bucket_names.extend(
runtime_targets_by_bucket
.keys()
.filter(|bucket| {
!all_bucket_stats.contains_key(*bucket)
&& !durable_buckets.contains_key(*bucket)
&& !mrf_observability_buckets.contains_key(*bucket)
})
.cloned(),
);
let mut buckets = Vec::with_capacity(bucket_names.len());
for bucket in bucket_names {
@@ -327,14 +410,20 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
let runtime_targets = runtime_targets_by_bucket.remove(&bucket).unwrap_or_default();
let durable_targets = durable_targets_by_bucket.remove(&bucket).unwrap_or_default();
let mrf_observability = mrf_observability_buckets.get(&bucket).cloned().unwrap_or_default();
buckets.push(bucket_replication_stats_snapshot_from_parts(
bucket,
runtime,
proxy,
durable_mrf_available,
durable_bucket,
mrf_observability,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available,
durable_bucket,
runtime_targets,
durable_targets,
mrf_observability,
},
));
}
@@ -426,20 +515,34 @@ mod tests {
proxied_get_requests_failures: 1,
..Default::default()
},
true,
DurableMrfBucketBacklog {
bucket: "runtime-bucket".to_string(),
count: 5,
bytes: 8192,
},
MrfBucketBacklogObservability {
bucket: "runtime-bucket".to_string(),
pending_count: 1,
pending_bytes: 512,
dropped_count: 2,
missed_count: 3,
flush_failure_count: 4,
last_flush_duration_millis: 5,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
durable_bucket: DurableMrfBucketBacklog {
bucket: "runtime-bucket".to_string(),
count: 5,
bytes: 8192,
},
runtime_targets: vec![RuntimeReplicationTargetBacklog {
bucket: "runtime-bucket".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
count: 3,
bytes: 4096,
}],
durable_targets: vec![DurableMrfTargetBacklog {
bucket: "runtime-bucket".to_string(),
target_arn: "arn:rustfs:replication:target-a".to_string(),
count: 2,
bytes: 4096,
}],
mrf_observability: MrfBucketBacklogObservability {
bucket: "runtime-bucket".to_string(),
pending_count: 1,
pending_bytes: 512,
dropped_count: 2,
missed_count: 3,
flush_failure_count: 4,
last_flush_duration_millis: 5,
},
},
);
@@ -449,6 +552,12 @@ mod tests {
assert!(snapshot.durable_mrf_available);
assert_eq!(snapshot.durable_mrf_backlog_count, 5);
assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
assert_eq!(snapshot.target_backlogs.len(), 1);
assert_eq!(snapshot.target_backlogs[0].target_arn, "arn:rustfs:replication:target-a");
assert_eq!(snapshot.target_backlogs[0].current_backlog_count, 3);
assert_eq!(snapshot.target_backlogs[0].current_backlog_bytes, 4096);
assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_count, 2);
assert_eq!(snapshot.target_backlogs[0].durable_mrf_backlog_bytes, 4096);
assert_eq!(snapshot.mrf_pending_count, 1);
assert_eq!(snapshot.mrf_pending_bytes, 512);
assert_eq!(snapshot.mrf_dropped_count, 2);
@@ -466,13 +575,15 @@ mod tests {
"durable-only".to_string(),
ObsBucketReplicationRuntimeSnapshot::default(),
ObsBucketReplicationProxySnapshot::default(),
true,
DurableMrfBucketBacklog {
bucket: "durable-only".to_string(),
count: 11,
bytes: 2048,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
durable_bucket: DurableMrfBucketBacklog {
bucket: "durable-only".to_string(),
count: 11,
bytes: 2048,
},
..Default::default()
},
MrfBucketBacklogObservability::default(),
);
assert_eq!(snapshot.bucket, "durable-only");
@@ -488,16 +599,18 @@ mod tests {
"mrf-observability-only".to_string(),
ObsBucketReplicationRuntimeSnapshot::default(),
ObsBucketReplicationProxySnapshot::default(),
true,
DurableMrfBucketBacklog::default(),
MrfBucketBacklogObservability {
bucket: "mrf-observability-only".to_string(),
pending_count: 13,
pending_bytes: 4096,
dropped_count: 1,
missed_count: 2,
flush_failure_count: 3,
last_flush_duration_millis: 4,
ObsBucketReplicationBacklogSnapshot {
durable_mrf_available: true,
mrf_observability: MrfBucketBacklogObservability {
bucket: "mrf-observability-only".to_string(),
pending_count: 13,
pending_bytes: 4096,
dropped_count: 1,
missed_count: 2,
flush_failure_count: 3,
last_flush_duration_millis: 4,
},
..Default::default()
},
);
@@ -525,9 +638,7 @@ mod tests {
..Default::default()
},
ObsBucketReplicationProxySnapshot::default(),
false,
DurableMrfBucketBacklog::default(),
MrfBucketBacklogObservability::default(),
ObsBucketReplicationBacklogSnapshot::default(),
);
assert_eq!(snapshot.current_backlog_count, 1);