gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 10:43:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Expose replication backlog gauges (#5557)

Browse Source 
* fix(replication): count backlog at queue admission

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): expose bucket replication backlog gauges

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): report recent backlog from queued work

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): preserve legacy backlog metric semantics

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(obs): expose durable MRF backlog gauges

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(obs): cover replication backlog metric scope

Co-Authored-By: heihutu <heihutu@gmail.com>

* fix(obs): keep backlog metrics API-compatible

Co-Authored-By: heihutu <heihutu@gmail.com>

* refactor(obs): streamline replication backlog metrics

Keep MRF backlog accounting and OBS metric collection on a single, cheaper path.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(kms): update aws capability snapshot

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-01 16:48:45 +08:00
committed by GitHub
parent 8218248000
commit 62d44d10b8
13 changed files with 1025 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/replication/src/stats.rs
+47 -4
View File
@@ -241,6 +241,16 @@ impl InQueueStats {
Self::default()
}
pub fn add_current(&self, bytes: i64, count: i64) {
self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
}
pub fn subtract_current(&self, bytes: i64, count: i64) {
saturating_atomic_sub(&self.now_bytes, bytes.max(0));
saturating_atomic_sub(&self.now_count, count.max(0));
}
pub fn get_current_bytes(&self) -> i64 {
self.now_bytes.load(Ordering::Relaxed)
}
@@ -250,6 +260,14 @@ impl InQueueStats {
}
}
fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
if delta == 0 {
return;
}
let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct InQueueMetric {
pub curr: InQueueStats,
@@ -359,6 +377,18 @@ impl QueueCache {
Self::default()
}
pub fn inc(&mut self, bucket: &str, size: i64) {
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
stats.curr.add_current(size, 1);
self.sr_queue_stats.curr.add_current(size, 1);
}
pub fn dec(&mut self, bucket: &str, size: i64) {
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
stats.curr.subtract_current(size, 1);
self.sr_queue_stats.curr.subtract_current(size, 1);
}
pub fn update(&mut self) {
let observed_at = Instant::now();
self.sr_queue_stats.observe(observed_at);
@@ -808,12 +838,10 @@ mod tests {
#[test]
fn in_queue_metric_observe_updates_rolling_stats() {
let mut metric = InQueueMetric::default();
metric.curr.now_bytes.store(128, Ordering::Relaxed);
metric.curr.now_count.store(4, Ordering::Relaxed);
metric.curr.add_current(128, 4);
metric.observe(Instant::now());
metric.curr.now_bytes.store(256, Ordering::Relaxed);
metric.curr.now_count.store(6, Ordering::Relaxed);
metric.curr.add_current(128, 2);
metric.observe(Instant::now());
assert_eq!(metric.curr.bytes, 256);
@@ -824,6 +852,21 @@ mod tests {
assert_eq!(metric.last_minute.count, 5);
}
#[test]
fn queue_cache_decrement_saturates_at_zero() {
let mut cache = QueueCache::new();
cache.inc("bucket", 128);
cache.dec("bucket", 512);
cache.dec("bucket", 1);
let bucket = cache.get_bucket_stats("bucket");
let site = cache.get_site_stats();
assert_eq!(bucket.curr.count, 0);
assert_eq!(bucket.curr.bytes, 0);
assert_eq!(site.curr.count, 0);
assert_eq!(site.curr.bytes, 0);
}
#[test]
fn fail_stats_recent_since_tracks_windows() {
let mut stats = FailStats::default();