gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 19:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): harden backlog observability (#5564)

Browse Source 
Add RAII guards for replication runtime backlog tickets so active worker and queue counters unwind on every terminal path.

Expose node-local MRF pending, dropped, missed, and flush-failure metrics through the bucket replication Prometheus collector while keeping the existing current backlog and durable MRF gauges additive.

Update durable MRF summary maintenance to aggregate incrementally during the persister loop, avoiding repeated full-entry scans on each successful flush.

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-08-01 19:12:58 +08:00
committed by GitHub
parent b7805caa58
commit da389c0e21
7 changed files with 1024 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/metrics/schema/bucket_replication.rs
+60
View File
@@ -38,6 +38,12 @@ const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
const MRF_PENDING_COUNT: &str = "mrf_pending_count";
const MRF_PENDING_BYTES: &str = "mrf_pending_bytes";
const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
const MRF_MISSED_COUNT: &str = "mrf_missed_count";
const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures";
const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis";
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -129,6 +135,60 @@ pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor>
)
});
pub static BUCKET_REPL_MRF_PENDING_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(MRF_PENDING_COUNT),
"Current number of MRF entries waiting to be flushed to the durable recovery file for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_PENDING_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(MRF_PENDING_BYTES),
"Current bytes represented by MRF entries waiting to be flushed to the durable recovery file for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_DROPPED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(MRF_DROPPED_COUNT),
"Total number of MRF entries dropped after the bounded pending backlog cap was reached for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_MISSED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(MRF_MISSED_COUNT),
"Total number of MRF entries that could not be admitted to the save channel for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_FLUSH_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::from(MRF_FLUSH_FAILURES),
"Total number of durable MRF flush failures observed while a bucket had pending MRF entries on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_MRF_LAST_FLUSH_DURATION_MILLIS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(MRF_LAST_FLUSH_DURATION_MILLIS),
"Duration in milliseconds of the last durable MRF flush that touched pending entries for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ProxiedDeleteTaggingRequestsTotal,