Expose replication backlog gauges (#5557)

* fix(replication): count backlog at queue admission Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): expose bucket replication backlog gauges Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): report recent backlog from queued work Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): preserve legacy backlog metric semantics Co-Authored-By: heihutu <heihutu@gmail.com> * feat(obs): expose durable MRF backlog gauges Co-Authored-By: heihutu <heihutu@gmail.com> * test(obs): cover replication backlog metric scope Co-Authored-By: heihutu <heihutu@gmail.com> * fix(obs): keep backlog metrics API-compatible Co-Authored-By: heihutu <heihutu@gmail.com> * refactor(obs): streamline replication backlog metrics Keep MRF backlog accounting and OBS metric collection on a single, cheaper path. Co-Authored-By: heihutu <heihutu@gmail.com> * test(kms): update aws capability snapshot Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-20 03:22:18 +00:00 · 2026-08-01 16:48:45 +08:00
parent 8218248000
commit 62d44d10b8
13 changed files with 1025 additions and 244 deletions
@@ -16,22 +16,25 @@

 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::bucket_replication::{
-    BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
-    BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
-    BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
-    BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
-    BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
-    BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
-    BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
+    BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
+    BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
+    BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
+    BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
+    BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
+    BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
+    BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
+    BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
+    TARGET_ARN_L,
 };
 use std::borrow::Cow;

 const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
+const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 5;

 #[derive(Debug, Clone, Default)]
 pub struct BucketReplicationTargetStats {
@@ -80,6 +83,16 @@ pub struct BucketReplicationStats {
    pub targets: Vec<BucketReplicationTargetStats>,
 }

+#[derive(Debug, Clone, Default)]
+pub(crate) struct BucketReplicationBacklogStats {
+    pub(crate) bucket: String,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_available: bool,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
+}
+
 pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
    if stats.is_empty() {
        return Vec::new();
@@ -249,7 +262,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
            PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
                .with_label(BUCKET_L, bucket_label.clone()),
        );
-
        for target in &stat.targets {
            let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
            metrics.push(
@@ -265,6 +277,43 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
    metrics
 }

+pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
+    for stat in stats {
+        let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, stat.current_backlog_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, stat.current_backlog_bytes as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(
+                &BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD,
+                if stat.durable_mrf_available { 1.0 } else { 0.0 },
+            )
+            .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, stat.durable_mrf_backlog_count as f64)
+                .with_label(BUCKET_L, bucket_label.clone()),
+        );
+        metrics.push(
+            PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, stat.durable_mrf_backlog_bytes as f64)
+                .with_label(BUCKET_L, bucket_label),
+        );
+    }
+
+    metrics
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -369,6 +418,56 @@ mod tests {
        }));
    }

+    #[test]
+    fn test_collect_bucket_replication_backlog_metrics() {
+        let stats = vec![BucketReplicationBacklogStats {
+            bucket: "b1".to_string(),
+            current_backlog_count: 3,
+            current_backlog_bytes: 4096,
+            durable_mrf_available: true,
+            durable_mrf_backlog_count: 2,
+            durable_mrf_backlog_bytes: 2048,
+        }];
+
+        let metrics = collect_bucket_replication_backlog_metrics(&stats);
+        assert_eq!(metrics.len(), 5);
+
+        let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == backlog_count_name
+                && metric.value == 3.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let backlog_bytes_name = BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == backlog_bytes_name
+                && metric.value == 4096.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_available_name = BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_available_name
+                && metric.value == 1.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_count_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_count_name
+                && metric.value == 2.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+
+        let durable_bytes_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|metric| {
+            metric.name == durable_bytes_name
+                && metric.value == 2048.0
+                && metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
+        }));
+    }
+
    #[test]
    fn test_collect_bucket_replication_metrics_empty() {
        let stats: Vec<BucketReplicationStats> = Vec::new();
@@ -376,6 +475,41 @@ mod tests {
        assert!(metrics.is_empty());
    }

+    #[test]
+    fn backlog_metrics_are_bucket_scoped_without_target_labels() {
+        let stats = vec![BucketReplicationBacklogStats {
+            bucket: "scope-bucket".to_string(),
+            current_backlog_count: 5,
+            current_backlog_bytes: 8192,
+            durable_mrf_available: true,
+            durable_mrf_backlog_count: 2,
+            durable_mrf_backlog_bytes: 4096,
+        }];
+
+        let metrics = collect_bucket_replication_backlog_metrics(&stats);
+        let backlog_names = [
+            BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name(),
+            BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name(),
+        ];
+
+        for name in backlog_names {
+            let metric = metrics
+                .iter()
+                .find(|metric| metric.name == name)
+                .expect("backlog metric should be emitted");
+            assert!(
+                metric
+                    .labels
+                    .iter()
+                    .any(|(key, value)| *key == BUCKET_L && value == "scope-bucket")
+            );
+            assert!(!metric.labels.iter().any(|(key, _)| *key == TARGET_ARN_L));
+        }
+    }
+
    #[test]
    fn test_collect_bucket_replication_bandwidth_metrics() {
        let stats = vec![BucketReplicationBandwidthStats {
@@ -42,6 +42,7 @@ pub mod system_process;

 pub use audit::{AuditTargetStats, collect_audit_metrics};
 pub use bucket::{BucketStats, collect_bucket_metrics};
+pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
 pub use bucket_replication::{
    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
    collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
@@ -31,9 +31,9 @@ pub use scheduler::{
    metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
 };
 pub(crate) use storage_api::metrics::{
-    BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, StorageAdminApi,
-    obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
-    obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
+    BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
+    StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
+    obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
    obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
    obs_resolve_object_store_handle, obs_transition_state_handle,
 };
@@ -35,6 +35,7 @@ use crate::metrics::collectors::{
    ProcessMemoryStats,
    collect_audit_metrics,
    collect_bucket_metrics,
+    collect_bucket_replication_backlog_metrics,
    collect_bucket_replication_bandwidth_metrics,
    collect_bucket_replication_metrics,
    collect_bucket_usage_metrics,
@@ -94,7 +95,7 @@ use crate::metrics::schema::notification_target::{
 };
 use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
 use crate::metrics::stats_collector::{
-    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
+    ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle,
    collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
    collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
    collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
@@ -1348,8 +1349,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
                            // Phase-1 action: force zero for removed keys during tombstone cycles.
                            metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));

-                            let bucket_replication = collect_bucket_replication_detail_stats().await;
+                            let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
                            metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
+                            metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
                            let replication = collect_replication_stats().await;
                            metrics.extend(collect_replication_metrics(&replication));
                            report_metrics(&metrics);
@@ -33,6 +33,11 @@ const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
 const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
 const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
 const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
+const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
+const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
+const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
+const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
+const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";

 pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
@@ -79,6 +84,51 @@ pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new
    )
 });

+pub static BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_BACKLOG_BYTES),
+        "Current number of bytes admitted to the in-memory replication worker queues for a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(CURRENT_BACKLOG_COUNT),
+        "Current number of objects admitted to the in-memory replication worker queues for a bucket",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_AVAILABLE),
+        "Whether the durable MRF backlog snapshot is available for this bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_BACKLOG_COUNT),
+        "Current number of objects in the durable MRF backlog file for a bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::from(DURABLE_MRF_BACKLOG_BYTES),
+        "Current bytes in the durable MRF backlog file for a bucket on this node",
+        &[BUCKET_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_counter_md(
        MetricName::ProxiedDeleteTaggingRequestsTotal,
@@ -128,7 +128,7 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
 pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
    new_gauge_md(
        MetricName::ReplicationRecentBacklogCount,
-        "Total number of objects currently in replication backlog (failed plus queued)",
+        "Legacy replication backlog indicator: failed target objects plus objects currently queued on this node",
        &[],
        subsystems::REPLICATION,
    )
@@ -21,17 +21,18 @@
 //! and convert them to the Stats structs used by collectors.

 use crate::metrics::collectors::{
-    BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
-    ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats,
-    DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats,
-    ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
+    BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
+    BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
+    CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
+    IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
+    ScannerStats,
 };
 use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
 use crate::metrics::{
-    BucketOperations, BucketOptions, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot,
-    obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
-    obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
-    obs_resolve_object_store_handle,
+    BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
+    obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
+    obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
+    obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
 };
 use crate::node_identity::current_local_node_identity;
 use chrono::Utc;
@@ -192,49 +193,65 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
    ilm_runtime_snapshot().await
 }

-async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
-    obs_bucket_replication_stats_snapshot()
-        .await
-        .into_iter()
-        .map(|stats| BucketReplicationStats {
-            bucket: stats.bucket,
-            total_failed_bytes: stats.total_failed_bytes,
-            total_failed_count: stats.total_failed_count,
-            last_min_failed_bytes: stats.last_min_failed_bytes,
-            last_min_failed_count: stats.last_min_failed_count,
-            last_hour_failed_bytes: stats.last_hour_failed_bytes,
-            last_hour_failed_count: stats.last_hour_failed_count,
-            sent_bytes: stats.sent_bytes,
-            sent_count: stats.sent_count,
-            proxied_get_requests_total: stats.proxied_get_requests_total,
-            proxied_get_requests_failures: stats.proxied_get_requests_failures,
-            proxied_head_requests_total: stats.proxied_head_requests_total,
-            proxied_head_requests_failures: stats.proxied_head_requests_failures,
-            proxied_put_requests_total: stats.proxied_put_requests_total,
-            proxied_put_requests_failures: stats.proxied_put_requests_failures,
-            proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
-            proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
-            proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
-            proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
-            proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
-            proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
-            resync_started_count: stats.resync_started_count,
-            resync_completed_count: stats.resync_completed_count,
-            resync_failed_count: stats.resync_failed_count,
-            resync_canceled_count: stats.resync_canceled_count,
-            resync_duration_ms: stats.resync_duration_ms,
-            targets: stats
-                .targets
-                .into_iter()
-                .map(|target| BucketReplicationTargetStats {
-                    target_arn: target.target_arn,
-                    bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
-                    current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
-                    latency_ms: target.latency_ms,
-                })
-                .collect(),
-        })
-        .collect()
+async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
+    let snapshots = obs_bucket_replication_stats_snapshot().await;
+    let mut detail_stats = Vec::with_capacity(snapshots.len());
+    let mut backlog_stats = Vec::with_capacity(snapshots.len());
+
+    for stats in snapshots {
+        backlog_stats.push(BucketReplicationBacklogStats {
+            bucket: stats.bucket.clone(),
+            current_backlog_count: stats.current_backlog_count,
+            current_backlog_bytes: stats.current_backlog_bytes,
+            durable_mrf_available: stats.durable_mrf_available,
+            durable_mrf_backlog_count: stats.durable_mrf_backlog_count,
+            durable_mrf_backlog_bytes: stats.durable_mrf_backlog_bytes,
+        });
+        detail_stats.push(bucket_replication_detail_from_snapshot(stats));
+    }
+
+    (detail_stats, backlog_stats)
+}
+
+fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
+    BucketReplicationStats {
+        bucket: stats.bucket,
+        total_failed_bytes: stats.total_failed_bytes,
+        total_failed_count: stats.total_failed_count,
+        last_min_failed_bytes: stats.last_min_failed_bytes,
+        last_min_failed_count: stats.last_min_failed_count,
+        last_hour_failed_bytes: stats.last_hour_failed_bytes,
+        last_hour_failed_count: stats.last_hour_failed_count,
+        sent_bytes: stats.sent_bytes,
+        sent_count: stats.sent_count,
+        proxied_get_requests_total: stats.proxied_get_requests_total,
+        proxied_get_requests_failures: stats.proxied_get_requests_failures,
+        proxied_head_requests_total: stats.proxied_head_requests_total,
+        proxied_head_requests_failures: stats.proxied_head_requests_failures,
+        proxied_put_requests_total: stats.proxied_put_requests_total,
+        proxied_put_requests_failures: stats.proxied_put_requests_failures,
+        proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
+        proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
+        proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
+        proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
+        proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
+        proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
+        resync_started_count: stats.resync_started_count,
+        resync_completed_count: stats.resync_completed_count,
+        resync_failed_count: stats.resync_failed_count,
+        resync_canceled_count: stats.resync_canceled_count,
+        resync_duration_ms: stats.resync_duration_ms,
+        targets: stats
+            .targets
+            .into_iter()
+            .map(|target| BucketReplicationTargetStats {
+                target_arn: target.target_arn,
+                bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
+                current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
+                latency_ms: target.latency_ms,
+            })
+            .collect(),
+    }
 }

 async fn obs_site_replication_stats() -> ReplicationStats {
@@ -526,7 +543,16 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand

 /// Collect bucket and target level replication stats from the global replication runtime.
 pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
-    obs_bucket_replication_detail_stats().await
+    obs_bucket_replication_stats_snapshot()
+        .await
+        .into_iter()
+        .map(bucket_replication_detail_from_snapshot)
+        .collect()
+}
+
+pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
+{
+    obs_bucket_replication_stats_bundle().await
 }

 /// Collect site-level replication stats from the global replication runtime.
@@ -12,11 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use std::collections::HashMap;
 use std::time::Duration;

 pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
-use rustfs_ecstore::api::bucket::replication::get_global_replication_stats;
+use rustfs_ecstore::api::bucket::replication::{
+    DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
+};
 pub(crate) use rustfs_ecstore::api::capacity::{
    get_total_usable_capacity as obs_get_total_usable_capacity,
    get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
@@ -68,9 +71,50 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
    pub(crate) resync_failed_count: u64,
    pub(crate) resync_canceled_count: u64,
    pub(crate) resync_duration_ms: u64,
+    pub(crate) current_backlog_count: u64,
+    pub(crate) current_backlog_bytes: u64,
+    pub(crate) durable_mrf_available: bool,
+    pub(crate) durable_mrf_backlog_count: u64,
+    pub(crate) durable_mrf_backlog_bytes: u64,
    pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
 }

+#[derive(Debug, Clone, Default, PartialEq)]
+struct ObsBucketReplicationRuntimeSnapshot {
+    total_failed_bytes: u64,
+    total_failed_count: u64,
+    last_min_failed_bytes: u64,
+    last_min_failed_count: u64,
+    last_hour_failed_bytes: u64,
+    last_hour_failed_count: u64,
+    sent_bytes: u64,
+    sent_count: u64,
+    resync_started_count: u64,
+    resync_completed_count: u64,
+    resync_failed_count: u64,
+    resync_canceled_count: u64,
+    resync_duration_ms: u64,
+    current_backlog_count: u64,
+    current_backlog_bytes: u64,
+    targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
+}
+
+#[derive(Debug, Clone, Default, PartialEq)]
+struct ObsBucketReplicationProxySnapshot {
+    proxied_get_requests_total: u64,
+    proxied_get_requests_failures: u64,
+    proxied_head_requests_total: u64,
+    proxied_head_requests_failures: u64,
+    proxied_put_requests_total: u64,
+    proxied_put_requests_failures: u64,
+    proxied_put_tagging_requests_total: u64,
+    proxied_put_tagging_requests_failures: u64,
+    proxied_get_tagging_requests_total: u64,
+    proxied_get_tagging_requests_failures: u64,
+    proxied_delete_tagging_requests_total: u64,
+    proxied_delete_tagging_requests_failures: u64,
+}
+
 #[derive(Debug, Clone, Default, PartialEq)]
 pub(crate) struct ObsReplicationSiteStatsSnapshot {
    pub(crate) average_active_workers: f64,
@@ -102,59 +146,85 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
    failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
 }

+fn bucket_replication_stats_snapshot_from_parts(
+    bucket: String,
+    runtime: ObsBucketReplicationRuntimeSnapshot,
+    proxy: ObsBucketReplicationProxySnapshot,
+    durable_mrf_available: bool,
+    durable_bucket: DurableMrfBucketBacklog,
+) -> ObsBucketReplicationStatsSnapshot {
+    ObsBucketReplicationStatsSnapshot {
+        bucket,
+        total_failed_bytes: runtime.total_failed_bytes,
+        total_failed_count: runtime.total_failed_count,
+        last_min_failed_bytes: runtime.last_min_failed_bytes,
+        last_min_failed_count: runtime.last_min_failed_count,
+        last_hour_failed_bytes: runtime.last_hour_failed_bytes,
+        last_hour_failed_count: runtime.last_hour_failed_count,
+        sent_bytes: runtime.sent_bytes,
+        sent_count: runtime.sent_count,
+        proxied_get_requests_total: proxy.proxied_get_requests_total,
+        proxied_get_requests_failures: proxy.proxied_get_requests_failures,
+        proxied_head_requests_total: proxy.proxied_head_requests_total,
+        proxied_head_requests_failures: proxy.proxied_head_requests_failures,
+        proxied_put_requests_total: proxy.proxied_put_requests_total,
+        proxied_put_requests_failures: proxy.proxied_put_requests_failures,
+        proxied_put_tagging_requests_total: proxy.proxied_put_tagging_requests_total,
+        proxied_put_tagging_requests_failures: proxy.proxied_put_tagging_requests_failures,
+        proxied_get_tagging_requests_total: proxy.proxied_get_tagging_requests_total,
+        proxied_get_tagging_requests_failures: proxy.proxied_get_tagging_requests_failures,
+        proxied_delete_tagging_requests_total: proxy.proxied_delete_tagging_requests_total,
+        proxied_delete_tagging_requests_failures: proxy.proxied_delete_tagging_requests_failures,
+        resync_started_count: runtime.resync_started_count,
+        resync_completed_count: runtime.resync_completed_count,
+        resync_failed_count: runtime.resync_failed_count,
+        resync_canceled_count: runtime.resync_canceled_count,
+        resync_duration_ms: runtime.resync_duration_ms,
+        current_backlog_count: runtime.current_backlog_count,
+        current_backlog_bytes: runtime.current_backlog_bytes,
+        durable_mrf_available,
+        durable_mrf_backlog_count: durable_bucket.count,
+        durable_mrf_backlog_bytes: durable_bucket.bytes,
+        targets: runtime.targets,
+    }
+}
+
 pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketReplicationStatsSnapshot> {
-    let Some(stats) = get_global_replication_stats() else {
-        return Vec::new();
+    let stats = get_global_replication_stats();
+    let all_bucket_stats = if let Some(stats) = &stats {
+        stats.get_all().await
+    } else {
+        HashMap::new()
    };
+    let durable_mrf_summary = if obs_resolve_object_store_handle().is_some() {
+        durable_mrf_backlog_summary_snapshot()
+    } else {
+        Default::default()
+    };
+    let durable_mrf_available = durable_mrf_summary.available;
+    let durable_buckets = durable_mrf_summary
+        .buckets
+        .into_iter()
+        .map(|bucket| (bucket.bucket.clone(), bucket))
+        .collect::<HashMap<String, DurableMrfBucketBacklog>>();
+    let mut bucket_names = Vec::with_capacity(all_bucket_stats.len().saturating_add(durable_buckets.len()));
+    bucket_names.extend(all_bucket_stats.keys().cloned());
+    bucket_names.extend(
+        durable_buckets
+            .keys()
+            .filter(|bucket| !all_bucket_stats.contains_key(*bucket))
+            .cloned(),
+    );
+    let mut buckets = Vec::with_capacity(bucket_names.len());

-    let all_bucket_stats = stats.get_all().await;
-    let mut buckets = Vec::with_capacity(all_bucket_stats.len());
-
-    for (bucket, bucket_stats) in all_bucket_stats {
-        let proxy = stats.get_proxy_stats(&bucket).await;
-        let mut total_failed_bytes = 0u64;
-        let mut total_failed_count = 0u64;
-        let mut last_min_failed_bytes = 0u64;
-        let mut last_min_failed_count = 0u64;
-        let mut last_hour_failed_bytes = 0u64;
-        let mut last_hour_failed_count = 0u64;
-        let mut sent_bytes = 0u64;
-        let mut sent_count = 0u64;
-        let mut targets = Vec::with_capacity(bucket_stats.stats.len());
-
-        for (target_arn, target_stats) in bucket_stats.stats {
-            total_failed_bytes = total_failed_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
-            total_failed_count = total_failed_count.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
-
-            let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
-            last_min_failed_bytes = last_min_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_min.size));
-            last_min_failed_count = last_min_failed_count.saturating_add(i64_to_u64_floor_zero(last_min.count));
-
-            let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
-            last_hour_failed_bytes = last_hour_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_hour.size));
-            last_hour_failed_count = last_hour_failed_count.saturating_add(i64_to_u64_floor_zero(last_hour.count));
-
-            sent_bytes = sent_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
-            sent_count = sent_count.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
-
-            targets.push(ObsBucketReplicationTargetStatsSnapshot {
-                target_arn,
-                bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
-                current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
-                latency_ms: target_stats.latency.curr,
-            });
-        }
-
-        buckets.push(ObsBucketReplicationStatsSnapshot {
-            bucket,
-            total_failed_bytes,
-            total_failed_count,
-            last_min_failed_bytes,
-            last_min_failed_count,
-            last_hour_failed_bytes,
-            last_hour_failed_count,
-            sent_bytes,
-            sent_count,
+    for bucket in bucket_names {
+        let bucket_stats = all_bucket_stats.get(&bucket);
+        let proxy = if let Some(stats) = &stats {
+            stats.get_proxy_stats(&bucket).await
+        } else {
+            Default::default()
+        };
+        let proxy = ObsBucketReplicationProxySnapshot {
            proxied_get_requests_total: i64_to_u64_floor_zero(proxy.get_total),
            proxied_get_requests_failures: i64_to_u64_floor_zero(proxy.get_failed),
            proxied_head_requests_total: i64_to_u64_floor_zero(proxy.head_total),
@@ -167,13 +237,67 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
            proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
            proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
            proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
-            resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
-            resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
-            resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
-            resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
-            resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
-            targets,
-        });
+        };
+        let mut runtime = ObsBucketReplicationRuntimeSnapshot {
+            targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
+            ..Default::default()
+        };
+
+        if let Some(bucket_stats) = bucket_stats {
+            for (target_arn, target_stats) in &bucket_stats.stats {
+                runtime.total_failed_bytes = runtime
+                    .total_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
+                runtime.total_failed_count = runtime
+                    .total_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
+
+                let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
+                runtime.last_min_failed_bytes = runtime
+                    .last_min_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(last_min.size));
+                runtime.last_min_failed_count = runtime
+                    .last_min_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(last_min.count));
+
+                let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
+                runtime.last_hour_failed_bytes = runtime
+                    .last_hour_failed_bytes
+                    .saturating_add(i64_to_u64_floor_zero(last_hour.size));
+                runtime.last_hour_failed_count = runtime
+                    .last_hour_failed_count
+                    .saturating_add(i64_to_u64_floor_zero(last_hour.count));
+
+                runtime.sent_bytes = runtime
+                    .sent_bytes
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
+                runtime.sent_count = runtime
+                    .sent_count
+                    .saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
+
+                runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
+                    target_arn: target_arn.clone(),
+                    bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
+                    current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
+                    latency_ms: target_stats.latency.curr,
+                });
+            }
+            runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
+            runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
+            runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
+            runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
+            runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
+            runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
+            runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
+        }
+        let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
+        buckets.push(bucket_replication_stats_snapshot_from_parts(
+            bucket,
+            runtime,
+            proxy,
+            durable_mrf_available,
+            durable_bucket,
+        ));
    }

    buckets
@@ -243,15 +367,97 @@ mod tests {
    fn replication_backlog_count_floors_negative_failed_and_queue_values() {
        assert_eq!(replication_backlog_count([-3, 4].into_iter(), -2), 4);
    }
+
+    #[test]
+    fn replication_backlog_count_keeps_legacy_failed_backlog_semantics() {
+        assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "runtime-bucket".to_string(),
+            ObsBucketReplicationRuntimeSnapshot {
+                current_backlog_count: 3,
+                current_backlog_bytes: 4096,
+                resync_failed_count: 2,
+                ..Default::default()
+            },
+            ObsBucketReplicationProxySnapshot {
+                proxied_get_requests_total: 7,
+                proxied_get_requests_failures: 1,
+                ..Default::default()
+            },
+            true,
+            DurableMrfBucketBacklog {
+                bucket: "runtime-bucket".to_string(),
+                count: 5,
+                bytes: 8192,
+            },
+        );
+
+        assert_eq!(snapshot.bucket, "runtime-bucket");
+        assert_eq!(snapshot.current_backlog_count, 3);
+        assert_eq!(snapshot.current_backlog_bytes, 4096);
+        assert!(snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 5);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
+        assert_eq!(snapshot.resync_failed_count, 2);
+        assert_eq!(snapshot.proxied_get_requests_total, 7);
+        assert_eq!(snapshot.proxied_get_requests_failures, 1);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_reports_durable_only_bucket() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "durable-only".to_string(),
+            ObsBucketReplicationRuntimeSnapshot::default(),
+            ObsBucketReplicationProxySnapshot::default(),
+            true,
+            DurableMrfBucketBacklog {
+                bucket: "durable-only".to_string(),
+                count: 11,
+                bytes: 2048,
+            },
+        );
+
+        assert_eq!(snapshot.bucket, "durable-only");
+        assert_eq!(snapshot.current_backlog_count, 0);
+        assert!(snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 11);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 2048);
+    }
+
+    #[test]
+    fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
+        let snapshot = bucket_replication_stats_snapshot_from_parts(
+            "runtime-only".to_string(),
+            ObsBucketReplicationRuntimeSnapshot {
+                current_backlog_count: 1,
+                current_backlog_bytes: 512,
+                ..Default::default()
+            },
+            ObsBucketReplicationProxySnapshot::default(),
+            false,
+            DurableMrfBucketBacklog::default(),
+        );
+
+        assert_eq!(snapshot.current_backlog_count, 1);
+        assert_eq!(snapshot.current_backlog_bytes, 512);
+        assert!(!snapshot.durable_mrf_available);
+        assert_eq!(snapshot.durable_mrf_backlog_count, 0);
+        assert_eq!(snapshot.durable_mrf_backlog_bytes, 0);
+    }
 }

 pub(crate) mod metrics {
    pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};

    pub(crate) use super::{
-        ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle,
-        obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
-        obs_is_disk_compression_enabled, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
-        obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
+        ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
+        obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
+        obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
+        obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
+        obs_resolve_object_store_handle, obs_transition_state_handle,
    };
 }