feat(replication): add bandwidth-aware reporting for bucket replication metrics (#2141)

crates/metrics/src/collectors/bucket_replication.rs

@@ -0,0 +1,119 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Bucket replication bandwidth metrics collector.
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+use std::borrow::Cow;
+
+/// Bucket replication bandwidth stats for one replication target.
+#[derive(Debug, Clone, Default)]
+pub struct BucketReplicationBandwidthStats {
+    pub bucket: String,
+    pub target_arn: String,
+    pub limit_bytes_per_sec: i64,
+    pub current_bandwidth_bytes_per_sec: f64,
+}
+
+const BUCKET_LABEL: &str = "bucket";
+const TARGET_ARN_LABEL: &str = "targetArn";
+
+const METRIC_BANDWIDTH_LIMIT: &str = "rustfs_bucket_replication_bandwidth_limit_bytes_per_second";
+const METRIC_BANDWIDTH_CURRENT: &str = "rustfs_bucket_replication_bandwidth_current_bytes_per_second";
+
+const HELP_BANDWIDTH_LIMIT: &str = "Configured bandwidth limit for replication in bytes per second";
+const HELP_BANDWIDTH_CURRENT: &str = "Current replication bandwidth in bytes per second (EWMA)";
+
+/// Collect bucket replication bandwidth metrics for Prometheus/OpenTelemetry export.
+#[must_use]
+#[inline]
+pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
+    if stats.is_empty() {
+        return Vec::new();
+    }
+
+    let mut metrics = Vec::with_capacity(stats.len() * 2);
+    for stat in stats {
+        let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
+        let target_arn_label: Cow<'static, str> = Cow::Owned(stat.target_arn.clone());
+
+        metrics.push(
+            PrometheusMetric::new(
+                METRIC_BANDWIDTH_LIMIT,
+                MetricType::Gauge,
+                HELP_BANDWIDTH_LIMIT,
+                stat.limit_bytes_per_sec as f64,
+            )
+            .with_label(BUCKET_LABEL, bucket_label.clone())
+            .with_label(TARGET_ARN_LABEL, target_arn_label.clone()),
+        );
+
+        metrics.push(
+            PrometheusMetric::new(
+                METRIC_BANDWIDTH_CURRENT,
+                MetricType::Gauge,
+                HELP_BANDWIDTH_CURRENT,
+                stat.current_bandwidth_bytes_per_sec,
+            )
+            .with_label(BUCKET_LABEL, bucket_label)
+            .with_label(TARGET_ARN_LABEL, target_arn_label),
+        );
+    }
+
+    metrics
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_bucket_replication_bandwidth_metrics() {
+        let stats = vec![BucketReplicationBandwidthStats {
+            bucket: "b1".to_string(),
+            target_arn: "arn:rustfs:replication:us-east-1:1:test-2".to_string(),
+            limit_bytes_per_sec: 1_048_576,
+            current_bandwidth_bytes_per_sec: 204_800.0,
+        }];
+
+        let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+        assert_eq!(metrics.len(), 2);
+
+        let limit_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_LIMIT);
+        assert!(limit_metric.is_some());
+        assert_eq!(limit_metric.map(|m| m.value), Some(1_048_576.0));
+        assert!(
+            limit_metric
+                .and_then(|m| {
+                    m.labels
+                        .iter()
+                        .find(|(k, _)| *k == TARGET_ARN_LABEL)
+                        .map(|(_, v)| v.as_ref() == "arn:rustfs:replication:us-east-1:1:test-2")
+                })
+                .unwrap_or(false)
+        );
+
+        let current_metric = metrics.iter().find(|m| m.name == METRIC_BANDWIDTH_CURRENT);
+        assert!(current_metric.is_some());
+        assert_eq!(current_metric.map(|m| m.value), Some(204_800.0));
+    }
+
+    #[test]
+    fn test_collect_bucket_replication_bandwidth_metrics_empty() {
+        let stats: Vec<BucketReplicationBandwidthStats> = Vec::new();
+        let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+        assert!(metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/global.rs

@@ -13,17 +13,19 @@
 // limitations under the License.
 
 use crate::collectors::{
-    BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics, collect_cluster_metrics, collect_node_metrics,
-    collect_resource_metrics,
+    BucketReplicationBandwidthStats, BucketStats, ClusterStats, DiskStats, ResourceStats, collect_bucket_metrics,
+    collect_bucket_replication_bandwidth_metrics, collect_cluster_metrics, collect_node_metrics, collect_resource_metrics,
 };
 use crate::constants::{
-    DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL,
-    DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
+    DEFAULT_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL,
+    DEFAULT_NODE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL, ENV_BUCKET_METRICS_INTERVAL,
+    ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL, ENV_CLUSTER_METRICS_INTERVAL, ENV_DEFAULT_METRICS_INTERVAL,
     ENV_NODE_METRICS_INTERVAL, ENV_RESOURCE_METRICS_INTERVAL,
 };
 use crate::format::report_metrics;
 use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
 use rustfs_ecstore::data_usage::load_data_usage_from_backend;
+use rustfs_ecstore::global::get_global_bucket_monitor;
 use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free};
 use rustfs_ecstore::store_api::{BucketOperations, BucketOptions};
 use rustfs_ecstore::{StorageAPI, new_object_layer_fn};
@@ -147,6 +149,25 @@ async fn collect_bucket_stats() -> Vec<BucketStats> {
     stats
 }
 
+/// Collect bucket replication bandwidth stats from the global monitor.
+fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBandwidthStats> {
+    let Some(monitor) = get_global_bucket_monitor() else {
+        return Vec::new();
+    };
+
+    monitor
+        .get_report(|_| true)
+        .bucket_stats
+        .into_iter()
+        .map(|(opts, details)| BucketReplicationBandwidthStats {
+            bucket: opts.name,
+            target_arn: opts.replication_arn,
+            limit_bytes_per_sec: details.limit_bytes_per_sec,
+            current_bandwidth_bytes_per_sec: details.current_bandwidth_bytes_per_sec,
+        })
+        .collect()
+}
+
 /// Collect disk statistics from the storage layer.
 async fn collect_disk_stats() -> Vec<DiskStats> {
     let Some(store) = new_object_layer_fn() else {
@@ -235,6 +256,10 @@ pub fn init_metrics_collectors(token: CancellationToken) {
 
     let cluster_interval = get_interval(ENV_CLUSTER_METRICS_INTERVAL, DEFAULT_CLUSTER_METRICS_INTERVAL);
     let bucket_interval = get_interval(ENV_BUCKET_METRICS_INTERVAL, DEFAULT_BUCKET_METRICS_INTERVAL);
+    let bucket_replication_bandwidth_interval = get_interval(
+        ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+        DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL,
+    );
     let node_interval = get_interval(ENV_NODE_METRICS_INTERVAL, DEFAULT_NODE_METRICS_INTERVAL);
     let resource_interval = get_interval(ENV_RESOURCE_METRICS_INTERVAL, DEFAULT_RESOURCE_METRICS_INTERVAL);
 
@@ -295,6 +320,25 @@ pub fn init_metrics_collectors(token: CancellationToken) {
         }
     });
 
+    // Spawn task for bucket replication bandwidth metrics
+    let token_clone = token.clone();
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(bucket_replication_bandwidth_interval);
+        loop {
+            tokio::select! {
+                _ = interval.tick() => {
+                    let stats = collect_bucket_replication_bandwidth_stats();
+                    let metrics = collect_bucket_replication_bandwidth_metrics(&stats);
+                    report_metrics(&metrics);
+                }
+                _ = token_clone.cancelled() => {
+                    warn!("Metrics collection for bucket replication bandwidth stats cancelled.");
+                    return;
+                }
+            }
+        }
+    });
+
     // Spawn task for resource metrics
     let token_clone = token.clone();
     tokio::spawn(async move {

crates/metrics/src/collectors/mod.rs

@@ -19,6 +19,7 @@
 //!
 //! - [`cluster`]: Cluster-wide capacity and object statistics
 //! - [`bucket`]: Per-bucket usage and quota metrics
+//! - [`bucket_replication`]: Per-target replication bandwidth metrics
 //! - [`node`]: Per-node disk capacity and health metrics
 //! - [`resource`]: System resource metrics (CPU, memory, uptime)
 //!
@@ -61,12 +62,14 @@
 //! ```
 
 mod bucket;
+mod bucket_replication;
 mod cluster;
 pub(crate) mod global;
 mod node;
 mod resource;
 
 pub use bucket::{BucketStats, collect_bucket_metrics};
+pub use bucket_replication::{BucketReplicationBandwidthStats, collect_bucket_replication_bandwidth_metrics};
 pub use cluster::{ClusterStats, collect_cluster_metrics};
 pub use global::init_metrics_collectors;
 pub use node::{DiskStats, collect_node_metrics};

crates/metrics/src/constants/mod.rs

@@ -40,3 +40,8 @@ pub const DEFAULT_NODE_METRICS_INTERVAL: Duration = Duration::from_secs(60);
 pub const ENV_RESOURCE_METRICS_INTERVAL: &str = "RUSTFS_METRICS_RESOURCE_INTERVAL_SEC";
 /// Default interval for collecting system resource metrics (CPU, memory).
 pub const DEFAULT_RESOURCE_METRICS_INTERVAL: Duration = Duration::from_secs(15);
+
+/// Environment variable key for replication bandwidth metrics interval (seconds).
+pub const ENV_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL: &str = "RUSTFS_METRICS_BUCKET_REPLICATION_BANDWIDTH_INTERVAL_SEC";
+/// Default interval for collecting replication bandwidth metrics.
+pub const DEFAULT_BUCKET_REPLICATION_BANDWIDTH_METRICS_INTERVAL: Duration = Duration::from_secs(30);

crates/metrics/src/metrics_type/bucket_replication.rs

@@ -190,6 +190,24 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyL
     )
 });
 
+pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::BandwidthLimitBytesPerSecond,
+        "Configured bandwidth limit for replication in bytes per second",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
+pub static BUCKET_REPL_BANDWIDTH_CURRENT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::BandwidthCurrentBytesPerSecond,
+        "Current replication bandwidth in bytes per second (EWMA)",
+        &[BUCKET_L, TARGET_ARN_L],
+        subsystems::BUCKET_REPLICATION,
+    )
+});
+
 // TODO - add a metric for the number of DELETE requests proxied to replication target
 pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(

crates/metrics/src/metrics_type/entry/metric_name.rs

@@ -263,6 +263,8 @@ pub enum MetricName {
     ReplicationMaxQueuedCount,
     ReplicationMaxDataTransferRate,
     ReplicationRecentBacklogCount,
+    BandwidthLimitBytesPerSecond,
+    BandwidthCurrentBytesPerSecond,
 
     // Scanner-related metrics
     ScannerBucketScansFinished,
@@ -580,6 +582,8 @@ impl MetricName {
             Self::ReplicationMaxQueuedCount => "max_queued_count".to_string(),
             Self::ReplicationMaxDataTransferRate => "max_data_transfer_rate".to_string(),
             Self::ReplicationRecentBacklogCount => "recent_backlog_count".to_string(),
+            Self::BandwidthLimitBytesPerSecond => "bandwidth_limit_bytes_per_second".to_string(),
+            Self::BandwidthCurrentBytesPerSecond => "bandwidth_current_bytes_per_second".to_string(),
 
             // Scanner-related metrics
             Self::ScannerBucketScansFinished => "bucket_scans_finished".to_string(),