feat(replication): add bandwidth-aware reporting for bucket replication metrics (#2141)

crates/ecstore/src/bucket/replication/mod.rs

@@ -23,4 +23,5 @@ pub use config::*;
 pub use datatypes::*;
 pub use replication_pool::*;
 pub use replication_resyncer::*;
+pub use replication_state::BucketStats;
 pub use rule::*;

crates/ecstore/src/bucket/replication/replication_resyncer.rs

@@ -18,6 +18,7 @@ use crate::bucket::bucket_target_sys::{
 };
 use crate::bucket::metadata_sys;
 use crate::bucket::replication::ResyncStatusType;
+use crate::bucket::replication::replication_pool::GLOBAL_REPLICATION_STATS;
 use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
 use crate::bucket::tagging::decode_tags_to_map;
 use crate::bucket::target::BucketTargets;
@@ -1442,9 +1443,13 @@ pub async fn replicate_delete<S: StorageAPI>(dobj: DeletedObjectReplicationInfo,
         )
     };
 
-    for tgt in rinfos.targets.iter() {
-        if tgt.replication_status != tgt.prev_replication_status {
-            // TODO: update global replication status
+    if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+        for tgt in rinfos.targets.iter() {
+            if tgt.replication_status != tgt.prev_replication_status {
+                stats
+                    .update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone())
+                    .await;
+            }
         }
     }
 
@@ -1900,7 +1905,15 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
             object_info = u;
         }
 
-        // TODO: update stats
+        if let Some(stats) = GLOBAL_REPLICATION_STATS.get() {
+            for tgt in &rinfos.targets {
+                if tgt.replication_status != tgt.prev_replication_status {
+                    stats
+                        .update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone())
+                        .await;
+                }
+            }
+        }
     }
 
     let event_name = if replication_status == ReplicationStatusType::Completed {
@@ -1918,9 +1931,17 @@ pub async fn replicate_object<S: StorageAPI>(roi: ReplicateObjectInfo, storage:
         ..Default::default()
     });
 
-    if rinfos.replication_status() != ReplicationStatusType::Completed {
-        // TODO: update stats
-        // pool
+    if rinfos.replication_status() != ReplicationStatusType::Completed
+        && roi.replication_status_internal == rinfos.replication_status_internal()
+        && let Some(stats) = GLOBAL_REPLICATION_STATS.get()
+    {
+        for tgt in &rinfos.targets {
+            if tgt.replication_status != tgt.prev_replication_status {
+                stats
+                    .update(&bucket, tgt, tgt.replication_status.clone(), tgt.prev_replication_status.clone())
+                    .await;
+            }
+        }
     }
 }
 

crates/ecstore/src/bucket/replication/replication_state.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::error::Error;
+use crate::global::get_global_bucket_monitor;
 use rustfs_filemeta::{ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -585,6 +586,8 @@ pub struct BucketReplicationStat {
     pub latency: LatencyStats,
     pub xfer_rate_lrg: XferStats,
     pub xfer_rate_sml: XferStats,
+    pub bandwidth_limit_bytes_per_sec: i64,
+    pub current_bandwidth_bytes_per_sec: f64,
 }
 
 impl BucketReplicationStat {
@@ -1019,6 +1022,9 @@ impl ReplicationStats {
                     latency: stat.latency.merge(&old_stat.latency),
                     xfer_rate_lrg: lrg,
                     xfer_rate_sml: sml,
+                    bandwidth_limit_bytes_per_sec: stat.bandwidth_limit_bytes_per_sec,
+                    current_bandwidth_bytes_per_sec: stat.current_bandwidth_bytes_per_sec
+                        + old_stat.current_bandwidth_bytes_per_sec,
                 };
 
                 tot_replicated_size += stat.replicated_size;
@@ -1069,24 +1075,43 @@ impl ReplicationStats {
         // In actual implementation, statistics would be obtained from cluster
         // This is simplified to get from local cache
         let cache = self.cache.read().await;
-        if let Some(stats) = cache.get(bucket) {
-            BucketStats {
-                uptime: SystemTime::now()
-                    .duration_since(SystemTime::UNIX_EPOCH)
-                    .unwrap_or_default()
-                    .as_secs() as i64,
-                replication_stats: stats.clone_stats(),
-                queue_stats: Default::default(),
-                proxy_stats: ProxyMetric::default(),
-            }
+        let mut replication_stats = if let Some(stats) = cache.get(bucket) {
+            stats.clone_stats()
         } else {
-            BucketStats {
-                uptime: 0,
-                replication_stats: BucketReplicationStats::new(),
-                queue_stats: Default::default(),
-                proxy_stats: ProxyMetric::default(),
+            BucketReplicationStats::new()
+        };
+        let uptime = if cache.contains_key(bucket) {
+            SystemTime::now()
+                .duration_since(SystemTime::UNIX_EPOCH)
+                .unwrap_or_default()
+                .as_secs() as i64
+        } else {
+            0
+        };
+        drop(cache);
+
+        if let Some(monitor) = get_global_bucket_monitor() {
+            let bw_report = monitor.get_report(|name| name == bucket);
+            for (opts, bw) in bw_report.bucket_stats {
+                let stat = replication_stats
+                    .stats
+                    .entry(opts.replication_arn)
+                    .or_insert_with(|| BucketReplicationStat {
+                        xfer_rate_lrg: XferStats::new(),
+                        xfer_rate_sml: XferStats::new(),
+                        ..Default::default()
+                    });
+                stat.bandwidth_limit_bytes_per_sec = bw.limit_bytes_per_sec;
+                stat.current_bandwidth_bytes_per_sec = bw.current_bandwidth_bytes_per_sec;
             }
         }
+
+        BucketStats {
+            uptime,
+            replication_stats,
+            queue_stats: Default::default(),
+            proxy_stats: ProxyMetric::default(),
+        }
     }
 
     /// Increase queue statistics