merge main

common/common/Cargo.toml

@@ -9,7 +9,7 @@ workspace = true
 [dependencies]
 async-trait.workspace = true
 lazy_static.workspace = true
-scopeguard = "1.2.0"
+scopeguard = { workspace = true }
 tokio.workspace = true
 tonic = { workspace = true }
 tracing-error.workspace = true

common/common/src/bucket_stats.rs

@@ -0,0 +1,73 @@
+use std::collections::HashMap;
+
+use crate::last_minute::{self};
+pub struct ReplicationLatency {
+    // 单个和多部分 PUT 请求的延迟
+    upload_histogram: last_minute::LastMinuteHistogram,
+}
+
+impl ReplicationLatency {
+    // 合并两个 ReplicationLatency
+    pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
+        self.upload_histogram.merge(&other.upload_histogram);
+        self
+    }
+
+    // 获取上传延迟（按对象大小区间分类）
+    pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
+        let mut ret = HashMap::new();
+        let avg = self.upload_histogram.get_avg_data();
+        for (i, v) in avg.iter().enumerate() {
+            let avg_duration = v.avg();
+            ret.insert(self.size_tag_to_string(i), avg_duration.as_millis() as u64);
+        }
+        ret
+    }
+    pub fn update(&mut self, size: i64, during: std::time::Duration) {
+        self.upload_histogram.add(size, during);
+    }
+
+    // 模拟从 size tag 到字符串的转换
+    fn size_tag_to_string(&self, tag: usize) -> String {
+        match tag {
+            0 => String::from("Size < 1 KiB"),
+            1 => String::from("Size < 1 MiB"),
+            2 => String::from("Size < 10 MiB"),
+            3 => String::from("Size < 100 MiB"),
+            4 => String::from("Size < 1 GiB"),
+            _ => String::from("Size > 1 GiB"),
+        }
+    }
+}
+
+// #[derive(Debug, Clone, Default)]
+// pub struct ReplicationLastMinute {
+//     pub last_minute: LastMinuteLatency,
+// }
+
+// impl ReplicationLastMinute {
+//     pub fn merge(&mut self, other: ReplicationLastMinute) -> ReplicationLastMinute {
+//         let mut nl = ReplicationLastMinute::default();
+//         nl.last_minute = self.last_minute.merge(&mut other.last_minute);
+//         nl
+//     }
+
+//     pub fn add_size(&mut self, n: i64) {
+//         let t = SystemTime::now()
+//             .duration_since(UNIX_EPOCH)
+//             .expect("Time went backwards")
+//             .as_secs();
+//         self.last_minute.add_all(t - 1, &AccElem { total: t - 1, size: n as u64, n: 1 });
+//     }
+
+//     pub fn get_total(&self) -> AccElem {
+//         self.last_minute.get_total()
+//     }
+// }
+
+// impl fmt::Display for ReplicationLastMinute {
+//     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+//         let t = self.last_minute.get_total();
+//         write!(f, "ReplicationLastMinute sz= {}, n= {}, dur= {}", t.size, t.n, t.total)
+//     }
+// }

common/common/src/last_minute.rs

@@ -1,6 +1,81 @@
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
-#[derive(Clone, Debug, Default)]
+#[allow(dead_code)]
+#[derive(Debug, Default)]
+struct TimedAction {
+    count: u64,
+    acc_time: u64,
+    min_time: Option<u64>,
+    max_time: Option<u64>,
+    bytes: u64,
+}
+
+#[allow(dead_code)]
+impl TimedAction {
+    // Avg returns the average time spent on the action.
+    pub fn avg(&self) -> Option<std::time::Duration> {
+        if self.count == 0 {
+            return None;
+        }
+        Some(std::time::Duration::from_nanos(self.acc_time / self.count))
+    }
+
+    // AvgBytes returns the average bytes processed.
+    pub fn avg_bytes(&self) -> u64 {
+        if self.count == 0 {
+            return 0;
+        }
+        self.bytes / self.count
+    }
+
+    // Merge other into t.
+    pub fn merge(&mut self, other: TimedAction) {
+        self.count += other.count;
+        self.acc_time += other.acc_time;
+        self.bytes += other.bytes;
+
+        if self.count == 0 {
+            self.min_time = other.min_time;
+        }
+        if let Some(other_min) = other.min_time {
+            self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
+        }
+
+        self.max_time = self
+            .max_time
+            .map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
+    }
+}
+
+#[allow(dead_code)]
+#[derive(Debug)]
+enum SizeCategory {
+    SizeLessThan1KiB = 0,
+    SizeLessThan1MiB,
+    SizeLessThan10MiB,
+    SizeLessThan100MiB,
+    SizeLessThan1GiB,
+    SizeGreaterThan1GiB,
+    // Add new entries here
+    SizeLastElemMarker,
+}
+
+impl std::fmt::Display for SizeCategory {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = match *self {
+            SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
+            SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
+            SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
+            SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
+            SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
+            SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
+            SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
+        };
+        write!(f, "{}", s)
+    }
+}
+
+#[derive(Clone, Debug, Default, Copy)]
 pub struct AccElem {
     pub total: u64,
     pub size: u64,
@@ -28,7 +103,7 @@ impl AccElem {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct LastMinuteLatency {
     pub totals: Vec<AccElem>,
     pub last_sec: u64,
@@ -44,10 +119,11 @@ impl Default for LastMinuteLatency {
 }
 
 impl LastMinuteLatency {
-    pub fn merge(&mut self, o: &mut LastMinuteLatency) -> LastMinuteLatency {
+    pub fn merge(&mut self, o: &LastMinuteLatency) -> LastMinuteLatency {
         let mut merged = LastMinuteLatency::default();
+        let mut x = o.clone();
         if self.last_sec > o.last_sec {
-            o.forward_to(self.last_sec);
+            x.forward_to(self.last_sec);
             merged.last_sec = self.last_sec;
         } else {
             self.forward_to(o.last_sec);
@@ -111,7 +187,6 @@ impl LastMinuteLatency {
         }
     }
 }
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -415,7 +490,7 @@ mod tests {
         latency2.totals[0].total = 20;
         latency2.totals[0].n = 3;
 
-        let merged = latency1.merge(&mut latency2);
+        let merged = latency1.merge(&latency2);
 
         assert_eq!(merged.last_sec, 1000);
         assert_eq!(merged.totals[0].total, 30); // 10 + 20
@@ -434,7 +509,7 @@ mod tests {
         latency1.totals[0].total = 10;
         latency2.totals[0].total = 20;
 
-        let merged = latency1.merge(&mut latency2);
+        let merged = latency1.merge(&latency2);
 
         assert_eq!(merged.last_sec, 1010); // Should use the later time
         assert_eq!(merged.totals[0].total, 30);
@@ -443,9 +518,9 @@ mod tests {
     #[test]
     fn test_last_minute_latency_merge_empty() {
         let mut latency1 = LastMinuteLatency::default();
-        let mut latency2 = LastMinuteLatency::default();
+        let latency2 = LastMinuteLatency::default();
 
-        let merged = latency1.merge(&mut latency2);
+        let merged = latency1.merge(&latency2);
 
         assert_eq!(merged.last_sec, 0);
         for elem in &merged.totals {
@@ -558,7 +633,7 @@ mod tests {
             n: 5,
         };
 
-        let cloned = elem.clone();
+        let cloned = elem;
         assert_eq!(elem.total, cloned.total);
         assert_eq!(elem.size, cloned.size);
         assert_eq!(elem.n, cloned.n);
@@ -755,3 +830,44 @@ mod tests {
         assert_eq!(elem.avg(), Duration::from_secs(0));
     }
 }
+
+const SIZE_LAST_ELEM_MARKER: usize = 10; // 这里假设你的 marker 是 10，请根据实际情况修改
+
+#[allow(dead_code)]
+#[derive(Debug, Default)]
+pub struct LastMinuteHistogram {
+    histogram: Vec<LastMinuteLatency>,
+    size: u32,
+}
+
+impl LastMinuteHistogram {
+    pub fn merge(&mut self, other: &LastMinuteHistogram) {
+        for i in 0..self.histogram.len() {
+            self.histogram[i].merge(&other.histogram[i]);
+        }
+    }
+
+    pub fn add(&mut self, size: i64, t: std::time::Duration) {
+        let index = size_to_tag(size);
+        self.histogram[index].add(&t);
+    }
+
+    pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
+        let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
+        for (i, elem) in self.histogram.iter_mut().enumerate() {
+            res[i] = elem.get_total();
+        }
+        res
+    }
+}
+
+fn size_to_tag(size: i64) -> usize {
+    match size {
+        _ if size < 1024 => 0,               // sizeLessThan1KiB
+        _ if size < 1024 * 1024 => 1,        // sizeLessThan1MiB
+        _ if size < 10 * 1024 * 1024 => 2,   // sizeLessThan10MiB
+        _ if size < 100 * 1024 * 1024 => 3,  // sizeLessThan100MiB
+        _ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
+        _ => 5,                              // sizeGreaterThan1GiB
+    }
+}

common/common/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod bucket_stats;
 // pub mod error;
 pub mod globals;
 pub mod last_minute;

common/lock/src/lrwmutex.rs

@@ -77,8 +77,8 @@ impl LRWMutex {
                 }
                 let sleep_time: u64;
                 {
-                    let mut rng = rand::thread_rng();
-                    sleep_time = rng.gen_range(10..=50);
+                    let mut rng = rand::rng();
+                    sleep_time = rng.random_range(10..=50);
                 }
                 sleep(Duration::from_millis(sleep_time)).await;
             }