temp(4)

Signed-off-by: junxiang Mu <1948535941@qq.com>

ecstore/src/heal/data_scanner.rs

@@ -12,7 +12,7 @@ use std::{
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
 
-use byteorder::{LittleEndian, ReadBytesExt};
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 use lazy_static::lazy_static;
 use rand::Rng;
 use rmp_serde::{Deserializer, Serializer};
@@ -29,12 +29,11 @@ use tokio::{
 use tracing::{error, info};
 
 use super::{
-    data_scanner_metric::globalScannerMetrics,
+    data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics},
     data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH},
     data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
     heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN},
 };
-use crate::disk::DiskAPI;
 use crate::heal::data_scanner_metric::current_path_updater;
 use crate::heal::data_usage::DATA_USAGE_ROOT;
 use crate::{
@@ -58,6 +57,10 @@ use crate::{
     store::{ECStore, ListPathOptions},
     utils::path::{path_join, path_to_bucket_object, path_to_bucket_object_with_base_path, SLASH_SEPARATOR},
 };
+use crate::{
+    disk::DiskAPI,
+    store_api::{FileInfo, ObjectInfo},
+};
 
 const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders.
 const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles.
@@ -132,6 +135,7 @@ async fn run_data_scanner() {
     }
 
     loop {
+        let stop_fn = ScannerMetrics::log(ScannerMetric::ScanCycle);
         cycle_info.current = cycle_info.next;
         cycle_info.started = SystemTime::now();
         {
@@ -155,6 +159,26 @@ async fn run_data_scanner() {
         tokio::spawn(async {
             store_data_usage_in_backend(rx).await;
         });
+        let mut res = HashMap::new();
+        res.insert("cycle".to_string(), cycle_info.current.to_string());
+        match store.ns_scanner(tx, cycle_info.current as usize, scan_mode).await {
+            Ok(_) => {
+                cycle_info.next += 1;
+                cycle_info.current = 0;
+                cycle_info.cycle_completed.push(SystemTime::now());
+                if cycle_info.cycle_completed.len() > DATA_USAGE_UPDATE_DIR_CYCLES as usize {
+                    cycle_info.cycle_completed = cycle_info.cycle_completed[cycle_info.cycle_completed.len() - DATA_USAGE_UPDATE_DIR_CYCLES as usize..].to_vec();
+                }
+                globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await;
+                let mut tmp = Vec::new();
+                tmp.write_u64::<LittleEndian>(cycle_info.next).unwrap();
+                let _ = save_config(store, &DATA_USAGE_BLOOM_NAME_PATH, &tmp).await;
+            },
+            Err(err) => {
+                res.insert("error".to_string(), err.to_string());
+            },
+        }
+        stop_fn(&res).await;
         sleep(Duration::from_secs(SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst))).await;
     }
 }
@@ -241,7 +265,7 @@ impl Default for CurrentScannerCycle {
 }
 
 impl CurrentScannerCycle {
-    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+    pub fn marshal_msg(&self, buf: &[u8]) -> Result<Vec<u8>> {
         let len: u32 = 4;
         let mut wr = Vec::new();
 
@@ -267,8 +291,9 @@ impl CurrentScannerCycle {
             .serialize(&mut Serializer::new(&mut buf))
             .expect("Serialization failed");
         rmp::encode::write_bin(&mut wr, &buf)?;
-
-        Ok(wr)
+        let mut result = buf.to_vec();
+        result.extend(wr.iter());
+        Ok(result)
     }
 
     #[tracing::instrument]
@@ -295,10 +320,10 @@ impl CurrentScannerCycle {
                     self.current = u;
                 }
 
-                "next" => {
-                    let u: u64 = rmp::decode::read_int(&mut cur)?;
-                    self.next = u;
-                }
+                // "next" => {
+                //     let u: u64 = rmp::decode::read_int(&mut cur)?;
+                //     self.next = u;
+                // }
                 "started" => {
                     let u: u64 = rmp::decode::read_int(&mut cur)?;
                     let started = timestamp_to_system_time(u);
@@ -329,22 +354,23 @@ fn timestamp_to_system_time(timestamp: u64) -> SystemTime {
     UNIX_EPOCH + std::time::Duration::new(timestamp, 0)
 }
 
-#[derive(Debug, Default)]
+#[derive(Clone, Debug, Default)]
 struct Heal {
     enabled: bool,
     bitrot: bool,
 }
 
+#[derive(Clone)]
 pub struct ScannerItem {
-    path: String,
-    bucket: String,
-    prefix: String,
-    object_name: String,
-    replication: Option<ReplicationConfiguration>,
+    pub path: String,
+    pub bucket: String,
+    pub prefix: String,
+    pub object_name: String,
+    pub replication: Option<ReplicationConfiguration>,
     // todo: lifecycle
     // typ: fs::Permissions,
-    heal: Heal,
-    debug: bool,
+    pub heal: Heal,
+    pub debug: bool,
 }
 
 impl ScannerItem {
@@ -365,6 +391,43 @@ impl ScannerItem {
     pub fn object_path(&self) -> PathBuf {
         path_join(&[PathBuf::from(self.prefix.clone()), PathBuf::from(self.object_name.clone())])
     }
+
+    pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
+        let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
+        if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst).try_into().unwrap() {
+            // todo
+        }
+
+        let mut cumulative_size = 0;
+        for obj_info in obj_infos.iter() {
+            cumulative_size += obj_info.size;
+        }
+
+        if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst).try_into().unwrap() {
+            //todo
+        }
+
+        Ok(obj_infos)
+    }
+
+    pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
+        let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
+        let mut object_infos = Vec::new();
+        for info in fivs.iter() {
+            object_infos.push(info.to_object_info(&self.bucket, &self.object_path().to_string_lossy(), false));
+        }
+        done().await;
+
+        Ok(object_infos)
+    }
+
+    pub async fn apply_actions(&self, oi: &ObjectInfo, size_s: &SizeSummary) -> (bool, usize) {
+        let done = ScannerMetrics::time(ScannerMetric::Ilm);
+        //todo: lifecycle
+        done().await;
+
+        (false, 0)
+    }
 }
 
 #[derive(Debug, Default)]
@@ -420,7 +483,7 @@ struct FolderScanner {
     last_update: SystemTime,
     update_current_path: UpdateCurrentPathFn,
     skip_heal: AtomicBool,
-    drive: DiskStore,
+    drive: LocalDrive,
 }
 
 impl FolderScanner {
@@ -951,9 +1014,10 @@ pub fn has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursi
     false
 }
 
+pub type LocalDrive = Arc<dyn DiskAPI>;
 pub async fn scan_data_folder(
     disks: &[Option<DiskStore>],
-    drive: &DiskStore,
+    drive: LocalDrive,
     cache: &DataUsageCache,
     get_size_fn: GetSizeFn,
     heal_scan_mode: HealScanMode,
@@ -964,6 +1028,11 @@ pub async fn scan_data_folder(
 
     let base_path = drive.to_string();
     let (update_path, close_disk) = current_path_updater(&base_path, &cache.info.name);
+    let skip_heal = if *GLOBAL_IsErasure.read().await || cache.info.skip_healing {
+        AtomicBool::new(true)
+    } else {
+        AtomicBool::new(false)
+    };
     let mut s = FolderScanner {
         root: base_path,
         get_size: get_size_fn,
@@ -978,11 +1047,7 @@ pub async fn scan_data_folder(
         update_current_path: update_path,
         disks: disks.to_vec(),
         disks_quorum: disks.len() / 2,
-        skip_heal: if *GLOBAL_IsErasure.read().await || cache.info.skip_healing {
-            AtomicBool::new(true)
-        } else {
-            AtomicBool::new(false)
-        },
+        skip_heal,
         drive: drive.clone(),
     };
 
@@ -1002,7 +1067,7 @@ pub async fn scan_data_folder(
     }
     s.new_cache
         .force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN.try_into().unwrap());
-    s.new_cache.info.last_update = SystemTime::now();
+    s.new_cache.info.last_update = Some(SystemTime::now());
     s.new_cache.info.next_cycle = cache.info.next_cycle;
     close_disk().await;
     Ok(s.new_cache)

ecstore/src/heal/data_scanner_metric.rs

@@ -1,6 +1,10 @@
+use common::last_minute::{AccElem, LastMinuteLatency};
 use lazy_static::lazy_static;
 use std::future::Future;
 use std::pin::Pin;
+use std::sync::atomic::AtomicU64;
+use std::sync::Once;
+use std::time::{Duration, UNIX_EPOCH};
 use std::{
     collections::HashMap,
     sync::{
@@ -53,8 +57,75 @@ pub enum ScannerMetric {
     Last,
 }
 
+static INIT: Once = Once::new();
+
+#[derive(Default)]
+pub struct LockedLastMinuteLatency {
+    cached_sec: AtomicU64,
+    cached: AccElem,
+    mu: RwLock<bool>,
+    latency: LastMinuteLatency,
+}
+
+impl Clone for LockedLastMinuteLatency {
+    fn clone(&self) -> Self {
+        Self {
+            cached_sec: AtomicU64::new(0),
+            cached: self.cached.clone(),
+            mu: RwLock::new(true),
+            latency: self.latency.clone(),
+        }
+    }
+}
+
+impl LockedLastMinuteLatency {
+    pub fn add(&mut self, value: &Duration) {
+        self.add_size(value, 0);
+    }
+
+    pub async fn add_size(&mut self, value: &Duration, sz: u64) {
+        let t = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        INIT.call_once(|| {
+            self.cached = AccElem::default();
+            self.cached_sec.store(t, Ordering::SeqCst);
+        });
+        let last_t = self.cached_sec.load(Ordering::SeqCst);
+        if last_t != t
+            && self
+                .cached_sec
+                .compare_exchange(last_t, t, Ordering::SeqCst, Ordering::SeqCst)
+                .is_ok()
+        {
+            let old = self.cached.clone();
+            self.cached = AccElem::default();
+            let mut a = AccElem::default();
+            a.size = old.size;
+            a.total = old.total;
+            a.n = old.n;
+            self.mu.write().await;
+            self.latency.add_all(t - 1, &a);
+        }
+        self.cached.n += 1;
+        self.cached.total += value.as_secs();
+        self.cached.size != sz;
+    }
+
+    pub async fn total(&mut self) -> AccElem {
+        self.mu.read().await;
+        self.latency.get_total()
+    }
+}
+
+pub type LogFn = Arc<dyn Fn(&HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TimeSizeFn = Arc<dyn Fn(u64) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+
 pub struct ScannerMetrics {
     operations: Vec<AtomicU32>,
+    latency: Vec<LockedLastMinuteLatency>,
     cycle_info: RwLock<Option<CurrentScannerCycle>>,
     current_paths: HashMap<String, String>,
 }
@@ -63,23 +134,63 @@ impl ScannerMetrics {
     pub fn new() -> Self {
         Self {
             operations: (0..ScannerMetric::Last as usize).map(|_| AtomicU32::new(0)).collect(),
+            latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
             cycle_info: RwLock::new(None),
             current_paths: HashMap::new(),
         }
     }
 
-    pub fn log(&mut self, s: ScannerMetric, _paths: &[String], _custom: &HashMap<String, String>, _start_time: SystemTime) {
-        // let duration = start_time.duration_since(start_time);
-        self.operations[s.clone() as usize].fetch_add(1, Ordering::SeqCst);
-        // Dodo
-    }
-
     pub async fn set_cycle(&mut self, c: Option<CurrentScannerCycle>) {
         *self.cycle_info.write().await = c;
     }
+
+    pub fn log(s: ScannerMetric) -> LogFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move |custom: &HashMap<String, String>| {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add(&duration);
+                }
+            })
+        })
+    }
+
+    pub fn time_size(s: ScannerMetric) -> TimeSizeFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move |sz: u64| {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add_size(&duration, sz);
+                }
+            })
+        })
+    }
+
+    pub fn time(s: ScannerMetric) -> TimeFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move || {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add(&duration);
+                }
+            })
+        })
+    }
 }
 
-pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
 pub fn current_path_updater(disk: &str, initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
     let disk_1 = disk.to_string();
     let disk_2 = disk.to_string();

ecstore/src/heal/data_usage.rs

@@ -34,13 +34,13 @@ lazy_static! {
 // - replica failed count
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct BucketTargetUsageInfo {
-    replication_pending_size: u64,
-    replication_failed_size: u64,
-    replicated_size: u64,
-    replica_size: u64,
-    replication_pending_count: u64,
-    replication_failed_count: u64,
-    replicated_count: u64,
+    pub replication_pending_size: u64,
+    pub replication_failed_size: u64,
+    pub replicated_size: u64,
+    pub replica_size: u64,
+    pub replication_pending_count: u64,
+    pub replication_failed_count: u64,
+    pub replicated_count: u64,
 }
 
 // BucketUsageInfo - bucket usage info provides
@@ -49,82 +49,63 @@ pub struct BucketTargetUsageInfo {
 // - object size histogram per bucket
 #[derive(Debug, Default, Serialize, Deserialize)]
 pub struct BucketUsageInfo {
-    size: u64,
+    pub size: u64,
     // Following five fields suffixed with V1 are here for backward compatibility
     // Total Size for objects that have not yet been replicated
-    replication_pending_size_v1: u64,
+    pub replication_pending_size_v1: u64,
     // Total size for objects that have witness one or more failures and will be retried
-    replication_failed_size_v1: u64,
+    pub replication_failed_size_v1: u64,
     // Total size for objects that have been replicated to destination
-    replicated_size_v1: u64,
+    pub replicated_size_v1: u64,
     // Total number of objects pending replication
-    replication_pending_count_v1: u64,
+    pub replication_pending_count_v1: u64,
     // Total number of objects that failed replication
-    replication_failed_count_v1: u64,
+    pub replication_failed_count_v1: u64,
 
-    objects_count: u64,
-    object_size_histogram: HashMap<String, u64>,
-    object_versions_histogram: HashMap<String, u64>,
-    versions_count: u64,
-    delete_markers_count: u64,
-    replica_size: u64,
-    replica_count: u64,
-    replication_info: HashMap<String, BucketTargetUsageInfo>,
+    pub objects_count: u64,
+    pub object_size_histogram: HashMap<String, u64>,
+    pub object_versions_histogram: HashMap<String, u64>,
+    pub versions_count: u64,
+    pub delete_markers_count: u64,
+    pub replica_size: u64,
+    pub replica_count: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
 }
 
 // DataUsageInfo represents data usage stats of the underlying Object API
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Default, Serialize, Deserialize)]
 pub struct DataUsageInfo {
-    total_capacity: u64,
-    total_used_capacity: u64,
-    total_free_capacity: u64,
+    pub total_capacity: u64,
+    pub total_used_capacity: u64,
+    pub total_free_capacity: u64,
 
     // LastUpdate is the timestamp of when the data usage info was last updated.
     // This does not indicate a full scan.
-    last_update: SystemTime,
+    pub last_update: Option<SystemTime>,
 
     // Objects total count across all buckets
-    objects_total_count: u64,
+    pub objects_total_count: u64,
     // Versions total count across all buckets
-    versions_total_count: u64,
+    pub versions_total_count: u64,
     // Delete markers total count across all buckets
-    delete_markers_total_count: u64,
+    pub delete_markers_total_count: u64,
     // Objects total size across all buckets
-    objects_total_size: u64,
-    replication_info: HashMap<String, BucketTargetUsageInfo>,
+    pub objects_total_size: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
 
     // Total number of buckets in this cluster
-    buckets_count: u64,
+    pub buckets_count: u64,
     // Buckets usage info provides following information across all buckets
     // - total size of the bucket
     // - total objects in a bucket
     // - object size histogram per bucket
-    buckets_usage: HashMap<String, BucketUsageInfo>,
+    pub buckets_usage: HashMap<String, BucketUsageInfo>,
     // Deprecated kept here for backward compatibility reasons.
-    bucket_sizes: HashMap<String, u64>,
+    pub bucket_sizes: HashMap<String, u64>,
     // Todo: TierStats
     // TierStats contains per-tier stats of all configured remote tiers
 }
 
-impl Default for DataUsageInfo {
-    fn default() -> Self {
-        Self {
-            total_capacity: Default::default(),
-            total_used_capacity: Default::default(),
-            total_free_capacity: Default::default(),
-            last_update: SystemTime::now(),
-            objects_total_count: Default::default(),
-            versions_total_count: Default::default(),
-            delete_markers_total_count: Default::default(),
-            objects_total_size: Default::default(),
-            replication_info: Default::default(),
-            buckets_count: Default::default(),
-            buckets_usage: Default::default(),
-            bucket_sizes: Default::default(),
-        }
-    }
-}
-
 pub async fn store_data_usage_in_backend(mut rx: Receiver<DataUsageInfo>) {
     let layer = new_object_layer_fn();
     let lock = layer.read().await;

ecstore/src/heal/data_usage_cache.rs

@@ -4,7 +4,8 @@ use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::new_object_layer_fn;
 use crate::set_disk::SetDisks;
-use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectOptions};
+use crate::store_api::{BucketInfo, HTTPRangeSpec, ObjectIO, ObjectOptions};
+use bytes::Bytes;
 use bytesize::ByteSize;
 use http::HeaderMap;
 use path_clean::PathClean;
@@ -23,7 +24,7 @@ use tokio::sync::mpsc::Sender;
 use tokio::time::sleep;
 
 use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS};
-use super::data_usage::DATA_USAGE_ROOT;
+use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, DATA_USAGE_ROOT};
 
 // DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
 pub const DATA_USAGE_BUCKET_LEN: usize = 11;
@@ -152,6 +153,22 @@ impl SizeHistogram {
             }
         }
     }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        let mut spl_count = 0;
+        for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) {
+            if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 {
+                res.insert(oh.name.to_string(), spl_count);
+            } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end <= ByteSize::mib(1).as_u64() - 1 {
+                spl_count += count;
+                res.insert(oh.name.to_string(), *count);
+            } else {
+                res.insert(oh.name.to_string(), *count);
+            }
+        }
+        res
+    }
 }
 
 // versionsHistogram is a histogram of number of versions in an object.
@@ -173,6 +190,14 @@ impl VersionsHistogram {
             }
         }
     }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) {
+            res.insert(ov.name.to_string(), *count);
+        }
+        res
+    }
 }
 
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -319,11 +344,11 @@ pub struct DataUsageEntryInfo {
     pub entry: DataUsageEntry,
 }
 
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Default, Serialize, Deserialize)]
 pub struct DataUsageCacheInfo {
     pub name: String,
     pub next_cycle: u32,
-    pub last_update: SystemTime,
+    pub last_update: Option<SystemTime>,
     pub skip_healing: bool,
     // todo: life_cycle
     // pub life_cycle:
@@ -333,18 +358,18 @@ pub struct DataUsageCacheInfo {
     pub replication: Option<ReplicationConfiguration>,
 }
 
-impl Default for DataUsageCacheInfo {
-    fn default() -> Self {
-        Self {
-            name: Default::default(),
-            next_cycle: Default::default(),
-            last_update: SystemTime::now(),
-            skip_healing: Default::default(),
-            updates: Default::default(),
-            replication: Default::default(),
-        }
-    }
-}
+// impl Default for DataUsageCacheInfo {
+//     fn default() -> Self {
+//         Self {
+//             name: Default::default(),
+//             next_cycle: Default::default(),
+//             last_update: SystemTime::now(),
+//             skip_healing: Default::default(),
+//             updates: Default::default(),
+//             replication: Default::default(),
+//         }
+//     }
+// }
 
 #[derive(Clone, Default, Serialize, Deserialize)]
 pub struct DataUsageCache {
@@ -410,8 +435,11 @@ impl DataUsageCache {
                 },
             }
             retries += 1;
-            let mut rng = rand::thread_rng();
-            sleep(Duration::from_millis(rng.gen_range(0..1_000))).await;
+            let dur = {
+                let mut rng = rand::thread_rng();
+                rng.gen_range(0..1_000)
+            };
+            sleep(Duration::from_millis(dur)).await;
         }
         Ok(d)
     }
@@ -655,6 +683,100 @@ impl DataUsageCache {
         n
     }
 
+    pub fn merge(&mut self, o: &DataUsageCache) {
+        let mut existing_root = self.root();
+        let other_root = o.root();
+        if existing_root.is_none() && other_root.is_none() {
+            return;
+        }
+        if other_root.is_none() {
+            return;
+        }
+        if existing_root.is_none() {
+            *self = o.clone();
+            return;
+        }
+        if o.info.last_update.gt(&self.info.last_update) {
+            self.info.last_update = o.info.last_update;
+        }
+
+        existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap());
+        self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap());
+        let e_hash = self.root_hash();
+        for key in other_root.as_ref().unwrap().children.iter() {
+            let entry = &o.cache[key];
+            let flat = o.flatten(entry);
+            let mut existing = self.cache[key].clone();
+            existing.merge(&flat);
+            self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing);
+        }
+    }
+
+    pub fn root_hash(&self) -> DataUsageHash {
+        hash_path(&self.info.name)
+    }
+
+    pub fn root(&self) -> Option<DataUsageEntry> {
+        self.find(&self.info.name)
+    }
+
+    pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo {
+        let e = match self.find(path) {
+            Some(e) => e,
+            None => return DataUsageInfo::default(),
+        };
+        let flat = self.flatten(&e);
+        let dui = DataUsageInfo {
+            last_update: self.info.last_update,
+            objects_total_count: flat.objects as u64,
+            versions_total_count: flat.versions as u64,
+            delete_markers_total_count: flat.delete_markers as u64,
+            objects_total_size: flat.size as u64,
+            buckets_count: e.children.len() as u64,
+            buckets_usage: self.buckets_usage_info(buckets),
+            ..Default::default()
+        };
+        dui
+    }
+
+    pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap<String, BucketUsageInfo> {
+        let mut dst = HashMap::new();
+        for bucket in buckets.iter() {
+            let e = match self.find(&bucket.name) {
+                Some(e) => e,
+                None => continue,
+            };
+            let flat = self.flatten(&e);
+            let mut bui = BucketUsageInfo {
+                size: flat.size as u64,
+                versions_count: flat.versions as u64,
+                objects_count: flat.objects as u64,
+                delete_markers_count: flat.delete_markers as u64,
+                object_size_histogram: flat.obj_sizes.to_map(),
+                object_versions_histogram: flat.obj_versions.to_map(),
+                ..Default::default()
+            };
+            if let Some(rs) = &flat.replication_stats {
+                bui.replica_size = rs.replica_size;
+                bui.replica_count = rs.replica_count;
+
+                for (arn, stat) in rs.targets.iter() {
+                    bui.replication_info.insert(arn.clone(), BucketTargetUsageInfo {
+                        replication_pending_size: stat.pending_size,
+                        replicated_size: stat.replicated_size,
+                        replication_failed_size: stat.failed_size,
+                        replication_pending_count: stat.pending_count,
+                        replication_failed_count: stat.failed_count,
+                        replicated_count: stat.replicated_count,
+                        ..Default::default()
+                    });
+                }
+            }
+            dst.insert(bucket.name.clone(), bui);
+        }
+        dst
+    }
+
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
         let mut buf = Vec::new();
 

ecstore/src/heal/error.rs

@@ -1 +1,2 @@
 pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage";
+pub const ERR_SKIP_FILE: &str = "skip this file";