tmp(2)

Signed-off-by: junxiang Mu <1948535941@qq.com>

ecstore/src/heal/data_usage_cache.rs

@@ -1,10 +1,16 @@
+use bytesize::ByteSize;
 use http::HeaderMap;
+use path_clean::PathClean;
 use rand::Rng;
 use rmp_serde::Serializer;
+use s3s::dto::ReplicationConfiguration;
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
+use std::hash::{DefaultHasher, Hash, Hasher};
 use std::path::Path;
+use std::sync::Arc;
 use std::time::{Duration, SystemTime};
+use std::u64;
 use s3s::{S3Error, S3ErrorCode};
 use tokio::sync::mpsc::Sender;
 use tokio::time::sleep;
@@ -14,21 +20,162 @@ use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
 use crate::error::{Error, Result};
 use crate::new_object_layer_fn;
 use crate::set_disk::SetDisks;
-use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectOptions, StorageAPI};
+use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectOptions};
 
+use super::data_scanner::SizeSummary;
 use super::data_usage::DATA_USAGE_ROOT;
 
 // DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
 pub const DATA_USAGE_BUCKET_LEN: usize = 11;
 pub const DATA_USAGE_VERSION_LEN: usize = 7;
 
-type DataUsageHashMap = HashSet<String>;
-// sizeHistogram is a size histogram.
-type SizeHistogram = Vec<u64>;
-// versionsHistogram is a histogram of number of versions in an object.
-type VersionsHistogram = Vec<u64>;
+pub type DataUsageHashMap = HashSet<String>;
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+struct ObjectHistogramInterval {
+    name: &'static str,
+    start: u64,
+    end: u64,
+}
+
+const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [
+    ObjectHistogramInterval {
+        name: "LESS_THAN_1024_B",
+        start: 0,
+        end: ByteSize::kib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024_B_AND_64_KB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::kib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_KB_AND_256_KB",
+        start: ByteSize::kib(64).as_u64(),
+        end: ByteSize::kib(256).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_256_KB_AND_512_KB",
+        start: ByteSize::kib(256).as_u64(),
+        end: ByteSize::kib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_512_KB_AND_1_MB",
+        start: ByteSize::kib(512).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024B_AND_1_MB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1_MB_AND_10_MB",
+        start: ByteSize::mib(1).as_u64(),
+        end: ByteSize::mib(10).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_MB_AND_64_MB",
+        start: ByteSize::mib(10).as_u64(),
+        end: ByteSize::mib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_MB_AND_128_MB",
+        start: ByteSize::mib(64).as_u64(),
+        end: ByteSize::mib(128).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_128_MB_AND_512_MB",
+        start: ByteSize::mib(128).as_u64(),
+        end: ByteSize::mib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_512_MB",
+        start: ByteSize::mib(512).as_u64(),
+        end: u64::MAX,
+    },
+];
+
+const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [
+    ObjectHistogramInterval {
+        name: "UNVERSIONED",
+        start: 0,
+        end: 0,
+    },
+    ObjectHistogramInterval {
+        name: "SINGLE_VERSION",
+        start: 1,
+        end: 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_2_AND_10",
+        start: 2,
+        end: 9,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_AND_100",
+        start: 10,
+        end: 99,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_100_AND_1000",
+        start: 100,
+        end: 999,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1000_AND_10000",
+        start: 1000,
+        end: 9999,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_10000",
+        start: 10000,
+        end: u64::MAX,
+    },
+];
+
+// sizeHistogram is a size histogram.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct SizeHistogram(Vec<u64>);
+
+impl Default for SizeHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_BUCKET_LEN])
+    }
+}
+
+impl SizeHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() {
+            if size  >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+}
+
+// versionsHistogram is a histogram of number of versions in an object.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct VersionsHistogram(Vec<u64>);
+
+impl Default for VersionsHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_VERSION_LEN])
+    }
+}
+
+impl VersionsHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() {
+            if size  >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct ReplicationStats {
     pub pending_size: u64,
     pub replicated_size: u64,
@@ -48,7 +195,7 @@ impl ReplicationStats {
     }
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct ReplicationAllStats {
     pub targets: HashMap<String, ReplicationStats>,
     pub replica_size: u64,
@@ -70,35 +217,117 @@ impl ReplicationAllStats {
     }
 }
 
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone, Default, Serialize, Deserialize)]
 pub struct DataUsageEntry {
     pub children: DataUsageHashMap,
     // These fields do no include any children.
-    pub size: i64,
-    pub objects: u64,
-    pub versions: u64,
-    pub delete_markers: u64,
+    pub size: usize,
+    pub objects: usize,
+    pub versions: usize,
+    pub delete_markers: usize,
     pub obj_sizes: SizeHistogram,
     pub obj_versions: VersionsHistogram,
-    pub replication_stats: ReplicationAllStats,
+    pub replication_stats: Option<ReplicationAllStats>,
     // Todo: tier
     // pub all_tier_stats: ,
     pub compacted: bool,
 }
 
+impl DataUsageEntry {
+    pub fn add_child(&mut self, hash: &DataUsageHash) {
+        if self.children.contains(&hash.key()) {
+            return;
+        }
+
+        self.children.insert(hash.key());
+    }
+
+    pub fn add_sizes(&mut self, summary: &SizeSummary) {
+        self.size += summary.total_size;
+        self.versions += summary.versions;
+        self.delete_markers += summary.delete_markers;
+        self.obj_sizes.add(summary.total_size as u64);
+        self.obj_versions.add(summary.versions as u64);
+
+        let replication_stats = if self.replication_stats.is_none() {
+            self.replication_stats = Some(ReplicationAllStats::default());
+            self.replication_stats.as_mut().unwrap()
+        } else {
+            self.replication_stats.as_mut().unwrap()
+        };
+        replication_stats.replica_size += summary.replica_size as u64;
+        replication_stats.replica_count += summary.replica_count as u64;
+
+        for (arn, st) in &summary.repl_target_stats {
+            let tgt_stat = replication_stats.targets.entry(arn.to_string()).or_insert(ReplicationStats::default());
+            tgt_stat.pending_size += st.pending_size as u64;
+            tgt_stat.failed_size += st.failed_size as u64;
+            tgt_stat.replicated_size += st.replicated_size as u64;
+            tgt_stat.replicated_count += st.replicated_count as u64;
+            tgt_stat.failed_count += st.failed_count as u64;
+            tgt_stat.pending_count += st.pending_count as u64;
+        }
+        // Todo:: tiers
+    }
+
+    pub fn merge(&mut self, other: &DataUsageEntry) {
+        self.objects += other.objects;
+        self.versions += other.versions;
+        self.delete_markers += other.delete_markers;
+        self.size += other.size;
+        if let Some(o_rep) = &other.replication_stats {
+            if self.replication_stats.is_none() {
+                self.replication_stats = Some(ReplicationAllStats::default());
+            }
+            let s_rep = self.replication_stats.as_mut().unwrap();
+            s_rep.targets.clear();
+            s_rep.replica_size += o_rep.replica_size;
+            s_rep.replica_count += o_rep.replica_count;
+            for (arn, stat) in o_rep.targets.iter() {
+                let st = s_rep.targets.entry(arn.clone()).or_insert(ReplicationStats::default());
+                *st = ReplicationStats {
+                    pending_size: stat.pending_size + st.pending_size,
+                    failed_size: stat.failed_size + st.failed_size,
+                    replicated_size: stat.replicated_size + st.replicated_size,
+                    pending_count: stat.pending_count + st.pending_count,
+                    failed_count: stat.failed_count + st.failed_count,
+                    replicated_count: stat.replicated_count + st.replicated_count,
+                    ..Default::default()
+                };
+            }
+        }
+
+        for (i, v) in other.obj_sizes.0.iter().enumerate() {
+            self.obj_sizes.0[i] += v;
+        }
+
+        for (i, v) in other.obj_versions.0.iter().enumerate() {
+            self.obj_versions.0[i] += v;
+        }
+
+        // todo: tiers
+    }
+}
+
+#[derive(Clone)]
+pub struct DataUsageEntryInfo {
+    pub name: String,
+    pub parent: String,
+    pub entry: DataUsageEntry,
+}
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct DataUsageCacheInfo {
     pub name: String,
-    pub next_cycle: usize,
+    pub next_cycle: u32,
     pub last_update: SystemTime,
     pub skip_healing: bool,
     // todo: life_cycle
     // pub life_cycle:
     #[serde(skip)]
     pub updates: Option<Sender<DataUsageEntry>>,
-    // Todo: replication
-    // #[serde(skip_serializing)]
-    // replication:
+    #[serde(skip)]
+    pub replication: Option<ReplicationConfiguration>,
 }
 
 impl Default for DataUsageCacheInfo {
@@ -109,6 +338,7 @@ impl Default for DataUsageCacheInfo {
             last_update: SystemTime::now(),
             skip_healing: Default::default(),
             updates: Default::default(),
+            replication: Default::default(),
         }
     }
 }
@@ -202,6 +432,195 @@ impl DataUsageCache {
     
     pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
         let hash = hash_path(path);
+        self.cache.insert(hash.key(), e);
+        if !parent.is_empty() {
+            let phash = hash_path(parent);
+            let p = {
+                let p = self.cache.entry(phash.key()).or_insert(DataUsageEntry::default());
+                p.add_child(&hash);
+                p.clone()
+            };
+            self.cache.insert(phash.key(), p);
+        }
+    }
+
+    pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
+        self.cache.insert(hash.key(), e.clone());
+        if let Some(parent) = parent {
+            self.cache.entry(parent.key()).or_insert(DataUsageEntry::default()).add_child(hash);
+        }
+    }
+
+    pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
+        self.cache.get(&hash_path(path).key()).cloned()
+    }
+
+    pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
+        self.cache.entry(h.string()).or_insert(DataUsageEntry::default()).children.clone()
+    }
+
+    pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
+        let mut root = root.clone();
+        for id in root.children.clone().iter() {
+            if let Some(e) = self.cache.get(id) {
+                let mut e = e.clone();
+                if !e.children.is_empty() {
+                    e = self.flatten(&e);
+                }
+                root.merge(&e);
+            }
+        }
+        root.children.clear();
+        root
+    }
+
+    pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
+        match src.cache.get(&hash.string()) {
+            Some(e) => {
+                self.cache.insert(hash.key(), e.clone());
+                for ch in e.children.iter() {
+                    if *ch == hash.key() {
+                        return;
+                    }
+                    self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
+                }
+                if let Some(parent) = parent {
+                    let p = self.cache.entry(parent.key()).or_insert(DataUsageEntry::default());
+                    p.add_child(hash);
+                }
+            },
+            None => return,
+        }
+    }
+
+    pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
+        let mut need_remove = Vec::new();
+        if let Some(v) = self.cache.get(&hash.string()) {
+            for child in v.children.iter() {
+                need_remove.push(child.clone());
+            }
+        }
+        self.cache.remove(&hash.string());
+        need_remove.iter().for_each(|child| {
+            self.delete_recursive(&DataUsageHash(child.to_string()));
+        });
+    }
+
+    pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
+        match self.find(path) {
+            Some(root) => {
+                if root.children.is_empty() {
+                    return None;
+                }
+                let mut flat = self.flatten(&root);
+                if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() {
+                    flat.replication_stats = None;
+                }
+                return Some(flat);
+            },
+            None => None
+        }
+    }
+
+    pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
+        let want = hash.key();
+        if let Some(last_index) = want.rfind('/') {
+            if let Some(v) = self.find(&want[0..last_index]) {
+                if v.children.contains(&want) {
+                    let found = hash_path(&want[0..last_index]);
+                    return Some(found);
+                }
+            }
+        }
+
+        for (k, v) in self.cache.iter() {
+            if v.children.contains(&want) {
+                let found = DataUsageHash(k.clone());
+                return Some(found);
+            }
+        }
+        None
+    }
+
+    pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
+        match self.cache.get(&hash.key()) {
+            Some(due) => {
+                due.compacted
+            },
+            None => false
+        }
+    }
+
+    pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
+        let e = match self.cache.get(&path.key()) {
+            Some(e) => e,
+            None => return,
+        };
+
+        if e.compacted {
+            return;
+        }
+
+        if e.children.len() > limit && compact_self {
+            let mut flat = self.size_recursive(&path.key()).unwrap_or(DataUsageEntry::default());
+            flat.compacted = true;
+            self.delete_recursive(path);
+            self.replace_hashed(path, &None, &flat);
+            return;
+        }
+        let total = self.total_children_rec(&path.key());
+        if total < limit {
+            return;
+        }
+
+        let mut leaves = Vec::new();
+        let mut remove = total - limit;
+        add(self, path, &mut leaves);
+        leaves.sort_by(|a, b| {
+            a.objects.cmp(&b.objects)
+        });
+
+        while remove > 0 && leaves.len() > 0 {
+            let e = leaves.first().unwrap();
+            let candidate = e.path.clone();
+            if candidate == *path && !compact_self {
+                break;
+            }
+            let removing = self.total_children_rec(&candidate.key());
+            let mut flat = match self.size_recursive(&candidate.key()) {
+                Some(flat) => flat,
+                None => {
+                    leaves.remove(0);
+                    continue;
+                }
+            };
+
+            flat.compacted = true;
+            self.delete_recursive(&candidate);
+            self.replace_hashed(&candidate, &None, &flat);
+
+            remove -= removing;
+            leaves.remove(0);
+        }
+
+    }
+
+    pub fn total_children_rec(&self, path: &str) -> usize {
+        let root = self.find(path);
+
+        if root.is_none() {
+            return 0;
+        }
+        let root = root.unwrap();
+        if root.children.is_empty() {
+            return 0;
+        }
+
+        let mut n = root.children.len();
+        for ch in root.children.iter() {
+            n += self.total_children_rec(&ch);
+        }
+        n
     }
 
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
@@ -218,10 +637,63 @@ impl DataUsageCache {
     }
 }
 
-struct DataUsageHash(String);
+#[derive(Default, Clone)]
+struct Inner {
+    objects: usize,
+    path: DataUsageHash,
+}
+
+fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec<Inner>) {
+    let e = match data_usage_cache.cache.get(&path.key()) {
+        Some(e) => e,
+        None => return,
+    };
+    if !e.children.is_empty() {
+        return;
+    }
+
+    let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or(DataUsageEntry::default());
+    leaves.push(Inner{objects: sz.objects, path: path.clone()});
+    for ch in e.children.iter() {
+        add(data_usage_cache, &DataUsageHash(ch.clone()), leaves);
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct DataUsageHash(pub String);
 
 impl DataUsageHash {
-    
+    pub fn string(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn key(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn mod_(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        hash as u32 % cycles == cycle % cycles
+    }
+
+    pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        (hash >> 32) as u32 % cycles == cycle % cycles
+    }
+
+    fn calculate_hash(&self) -> u64 {
+        let mut hasher = DefaultHasher::new();
+        self.0.hash(&mut hasher);
+        hasher.finish()
+    }
 }
 
 pub fn hash_path(data: &str) -> DataUsageHash {
@@ -229,6 +701,5 @@ pub fn hash_path(data: &str) -> DataUsageHash {
     if data != DATA_USAGE_ROOT {
         data = data.trim_matches('/');
     }
-    Path::new(&data);
-    todo!()
+    DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
 }