tmp

ecstore/src/heal/background_heal_ops.rs

@@ -1,14 +1,16 @@
-use std::sync::Arc;
+use std::{env, sync::Arc};
 
 use tokio::{
     select,
     sync::{
         broadcast::Receiver as B_Receiver,
-        mpsc::{self, Receiver, Sender},
+        mpsc::{self, Receiver, Sender}, RwLock,
     },
 };
 
-use crate::{error::Error, heal::heal_ops::NOP_HEAL, utils::path::SLASH_SEPARATOR};
+use crate::{
+    disk::error::DiskError, error::{Error, Result}, heal::heal_ops::NOP_HEAL, new_object_layer_fn, store_api::StorageAPI, utils::path::SLASH_SEPARATOR
+};
 
 use super::{
     heal_commands::{HealOpts, HealResultItem},
@@ -21,8 +23,8 @@ pub struct HealTask {
     pub object: String,
     pub version_id: String,
     pub opts: HealOpts,
-    pub resp_tx: Arc<Sender<HealResult>>,
-    pub resp_rx: Arc<Receiver<HealResult>>,
+    pub resp_tx: Option<Arc<Sender<HealResult>>>,
+    pub resp_rx: Option<Arc<Receiver<HealResult>>>,
 }
 
 impl HealTask {
@@ -33,15 +35,15 @@ impl HealTask {
             object: object.to_string(),
             version_id: version_id.to_string(),
             opts: opts.clone(),
-            resp_tx: tx.into(),
-            resp_rx: rx.into(),
+            resp_tx: Some(tx.into()),
+            resp_rx: Some(rx.into()),
         }
     }
 }
 
 pub struct HealResult {
     pub result: HealResultItem,
-    err: Error,
+    err: Option<Error>,
 }
 
 pub struct HealRoutine {
@@ -51,18 +53,78 @@ pub struct HealRoutine {
 }
 
 impl HealRoutine {
-    pub async fn add_worker(&mut self, mut ctx: B_Receiver<bool>, bgseq: &HealSequence) {
+    pub fn new() -> Arc<RwLock<Self>> {
+        let mut workers = num_cpus::get() / 2;
+        if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") {
+            if let Ok(num_healers) = env_heal_workers.parse::<usize>() {
+                workers = num_healers;
+            }
+        }
+
+        if workers == 0 {
+            workers = 4;
+        }
+
+        let (tx, rx) = mpsc::channel(100);
+        Arc::new(RwLock::new(Self {
+            tasks_tx: tx,
+            tasks_rx: rx,
+            workers,
+        }))
+    }
+
+    pub async fn add_worker(&mut self, mut ctx: B_Receiver<bool>, bgseq: &mut HealSequence) {
         loop {
             select! {
                 task = self.tasks_rx.recv() => {
-                    let mut res = HealResultItem::default();
-                    let mut err: Error;
+                    let mut d_res = HealResultItem::default();
+                    let d_err: Option<Error>;
                     match task {
                         Some(task) => {
                             if task.bucket == NOP_HEAL {
-                                err = Error::from_string("skip file");
+                                d_err = Some(Error::from_string("skip file"));
                             } else if task.bucket == SLASH_SEPARATOR {
-                                (res, err) = heal_disk_format(task.opts).await;
+                                match heal_disk_format(task.opts).await {
+                                    Ok((res, err)) => {
+                                        d_res = res;
+                                        d_err = err;
+                                    },
+                                    Err(err) => {d_err = Some(err)},
+                                }
+                            } else {
+                                let layer = new_object_layer_fn();
+                                let lock = layer.read().await;
+                                let store = lock
+                                    .as_ref()
+                                    .expect("Not init");
+                                if task.object.is_empty() {
+                                    match store.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts).await {
+                                        Ok((res, err)) => {
+                                            d_res = res;
+                                            d_err = err;
+                                        },
+                                        Err(err) => {d_err = Some(err)},
+                                    }
+                                } else {
+                                    match store.heal_object(&task.bucket, &task.object, &task.version_id, &task.opts).await {
+                                        Ok((res, err)) => {
+                                            d_res = res;
+                                            d_err = err;
+                                        },
+                                        Err(err) => {d_err = Some(err)},
+                                    }
+                                }
+                            }
+                            if let Some(resp_tx) = task.resp_tx {
+                                let _ = resp_tx.send(HealResult{result: d_res, err: d_err}).await;
+                            } else {
+                                // when respCh is not set caller is not waiting but we
+                                // update the relevant metrics for them
+                                if d_err.is_none() {
+                                    bgseq.count_healed(d_res.heal_item_type);
+                                } else {
+                                    bgseq.count_failed(d_res.heal_item_type);
+                                }
                             }
                         },
                         None => return,
@@ -80,6 +142,15 @@ impl HealRoutine {
 
 // }
 
-async fn heal_disk_format(opts: HealOpts) -> (HealResultItem, Error) {
-    todo!()
+async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Error>)> {
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = lock.as_ref().expect("Not init");
+    let (res, err) = store.heal_format(opts.dry_run).await?;
+    // return any error, ignore error returned when disks have
+	// already healed.
+    if err.is_some() {
+        return Ok((HealResultItem::default(), err));
+    }
+    return Ok((res, err));
 }

ecstore/src/heal/data_scanner.rs

@@ -0,0 +1,293 @@
+use std::{
+    io::{Cursor, Read},
+    sync::{atomic::{AtomicU32, AtomicU64}, Arc},
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use byteorder::{LittleEndian, ReadBytesExt};
+use lazy_static::lazy_static;
+use rand::Rng;
+use rmp_serde::{Deserializer, Serializer};
+use serde::{Deserialize, Serialize};
+use tokio::{sync::{mpsc, RwLock}, time::sleep};
+use tracing::{error, info};
+
+use crate::{
+    config::{common::{read_config, save_config}, heal::Config},
+    error::{Error, Result},
+    global::GLOBAL_IsErasureSD,
+    heal::data_usage::BACKGROUND_HEAL_INFO_PATH,
+    new_object_layer_fn,
+    store::ECStore,
+};
+
+use super::{data_scanner_metric::globalScannerMetrics, data_usage::{store_data_usage_in_backend, DATA_USAGE_BLOOM_NAME_PATH}, heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN}};
+
+const DATA_SCANNER_SLEEP_PER_FOLDER: Duration = Duration::from_millis(1); // Time to wait between folders.
+const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; // Visit all folders every n cycles.
+const DATA_SCANNER_COMPACT_LEAST_OBJECT: u64 = 500; // Compact when there are less than this many objects in a branch.
+const DATA_SCANNER_COMPACT_AT_CHILDREN: u64 = 10000; // Compact when there are this many children in a branch.
+const DATA_SCANNER_COMPACT_AT_FOLDERS: u64 = DATA_SCANNER_COMPACT_AT_CHILDREN / 4; // Compact when this many subfolders in a single folder.
+const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: u64 = 250_000; // Compact when this many subfolders in a single folder (even top level).
+const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(60); // Time to wait on startup and between cycles.
+
+const HEAL_DELETE_DANGLING: bool = true;
+const HEAL_OBJECT_SELECT_PROB: u64 = 1024; // Overall probability of a file being scanned; one in n.
+
+// static SCANNER_SLEEPER: () = new_dynamic_sleeper(2, Duration::from_secs(1), true); // Keep defaults same as config defaults
+static SCANNER_CYCLE: AtomicU64 = AtomicU64::new(DATA_SCANNER_START_DELAY.as_secs());
+static SCANNER_IDLE_MODE: AtomicU32 = AtomicU32::new(0); // default is throttled when idle
+static SCANNER_EXCESS_OBJECT_VERSIONS: AtomicU64 = AtomicU64::new(100);
+static SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE: AtomicU64 = AtomicU64::new(1024 * 1024 * 1024 * 1024); // 1 TB
+static SCANNER_EXCESS_FOLDERS: AtomicU64 = AtomicU64::new(50_000);
+
+lazy_static! {
+    pub static ref globalHealConfig: Arc<RwLock<Config>> = Arc::new(RwLock::new(Config::default()));
+}
+
+pub async fn init_data_scanner() {
+    let mut r = rand::thread_rng();
+    let random = r.gen_range(0.0..1.0);
+    tokio::spawn(async move {
+        loop {
+            run_data_scanner().await;
+            let duration = Duration::from_secs_f64(random * (SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst) as f64));
+            let sleep_duration = if duration < Duration::new(1, 0) {
+                Duration::new(1, 0)
+            } else {
+                duration
+            };
+            sleep(sleep_duration).await;
+        }
+    });
+}
+
+async fn run_data_scanner() {
+    let mut cycle_info = CurrentScannerCycle::default();
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => {
+            info!("errServerNotInitialized");
+            return;
+        }
+    };
+    let mut buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
+        .await
+        .map_or(Vec::new(), |buf| buf);
+    if buf.len() == 8 {
+        cycle_info.next = match Cursor::new(buf).read_u64::<LittleEndian>() {
+            Ok(buf) => buf,
+            Err(_) => {
+                error!("can not decode DATA_USAGE_BLOOM_NAME_PATH");
+                return;
+            }
+        };
+    } else if buf.len() > 8 {
+        cycle_info.next = match Cursor::new(buf[..8].to_vec()).read_u64::<LittleEndian>() {
+            Ok(buf) => buf,
+            Err(_) => {
+                error!("can not decode DATA_USAGE_BLOOM_NAME_PATH");
+                return;
+            }
+        };
+        let _ = cycle_info.unmarshal_msg(&buf.split_off(8));
+    }
+
+    loop {
+        cycle_info.current = cycle_info.next;
+        cycle_info.started = SystemTime::now();
+        {
+            globalScannerMetrics.write().await.set_cycle(Some(cycle_info.clone())).await;
+        }
+
+        let bg_heal_info = read_background_heal_info(store).await;
+        let scan_mode = get_cycle_scan_mode(cycle_info.current, bg_heal_info.bitrot_start_cycle, bg_heal_info.bitrot_start_time).await;
+        if bg_heal_info.current_scan_mode != scan_mode {
+            let mut new_heal_info = bg_heal_info;
+            new_heal_info.current_scan_mode = scan_mode;
+            if scan_mode == HEAL_DEEP_SCAN {
+                new_heal_info.bitrot_start_time = SystemTime::now();
+                new_heal_info.bitrot_start_cycle = cycle_info.current;
+            }
+            save_background_heal_info(store, &new_heal_info).await;
+        }
+        // Wait before starting next cycle and wait on startup.
+        let (tx, rx) = mpsc::channel(100);
+        tokio::spawn(async {
+            store_data_usage_in_backend(rx).await;
+        });
+        sleep(Duration::from_secs(SCANNER_CYCLE.load(std::sync::atomic::Ordering::SeqCst))).await;
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct BackgroundHealInfo {
+    bitrot_start_time: SystemTime,
+    bitrot_start_cycle: u64,
+    current_scan_mode: HealScanMode,
+}
+
+impl Default for BackgroundHealInfo {
+    fn default() -> Self {
+        Self {
+            bitrot_start_time: SystemTime::now(),
+            bitrot_start_cycle: Default::default(),
+            current_scan_mode: Default::default(),
+        }
+    }
+}
+
+async fn read_background_heal_info(store: &ECStore) -> BackgroundHealInfo {
+    if *GLOBAL_IsErasureSD.read().await {
+        return BackgroundHealInfo::default();
+    }
+
+    let buf = read_config(store, &BACKGROUND_HEAL_INFO_PATH)
+        .await
+        .map_or(Vec::new(), |buf| buf);
+    if buf.is_empty() {
+        return BackgroundHealInfo::default();
+    }
+    serde_json::from_slice::<BackgroundHealInfo>(&buf).map_or(BackgroundHealInfo::default(), |b| b)
+}
+
+async fn save_background_heal_info(store: &ECStore, info: &BackgroundHealInfo) {
+    if *GLOBAL_IsErasureSD.read().await {
+        return;
+    }
+    let b = match serde_json::to_vec(info) {
+        Ok(info) => info,
+        Err(_) => return,
+    };
+    let _ = save_config(store, &BACKGROUND_HEAL_INFO_PATH, &b).await;
+}
+
+async fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: SystemTime) -> HealScanMode {
+    let bitrot_cycle = globalHealConfig.read().await.bitrot_scan_cycle();
+    let v = bitrot_cycle.as_secs_f64() ;
+    if v == -1.0 {
+        return HEAL_NORMAL_SCAN;
+    } else if v == 0.0 {
+        return HEAL_DEEP_SCAN;
+    }
+
+    if current_cycle - bitrot_start_cycle < HEAL_OBJECT_SELECT_PROB {
+        return HEAL_DEEP_SCAN;
+    }
+
+    if bitrot_start_time.duration_since(SystemTime::now()).unwrap() > bitrot_cycle {
+        return HEAL_DEEP_SCAN;
+    }
+
+    HEAL_NORMAL_SCAN
+}
+
+#[derive(Clone, Debug)]
+pub struct CurrentScannerCycle {
+    pub current: u64,
+    pub next: u64,
+    pub started: SystemTime,
+    pub cycle_completed: Vec<SystemTime>,
+}
+
+impl Default for CurrentScannerCycle {
+    fn default() -> Self {
+        Self {
+            current: Default::default(),
+            next: Default::default(),
+            started: SystemTime::now(),
+            cycle_completed: Default::default(),
+        }
+    }
+}
+
+impl CurrentScannerCycle {
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let len: u32 = 4;
+        let mut wr = Vec::new();
+
+        // 字段数量
+        rmp::encode::write_map_len(&mut wr, len)?;
+
+        // write "current"
+        rmp::encode::write_str(&mut wr, "current")?;
+        rmp::encode::write_uint(&mut wr, self.current)?;
+
+        // write "next"
+        rmp::encode::write_str(&mut wr, "next")?;
+        rmp::encode::write_uint(&mut wr, self.next)?;
+
+        // write "started"
+        rmp::encode::write_str(&mut wr, "started")?;
+        rmp::encode::write_uint(&mut wr, system_time_to_timestamp(&self.started))?;
+
+        // write "cycle_completed"
+        rmp::encode::write_str(&mut wr, "cycle_completed")?;
+        let mut buf = Vec::new();
+        self.cycle_completed
+            .serialize(&mut Serializer::new(&mut buf))
+            .expect("Serialization failed");
+        rmp::encode::write_bin(&mut wr, &buf)?;
+
+        Ok(wr)
+    }
+
+    #[tracing::instrument]
+    pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> {
+        let mut cur = Cursor::new(buf);
+
+        let mut fields_len = rmp::decode::read_map_len(&mut cur)?;
+
+        while fields_len > 0 {
+            fields_len -= 1;
+
+            let str_len = rmp::decode::read_str_len(&mut cur)?;
+
+            // ！！！ Vec::with_capacity(str_len) 失败，vec!正常
+            let mut field_buff = vec![0u8; str_len as usize];
+
+            cur.read_exact(&mut field_buff)?;
+
+            let field = String::from_utf8(field_buff)?;
+
+            match field.as_str() {
+                "current" => {
+                    let u: u64 = rmp::decode::read_int(&mut cur)?;
+                    self.current = u;
+                }
+
+                "next" => {
+                    let u: u64 = rmp::decode::read_int(&mut cur)?;
+                    self.next = u;
+                }
+                "started" => {
+                    let u: u64 = rmp::decode::read_int(&mut cur)?;
+                    let started = timestamp_to_system_time(u);
+                    self.started = started;
+                }
+                "cycleCompleted" => {
+                    let mut buf = Vec::new();
+                    let _ = cur.read_to_end(&mut buf)?;
+                    let u: Vec<SystemTime> =
+                        Deserialize::deserialize(&mut Deserializer::new(&buf[..])).expect("Deserialization failed");
+                    self.cycle_completed = u;
+                }
+                name => return Err(Error::msg(format!("not suport field name {}", name))),
+            }
+        }
+
+        Ok(cur.position())
+    }
+}
+
+// 将 SystemTime 转换为时间戳
+fn system_time_to_timestamp(time: &SystemTime) -> u64 {
+    time.duration_since(UNIX_EPOCH).expect("Time went backwards").as_secs()
+}
+
+// 将时间戳转换为 SystemTime
+fn timestamp_to_system_time(timestamp: u64) -> SystemTime {
+    UNIX_EPOCH + std::time::Duration::new(timestamp, 0)
+}

ecstore/src/heal/data_scanner_metric.rs

@@ -0,0 +1,74 @@
+use std::{
+    collections::HashMap,
+    sync::{atomic::{AtomicU32, Ordering}, Arc},
+    time::SystemTime,
+};
+
+use lazy_static::lazy_static;
+use tokio::sync::RwLock;
+
+use super::data_scanner::CurrentScannerCycle;
+
+lazy_static! {
+    pub static ref globalScannerMetrics: Arc<RwLock<ScannerMetrics>> = Arc::new(RwLock::new(ScannerMetrics::new()));
+}
+
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
+pub enum ScannerMetric {
+    // START Realtime metrics, that only to records
+    // last minute latencies and total operation count.
+    ReadMetadata = 0,
+    CheckMissing,
+    SaveUsage,
+    ApplyAll,
+    ApplyVersion,
+    TierObjSweep,
+    HealCheck,
+    Ilm,
+    CheckReplication,
+    Yield,
+    CleanAbandoned,
+    ApplyNonCurrent,
+    HealAbandonedVersion,
+
+    // START Trace metrics:
+    StartTrace,
+    ScanObject, // Scan object. All operations included.
+    HealAbandonedObject,
+
+    // END realtime metrics:
+    LastRealtime,
+
+    // Trace only metrics:
+    ScanFolder,      // Scan a folder on disk, recursively.
+    ScanCycle,       // Full cycle, cluster global.
+    ScanBucketDrive, // Single bucket on one drive.
+    CompactFolder,   // Folder compacted.
+
+    // Must be last:
+    Last,
+}
+
+pub struct ScannerMetrics {
+    operations: Vec<AtomicU32>,
+    cycle_info: RwLock<Option<CurrentScannerCycle>>,
+}
+
+impl ScannerMetrics {
+    pub fn new() -> Self {
+        Self {
+            operations: (0..ScannerMetric::Last as usize).map(|_| AtomicU32::new(0)).collect(),
+            cycle_info: RwLock::new(None),
+        }
+    }
+
+    pub fn log(&mut self, s: ScannerMetric, _paths: &[String], _custom: &HashMap<String, String>, _start_time: SystemTime) {
+        // let duration = start_time.duration_since(start_time);
+        self.operations[s.clone() as usize].fetch_add(1, Ordering::SeqCst);
+        // Dodo
+    }
+
+    pub async fn set_cycle(&mut self, c: Option<CurrentScannerCycle>) {
+        *self.cycle_info.write().await = c;
+    }
+}

ecstore/src/heal/data_usage.rs

@@ -0,0 +1,158 @@
+use std::{collections::HashMap, time::SystemTime};
+
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use tokio::sync::mpsc::Receiver;
+use tracing::info;
+
+use crate::{
+    config::common::save_config,
+    disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
+    new_object_layer_fn,
+    utils::path::SLASH_SEPARATOR,
+};
+
+pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
+const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
+const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
+pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
+lazy_static! {
+    pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX);
+    pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME);
+    pub static ref DATA_USAGE_BLOOM_NAME_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME);
+    pub static ref BACKGROUND_HEAL_INFO_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json");
+}
+
+// BucketTargetUsageInfo - bucket target usage info provides
+// - replicated size for all objects sent to this target
+// - replica size for all objects received from this target
+// - replication pending size for all objects pending replication to this target
+// - replication failed size for all objects failed replication to this target
+// - replica pending count
+// - replica failed count
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketTargetUsageInfo {
+    replication_pending_size: u64,
+    replication_failed_size: u64,
+    replicated_size: u64,
+    replica_size: u64,
+    replication_pending_count: u64,
+    replication_failed_count: u64,
+    replicated_count: u64,
+}
+
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketUsageInfo {
+    size: u64,
+    // Following five fields suffixed with V1 are here for backward compatibility
+    // Total Size for objects that have not yet been replicated
+    replication_pending_size_v1: u64,
+    // Total size for objects that have witness one or more failures and will be retried
+    replication_failed_size_v1: u64,
+    // Total size for objects that have been replicated to destination
+    replicated_size_v1: u64,
+    // Total number of objects pending replication
+    replication_pending_count_v1: u64,
+    // Total number of objects that failed replication
+    replication_failed_count_v1: u64,
+
+    objects_count: u64,
+    object_size_histogram: HashMap<String, u64>,
+    object_versions_histogram: HashMap<String, u64>,
+    versions_count: u64,
+    delete_markers_count: u64,
+    replica_size: u64,
+    replica_count: u64,
+    replication_info: HashMap<String, BucketTargetUsageInfo>,
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
+#[derive(Debug, Serialize, Deserialize)]
+pub struct DataUsageInfo {
+    total_capacity: u64,
+    total_used_capacity: u64,
+    total_free_capacity: u64,
+
+    // LastUpdate is the timestamp of when the data usage info was last updated.
+    // This does not indicate a full scan.
+    last_update: SystemTime,
+
+    // Objects total count across all buckets
+    objects_total_count: u64,
+    // Versions total count across all buckets
+    versions_total_count: u64,
+    // Delete markers total count across all buckets
+    delete_markers_total_count: u64,
+    // Objects total size across all buckets
+    objects_total_size: u64,
+    replication_info: HashMap<String, BucketTargetUsageInfo>,
+
+    // Total number of buckets in this cluster
+    buckets_count: u64,
+    // Buckets usage info provides following information across all buckets
+    // - total size of the bucket
+    // - total objects in a bucket
+    // - object size histogram per bucket
+    buckets_usage: HashMap<String, BucketUsageInfo>,
+    // Deprecated kept here for backward compatibility reasons.
+    bucket_sizes: HashMap<String, u64>,
+    // Todo: TierStats
+    // TierStats contains per-tier stats of all configured remote tiers
+}
+
+impl Default for DataUsageInfo {
+    fn default() -> Self {
+        Self {
+            total_capacity: Default::default(),
+            total_used_capacity: Default::default(),
+            total_free_capacity: Default::default(),
+            last_update: SystemTime::now(),
+            objects_total_count: Default::default(),
+            versions_total_count: Default::default(),
+            delete_markers_total_count: Default::default(),
+            objects_total_size: Default::default(),
+            replication_info: Default::default(),
+            buckets_count: Default::default(),
+            buckets_usage: Default::default(),
+            bucket_sizes: Default::default(),
+        }
+    }
+}
+
+pub async fn store_data_usage_in_backend(mut rx: Receiver<DataUsageInfo>) {
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => {
+            info!("errServerNotInitialized");
+            return;
+        }
+    };
+    let mut attempts = 1;
+    loop {
+        match rx.recv().await {
+            Some(data_usage_info) => {
+                if let Ok(data) = serde_json::to_vec(&data_usage_info) {
+                    if attempts > 10 {
+                        let _ = save_config(store, &format!("{}{}", DATA_USAGE_OBJ_NAME_PATH.to_string(), ".bkp"), &data).await;
+                        attempts += 1;
+                    }
+                    let _ = save_config(store, &DATA_USAGE_OBJ_NAME_PATH, &data).await;
+                    attempts += 1;
+                } else {
+                    continue;
+                }
+            }
+            None => {
+                return;
+            }
+        }
+    }
+}

ecstore/src/heal/data_usage_cache.rs

@@ -0,0 +1,234 @@
+use http::HeaderMap;
+use rand::Rng;
+use rmp_serde::Serializer;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::path::Path;
+use std::time::{Duration, SystemTime};
+use s3s::{S3Error, S3ErrorCode};
+use tokio::sync::mpsc::Sender;
+use tokio::time::sleep;
+use crate::config::common::save_config;
+use crate::disk::error::DiskError;
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
+use crate::set_disk::SetDisks;
+use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectOptions, StorageAPI};
+
+use super::data_usage::DATA_USAGE_ROOT;
+
+// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
+pub const DATA_USAGE_BUCKET_LEN: usize = 11;
+pub const DATA_USAGE_VERSION_LEN: usize = 7;
+
+type DataUsageHashMap = HashSet<String>;
+// sizeHistogram is a size histogram.
+type SizeHistogram = Vec<u64>;
+// versionsHistogram is a histogram of number of versions in an object.
+type VersionsHistogram = Vec<u64>;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicationStats {
+    pub pending_size: u64,
+    pub replicated_size: u64,
+    pub failed_size: u64,
+    pub failed_count: u64,
+    pub pending_count: u64,
+    pub missed_threshold_size: u64,
+    pub after_threshold_size: u64,
+    pub missed_threshold_count: u64,
+    pub after_threshold_count: u64,
+    pub replicated_count: u64,
+}
+
+impl ReplicationStats {
+    pub fn empty(&self) -> bool {
+        self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ReplicationAllStats {
+    pub targets: HashMap<String, ReplicationStats>,
+    pub replica_size: u64,
+    pub replica_count: u64,
+}
+
+impl ReplicationAllStats {
+    pub fn empty(&self) -> bool {
+        if self.replica_size != 0 && self.replica_count != 0 {
+            return false;
+        }
+        for (_, v) in self.targets.iter() {
+            if !v.empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct DataUsageEntry {
+    pub children: DataUsageHashMap,
+    // These fields do no include any children.
+    pub size: i64,
+    pub objects: u64,
+    pub versions: u64,
+    pub delete_markers: u64,
+    pub obj_sizes: SizeHistogram,
+    pub obj_versions: VersionsHistogram,
+    pub replication_stats: ReplicationAllStats,
+    // Todo: tier
+    // pub all_tier_stats: ,
+    pub compacted: bool,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct DataUsageCacheInfo {
+    pub name: String,
+    pub next_cycle: usize,
+    pub last_update: SystemTime,
+    pub skip_healing: bool,
+    // todo: life_cycle
+    // pub life_cycle:
+    #[serde(skip)]
+    pub updates: Option<Sender<DataUsageEntry>>,
+    // Todo: replication
+    // #[serde(skip_serializing)]
+    // replication:
+}
+
+impl Default for DataUsageCacheInfo {
+    fn default() -> Self {
+        Self {
+            name: Default::default(),
+            next_cycle: Default::default(),
+            last_update: SystemTime::now(),
+            skip_healing: Default::default(),
+            updates: Default::default(),
+        }
+    }
+}
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct DataUsageCache {
+    pub info: DataUsageCacheInfo,
+    pub cache: HashMap<String, DataUsageEntry>,
+}
+
+impl DataUsageCache {
+    pub async fn load(store: &SetDisks, name: &str) -> Result<Self> {
+        let mut d = DataUsageCache::default();
+        let mut retries = 0;
+        while retries < 5 {
+            let path = Path::new(BUCKET_META_PREFIX).join(name);
+            match store
+                .get_object_reader(
+                    RUSTFS_META_BUCKET,
+                    path.to_str().unwrap(),
+                    HTTPRangeSpec::nil(),
+                    HeaderMap::new(),
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                Ok(mut reader) => {
+                    if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                        d = info
+                    }
+                    break;
+                }
+                Err(err) => match err.downcast_ref::<DiskError>() {
+                    Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
+                        match store
+                            .get_object_reader(
+                                RUSTFS_META_BUCKET,
+                                name,
+                                HTTPRangeSpec::nil(),
+                                HeaderMap::new(),
+                                &ObjectOptions {
+                                    no_lock: true,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                        {
+                            Ok(mut reader) => {
+                                if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                                    d = info
+                                }
+                                break;
+                            }
+                            Err(_) => match err.downcast_ref::<DiskError>() {
+                                Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
+                                    break;
+                                }
+                                _ => {}
+                            },
+                        }
+                    }
+                    _ => {}
+                },
+            }
+            retries += 1;
+            let mut rng = rand::thread_rng();
+            sleep(Duration::from_millis(rng.gen_range(0..1_000))).await;
+        }
+        Ok(d)
+    }
+    
+    pub async fn save(&self, name: &str) -> Result<()> {
+        let buf = self.marshal_msg()?;
+        let buf_clone = buf.clone();
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
+        };
+        let store_clone = store.clone();
+        let name_clone = name.to_string();
+        tokio::spawn(async move {
+            let _ = save_config(&store_clone, &format!("{}{}", &name_clone, ".bkp"), &buf_clone).await;
+        });
+        save_config(&store, name, &buf).await
+    }
+    
+    pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
+        let hash = hash_path(path);
+    }
+
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut Serializer::new(&mut buf))?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: Self = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+}
+
+struct DataUsageHash(String);
+
+impl DataUsageHash {
+    
+}
+
+pub fn hash_path(data: &str) -> DataUsageHash {
+    let mut data = data;
+    if data != DATA_USAGE_ROOT {
+        data = data.trim_matches('/');
+    }
+    Path::new(&data);
+    todo!()
+}

ecstore/src/heal/heal_commands.rs

@@ -7,12 +7,7 @@ use serde::{Deserialize, Serialize};
 use tokio::sync::RwLock;
 
 use crate::{
-    disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
-    error::{Error, Result},
-    heal::heal_ops::HEALING_TRACKER_FILENAME,
-    new_object_layer_fn,
-    store_api::{BucketInfo, StorageAPI},
-    utils::fs::read_file,
+    disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, error::{Error, Result}, global::GLOBAL_BackgroundHealState, heal::heal_ops::HEALING_TRACKER_FILENAME, new_object_layer_fn, store_api::{BucketInfo, StorageAPI}, utils::fs::read_file
 };
 
 pub type HealScanMode = usize;
@@ -50,7 +45,7 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct HealDriveInfo {
     pub uuid: String,
     pub endpoint: String,
@@ -259,7 +254,7 @@ impl HealingTracker {
 
         let htracker_bytes = self.marshal_msg()?;
 
-        // TODO: globalBackgroundHealState
+        GLOBAL_BackgroundHealState.write().await.update_heal_status(&self).await;
 
         if let Some(disk) = &self.disk {
             let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
@@ -430,10 +425,10 @@ async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker
     }
 }
 
-async fn init_healing_tracker(disk: DiskStore, heal_id: String) -> Result<HealingTracker> {
+pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> Result<HealingTracker> {
     let mut healing_tracker = HealingTracker::default();
     healing_tracker.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string());
-    healing_tracker.heal_id = heal_id;
+    healing_tracker.heal_id = heal_id.to_string();
     healing_tracker.path = disk.to_string();
     healing_tracker.endpoint = disk.endpoint().to_string();
     healing_tracker.started = SystemTime::now()

ecstore/src/heal/heal_ops.rs

@@ -141,6 +141,7 @@ impl HealSequence {
             ..Default::default()
         }
     }
+
 }
 
 impl HealSequence {
@@ -160,7 +161,7 @@ impl HealSequence {
         self.heal_failed_items_map.clone()
     }
 
-    fn count_failed(&mut self, heal_type: HealItemType) {
+    pub fn count_failed(&mut self, heal_type: HealItemType) {
         *self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -168,7 +169,7 @@ impl HealSequence {
             .as_secs();
     }
 
-    fn count_scanned(&mut self, heal_type: HealItemType) {
+    pub fn count_scanned(&mut self, heal_type: HealItemType) {
         *self.scanned_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -176,7 +177,7 @@ impl HealSequence {
             .as_secs();
     }
 
-    fn count_healed(&mut self, heal_type: HealItemType) {
+    pub fn count_healed(&mut self, heal_type: HealItemType) {
         *self.healed_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -353,10 +354,25 @@ pub struct AllHealState {
 }
 
 impl AllHealState {
-    pub fn new(cleanup: bool) -> Self {
-        let hstate = AllHealState::default();
+    pub fn new(cleanup: bool) -> Arc<RwLock<Self>> {
+        let hstate = Arc::new(RwLock::new(AllHealState::default()));
+        let (_, mut rx) = broadcast::channel(1);
         if cleanup {
-            // spawn(f);
+            let hstate_clone = hstate.clone();
+            tokio::spawn(async move {
+                loop {
+                    select! {
+                        result = rx.recv() =>{
+                            if let Ok(true) = result {
+                                return;
+                            }
+                        }
+                        _ = sleep(Duration::from_secs(5 * 60)) => {
+                            hstate_clone.write().await.periodic_heal_seqs_clean().await;
+                        }
+                    }
+                }
+            });
         }
 
         hstate
@@ -382,7 +398,7 @@ impl AllHealState {
         });
     }
 
-    async fn update_heal_status(&mut self, tracker: &HealingTracker) {
+    pub async fn update_heal_status(&mut self, tracker: &HealingTracker) {
         let _ = self.mu.write().await;
         let _ = tracker.mu.read().await;
 
@@ -427,30 +443,19 @@ impl AllHealState {
         });
     }
 
-    async fn periodic_heal_seqs_clean(&mut self, mut rx: Receiver<bool>) {
-        loop {
-            select! {
-                result = rx.recv() =>{
-                    if let Ok(true) = result {
-                        return;
-                    }
-                }
-                _ = sleep(Duration::from_secs(5 * 60)) => {
-                    let _ = self.mu.write().await;
-                    let now = SystemTime::now();
-
-                    let mut keys_to_reomve = Vec::new();
-                    for (k, v) in self.heal_seq_map.iter() {
-                        if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now {
-                            keys_to_reomve.push(k.clone())
-                        }
-                    }
-                    for key in keys_to_reomve.iter() {
-                        self.heal_seq_map.remove(key);
-                    }
-                }
+    async fn periodic_heal_seqs_clean(&mut self) {
+        let _ = self.mu.write().await;
+        let now = SystemTime::now();
+        
+        let mut keys_to_reomve = Vec::new();
+        for (k, v) in self.heal_seq_map.iter() {
+            if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now {
+                keys_to_reomve.push(k.clone())
             }
         }
+        for key in keys_to_reomve.iter() {
+            self.heal_seq_map.remove(key);
+        }
     }
 
     async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<HealSequence>, bool) {

ecstore/src/heal/mod.rs

@@ -1,3 +1,7 @@
 pub mod background_heal_ops;
+pub mod data_scanner;
+pub mod data_scanner_metric;
+pub mod data_usage;
 pub mod heal_commands;
 pub mod heal_ops;
+pub mod data_usage_cache;