auto heal(2)

Signed-off-by: junxiang Mu <1948535941@qq.com>

common/protos/src/generated/flatbuffers_generated/models.rs

@@ -1,9 +1,10 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
+
 // @generated
 
-use core::cmp::Ordering;
 use core::mem;
+use core::cmp::Ordering;
 
 extern crate flatbuffers;
 use self::flatbuffers::{EndianScalar, Follow};
@@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow};
 #[allow(unused_imports, dead_code)]
 pub mod models {
 
-    use core::cmp::Ordering;
+  use core::mem;
-    use core::mem;
+  use core::cmp::Ordering;
 
-    extern crate flatbuffers;
+  extern crate flatbuffers;
-    use self::flatbuffers::{EndianScalar, Follow};
+  use self::flatbuffers::{EndianScalar, Follow};
 
-    pub enum PingBodyOffset {}
+pub enum PingBodyOffset {}
-    #[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq)]
 
-    pub struct PingBody<'a> {
+pub struct PingBody<'a> {
-        pub _tab: flatbuffers::Table<'a>,
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+  type Inner = PingBody<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> PingBody<'a> {
+  pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+
+  pub const fn get_fully_qualified_name() -> &'static str {
+    "models.PingBody"
+  }
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    PingBody { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args PingBodyArgs<'args>
+  ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
+    let mut builder = PingBodyBuilder::new(_fbb);
+    if let Some(x) = args.payload { builder.add_payload(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for PingBody<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct PingBodyArgs<'a> {
+    pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for PingBodyArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    PingBodyArgs {
+      payload: None,
     }
+  }
+}
 
-    impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        type Inner = PingBody<'a>;
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        #[inline]
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
-        unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+}
-            Self {
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
-                _tab: flatbuffers::Table::new(buf, loc),
+  #[inline]
-            }
+  pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
-        }
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    PingBodyBuilder {
+      fbb_: _fbb,
+      start_: start,
     }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
 
-    impl<'a> PingBody<'a> {
+impl core::fmt::Debug for PingBody<'_> {
-        pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("PingBody");
+      ds.field("payload", &self.payload());
+      ds.finish()
+  }
+}
+}  // pub mod models
 
-        pub const fn get_fully_qualified_name() -> &'static str {
-            "models.PingBody"
-        }
-
-        #[inline]
-        pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
-            PingBody { _tab: table }
-        }
-        #[allow(unused_mut)]
-        pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
-            _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
-            args: &'args PingBodyArgs<'args>,
-        ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
-            let mut builder = PingBodyBuilder::new(_fbb);
-            if let Some(x) = args.payload {
-                builder.add_payload(x);
-            }
-            builder.finish()
-        }
-
-        #[inline]
-        pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
-            // Safety:
-            // Created from valid Table for this object
-            // which contains a valid value in this slot
-            unsafe {
-                self._tab
-                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
-            }
-        }
-    }
-
-    impl flatbuffers::Verifiable for PingBody<'_> {
-        #[inline]
-        fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> {
-            use self::flatbuffers::Verifiable;
-            v.visit_table(pos)?
-                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
-                .finish();
-            Ok(())
-        }
-    }
-    pub struct PingBodyArgs<'a> {
-        pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
-    }
-    impl<'a> Default for PingBodyArgs<'a> {
-        #[inline]
-        fn default() -> Self {
-            PingBodyArgs { payload: None }
-        }
-    }
-
-    pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
-    }
-    impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
-        #[inline]
-        pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b, u8>>) {
-            self.fbb_
-                .push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
-        }
-        #[inline]
-        pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
-            let start = _fbb.start_table();
-            PingBodyBuilder {
-                fbb_: _fbb,
-                start_: start,
-            }
-        }
-        #[inline]
-        pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
-            let o = self.fbb_.end_table(self.start_);
-            flatbuffers::WIPOffset::new(o.value())
-        }
-    }
-
-    impl core::fmt::Debug for PingBody<'_> {
-        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-            let mut ds = f.debug_struct("PingBody");
-            ds.field("payload", &self.payload());
-            ds.finish()
-        }
-    }
-} // pub mod models

ecstore/src/config/mod.rs

@@ -18,6 +18,8 @@ lazy_static! {
     pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
 }
 
+pub static RUSTFS_CONFIG_PREFIX: &str = "config";
+
 pub struct ConfigSys {}
 
 impl ConfigSys {

ecstore/src/disk/endpoint.rs

@@ -2,6 +2,7 @@ use crate::error::{Error, Result};
 use crate::utils::net;
 use path_absolutize::Absolutize;
 use path_clean::PathClean;
+use std::fs;
 use std::{fmt::Display, path::Path};
 use url::{ParseError, Url};
 
@@ -29,7 +30,7 @@ pub struct Endpoint {
 impl Display for Endpoint {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if self.url.scheme() == "file" {
-            write!(f, "{}", self.url.path())
+            write!(f, "{}", fs::canonicalize(self.url.path()).map_err(|_| std::fmt::Error)?.to_string_lossy().to_string())
         } else {
             write!(f, "{}", self.url)
         }

ecstore/src/disk/local.rs

@@ -6,7 +6,7 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
     os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions,
     FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-    UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+    UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
 };
 use crate::bitrot::bitrot_verify;
 use crate::bucket::metadata_sys::GLOBAL_BucketMetadataSys;
@@ -23,7 +23,8 @@ use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem,
 use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
 use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
 use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
-use crate::heal::heal_commands::HealScanMode;
+use crate::heal::heal_commands::{HealScanMode, HealingTracker};
+use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
 use crate::new_object_layer_fn;
 use crate::set_disk::{
     conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
@@ -32,7 +33,7 @@ use crate::set_disk::{
 use crate::store_api::{BitrotAlgorithm, StorageAPI};
 use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
 use crate::utils::os::get_info;
-use crate::utils::path::{clean, has_suffix, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
+use crate::utils::path::{clean, has_suffix, path_join, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
 use crate::{
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
@@ -2013,6 +2014,26 @@ impl DiskAPI for LocalDisk {
         data_usage_info.info.last_update = Some(SystemTime::now());
         Ok(data_usage_info)
     }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        let healing_file = path_join(&[
+            self.path(),
+            PathBuf::from(RUSTFS_META_BUCKET),
+            PathBuf::from(BUCKET_META_PREFIX),
+            PathBuf::from(HEALING_TRACKER_FILENAME),
+        ]);
+        let b = match fs::read(healing_file).await {
+            Ok(b) => b,
+            Err(_) => return None,
+        };
+        if b.is_empty() {
+            return None;
+        }
+        match HealingTracker::unmarshal_msg(&b) {
+            Ok(h) => Some(h),
+            Err(_) => Some(HealingTracker::default())
+        }
+    }
 }
 
 async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {

ecstore/src/disk/mod.rs

@@ -19,7 +19,7 @@ use crate::{
     file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
     heal::{
         data_usage_cache::{DataUsageCache, DataUsageEntry},
-        heal_commands::HealScanMode,
+        heal_commands::{HealScanMode, HealingTracker},
     },
     store_api::{FileInfo, RawFileInfo},
 };
@@ -349,11 +349,18 @@ impl DiskAPI for Disk {
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,
     ) -> Result<DataUsageCache> {
-        match &*self {
+        match self {
             Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await,
             Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await,
         }
     }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        match self {
+            Disk::Local(local_disk) => local_disk.healing().await,
+            Disk::Remote(remote_disk) => remote_disk.healing().await,
+        }
+    }
 }
 
 pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskStore> {
@@ -458,6 +465,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,
     ) -> Result<DataUsageCache>;
+    async fn healing(&self) -> Option<HealingTracker>;
 }
 
 #[derive(Debug, Default, Serialize, Deserialize)]

ecstore/src/disk/remote.rs

@@ -26,7 +26,7 @@ use crate::{
     error::{Error, Result},
     heal::{
         data_usage_cache::{DataUsageCache, DataUsageEntry},
-        heal_commands::HealScanMode,
+        heal_commands::{HealScanMode, HealingTracker},
     },
     store_api::{FileInfo, RawFileInfo},
 };
@@ -775,4 +775,8 @@ impl DiskAPI for RemoteDisk {
             }
         }
     }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        None
+    }
 }

ecstore/src/endpoints.rs

@@ -110,6 +110,10 @@ impl Endpoints {
         self.0
     }
 
+    pub fn into_ref(&self) -> &Vec<Endpoint> {
+        &self.0
+    }
+
     // GetString - returns endpoint string of i-th endpoint (0-based),
     // and empty string for invalid indexes.
     pub fn get_string(&self, i: usize) -> String {
@@ -119,6 +123,10 @@ impl Endpoints {
 
         self.0[i].to_string()
     }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
 }
 
 #[derive(Debug)]

ecstore/src/heal/background_heal_ops.rs

@@ -1,18 +1,17 @@
-use std::{env, sync::Arc};
+use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
 
-use tokio::sync::{
+use tokio::{
-    mpsc::{self, Receiver, Sender},
+    sync::{
-    RwLock,
+        mpsc::{self, Receiver, Sender},
+        RwLock,
+    },
+    time::interval,
 };
+use tracing::info;
+use uuid::Uuid;
 
 use crate::{
-    endpoints::Endpoints,
+    config::RUSTFS_CONFIG_PREFIX, disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, error::{Error, Result}, global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP}, heal::{data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT}, data_usage_cache::DataUsageCache, heal_commands::{init_healing_tracker, load_healing_tracker}, heal_ops::NOP_HEAL}, new_object_layer_fn, store::get_disk_via_endpoint, store_api::{BucketInfo, BucketOptions, StorageAPI}, utils::path::{path_join, SLASH_SEPARATOR}
-    error::{Error, Result},
-    global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
-    heal::heal_ops::NOP_HEAL,
-    new_object_layer_fn,
-    store_api::StorageAPI,
-    utils::path::SLASH_SEPARATOR,
 };
 
 use super::{
@@ -20,11 +19,17 @@ use super::{
     heal_ops::{new_bg_heal_sequence, HealSequence},
 };
 
+pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
+
 pub async fn init_auto_heal() {
     init_background_healing().await;
     if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
         if v == "on" {
-            // GLOBAL_BackgroundHealState.write().await.push_heal_local_disks(heal_local_disks).await;
+            GLOBAL_BackgroundHealState
+                .write()
+                .await
+                .push_heal_local_disks(&get_local_disks_to_heal().await)
+                .await;
         }
     }
 }
@@ -44,8 +49,142 @@ async fn init_background_healing() {
         .await;
 }
 
-async fn get_local_disks_to_heal() -> Endpoints {
+async fn get_local_disks_to_heal() -> Vec<Endpoint> {
-    for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {}
+    let mut disks_to_heal = Vec::new();
+    for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
+        if let Some(disk) = disk {
+            if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+                if let Some(DiskError::UnformattedDisk) = err.downcast_ref() {
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+            let h = disk.healing().await;
+            if let Some(h) = h {
+                if !h.finished {
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+        }
+    }
+
+    // todo
+    // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n {
+
+    // }
+    disks_to_heal
+}
+
+async fn monitor_local_disks_and_heal() {
+    let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL);
+
+    loop {
+        interval.tick().await;
+        let heal_disks = GLOBAL_BackgroundHealState.read().await.get_heal_local_disk_endpoints().await;
+        if heal_disks.is_empty() {
+            interval.reset();
+            continue;
+        }
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = lock.as_ref().expect("errServerNotInitialized");
+        if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
+            if let Some(DiskError::NoHealRequired) = err.downcast_ref() {
+            } else {
+                info!("heal format err: {}", err.to_string());
+                interval.reset();
+                continue;
+            }
+        }
+
+        for disk in heal_disks.into_ref().iter() {
+            let disk_clone = disk.clone();
+            tokio::spawn(async move {
+                GLOBAL_BackgroundHealState.write().await.set_disk_healing_status(disk_clone.clone(), true).await;
+                
+            });
+        }
+    }
+}
+
+async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
+    let (pool_idx, set_idx) = (endpoint.pool_idx.unwrap(), endpoint.disk_idx.unwrap());
+    let disk = match get_disk_via_endpoint(endpoint).await {
+        Some(disk) => disk,
+        None => return Err(Error::from_string(format!("Unexpected error disk must be initialized by now after formatting: {}", endpoint.to_string()))),
+    };
+
+    if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+        match err.downcast_ref() {
+            Some(DiskError::DriveIsRoot) => {
+                return Ok(());
+            },
+            Some(DiskError::UnformattedDisk) => {
+            },
+            _ => {
+                return Err(err);
+            }
+        }
+    }
+
+    let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
+        Ok(tracker) => tracker,
+        Err(err) => {
+            match err.downcast_ref() {
+                Some(DiskError::FileNotFound) => {
+                    return Ok(());
+                }
+                _ => {
+                    info!("Unable to load healing tracker on '{}': {}, re-initializing..", disk.to_string(), err.to_string());
+                },
+            }
+            init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await?
+        }
+    };
+
+    info!("Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint.to_string());
+
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => return Err(Error::msg("errServerNotInitialized")),
+    };
+    let mut buckets = store.list_bucket(&BucketOptions::default()).await?;
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)]).to_string_lossy().to_string(),
+        ..Default::default()
+    });
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]).to_string_lossy().to_string(),
+        ..Default::default()
+    });
+
+    buckets.sort_by(|a, b| {
+        let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET);
+        let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET);
+
+        match (a_has_prefix, b_has_prefix) {
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            _ => b.created.cmp(&a.created),
+        }
+    });
+
+    let mut cache = match DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await {
+        Ok(cache) => {
+            let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new());
+            tracker.objects_total_count = data_usage_info.objects_total_count;
+            tracker.objects_total_size = data_usage_info.objects_total_size;
+            cache
+        },
+        Err(_) => DataUsageCache::default()
+    };
+
+    let location = disk.get_disk_location();
+    tracker.set_queue_buckets(&buckets).await;
+    tracker.save().await?;
+
+    // store.pools[pool_idx].disk_set[set_idx].
     todo!()
 }
 

ecstore/src/heal/heal_commands.rs

@@ -161,8 +161,7 @@ pub struct HealingTracker {
 
 impl HealingTracker {
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
-        serde_json::to_string(self)
+        serde_json::to_vec(self)
-            .map(|s| s.as_bytes().to_vec())
             .map_err(|err| Error::from_string(err.to_string()))
     }
 
@@ -336,7 +335,7 @@ impl HealingTracker {
         self.queue_buckets.retain(|x| x != bucket);
     }
 
-    async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
+    pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
         let _ = self.mu.write().await;
 
         buckets.iter().for_each(|bucket| {
@@ -417,7 +416,7 @@ impl Clone for HealingTracker {
     }
 }
 
-async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
+pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
     if let Some(disk) = disk {
         let disk_id = disk.get_disk_id().await?;
         if let Some(disk_id) = disk_id {

ecstore/src/heal/heal_ops.rs

@@ -406,7 +406,7 @@ impl AllHealState {
         hstate
     }
 
-    async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
+    pub async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
         let _ = self.mu.write().await;
 
         self.heal_local_disks.retain(|k, _| {
@@ -433,7 +433,7 @@ impl AllHealState {
         self.heal_status.insert(tracker.id.clone(), tracker.clone());
     }
 
-    async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
+    pub async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
         let _ = self.mu.read().await;
 
         let mut dst = HashMap::new();
@@ -444,7 +444,7 @@ impl AllHealState {
         dst
     }
 
-    async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
+    pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
         let _ = self.mu.read().await;
 
         let mut endpoints = Vec::new();
@@ -471,7 +471,7 @@ impl AllHealState {
         });
     }
 
-    async fn periodic_heal_seqs_clean(&mut self) {
+    pub async fn periodic_heal_seqs_clean(&mut self) {
         let _ = self.mu.write().await;
         let now = SystemTime::now();
 
@@ -500,13 +500,13 @@ impl AllHealState {
         (None, false)
     }
 
-    async fn get_heal_sequence(&self, path: &str) -> Option<Arc<RwLock<HealSequence>>> {
+    pub async fn get_heal_sequence(&self, path: &str) -> Option<Arc<RwLock<HealSequence>>> {
         let _ = self.mu.read().await;
 
         self.heal_seq_map.get(path).cloned()
     }
 
-    async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
+    pub async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
         let mut hsp = HealStopSuccess::default();
         if let Some(he) = self.get_heal_sequence(path).await {
             let he = he.read().await;

ecstore/src/store.rs

@@ -3,7 +3,7 @@
 use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, set_bucket_metadata};
 use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
 use crate::config::{self, storageclass, GLOBAL_ConfigSys};
-use crate::disk::endpoint::EndpointType;
+use crate::disk::endpoint::{Endpoint, EndpointType};
 use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions, MetaCacheEntry};
 use crate::global::{
     is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION,
@@ -735,6 +735,14 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     None
 }
 
+pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
+    let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
+    if global_set_drives.is_empty() {
+        return GLOBAL_LOCAL_DISK_MAP.read().await[&endpoint.to_string()].clone();
+    }
+    global_set_drives[endpoint.pool_idx.unwrap()][endpoint.set_idx.unwrap()][endpoint.disk_idx.unwrap()].clone()
+}
+
 pub async fn all_local_disk_path() -> Vec<String> {
     let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
     disk_map.keys().cloned().collect()