merge heal

ecstore/src/disk/endpoint.rs

@@ -2,6 +2,7 @@ use crate::error::{Error, Result};
 use crate::utils::net;
 use path_absolutize::Absolutize;
 use path_clean::PathClean;
+use std::fs;
 use std::{fmt::Display, path::Path};
 use url::{ParseError, Url};
 
@@ -29,7 +30,13 @@ pub struct Endpoint {
 impl Display for Endpoint {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if self.url.scheme() == "file" {
-            write!(f, "{}", self.url.path())
+            write!(
+                f,
+                "{}",
+                fs::canonicalize(self.url.path())
+                    .map_err(|_| std::fmt::Error)?
+                    .to_string_lossy()
+            )
         } else {
             write!(f, "{}", self.url)
         }

ecstore/src/disk/error.rs

@@ -106,6 +106,9 @@ pub enum DiskError {
 
     #[error("part missing or corrupt")]
     PartMissingOrCorrupt,
+
+    #[error("No healing is required")]
+    NoHealRequired,
 }
 
 impl DiskError {
@@ -210,6 +213,7 @@ pub fn clone_disk_err(e: &DiskError) -> Error {
         DiskError::MoreData => Error::new(DiskError::MoreData),
         DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta),
         DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt),
+        DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired),
     }
 }
 
@@ -261,14 +265,7 @@ pub fn os_err_to_file_err(e: io::Error) -> Error {
 }
 
 pub fn is_err_file_not_found(err: &Error) -> bool {
-    if let Some(e) = err.downcast_ref::<DiskError>() {
-        match e {
-            DiskError::FileNotFound => true,
-            _ => false,
-        }
-    } else {
-        false
-    }
+    matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileNotFound))
 }
 
 pub fn is_sys_err_no_space(e: &io::Error) -> bool {
@@ -422,18 +419,32 @@ pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
 }
 
 pub fn is_all_not_found(errs: &[Option<Error>]) -> bool {
-    for err in errs.iter() {
-        if let Some(err) = err {
-            if let Some(err) = err.downcast_ref::<DiskError>() {
-                match err {
-                    DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
-                        continue;
-                    }
-                    _ => return false,
+    for err in errs.iter().flatten() {
+        if let Some(err) = err.downcast_ref::<DiskError>() {
+            match err {
+                DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
+                    continue;
                 }
+                _ => return false,
             }
         }
     }
 
     !errs.is_empty()
 }
+
+pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
+    if errs.is_empty() {
+        return false;
+    }
+    let mut not_found_count = 0;
+    for err in errs.iter().flatten() {
+        match err.downcast_ref() {
+            Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => {
+                not_found_count += 1;
+            }
+            _ => {}
+        }
+    }
+    errs.len() == not_found_count
+}

ecstore/src/disk/format.rs

@@ -1,4 +1,4 @@
-use super::error::DiskError;
+use super::{error::DiskError, DiskInfo};
 use crate::error::{Error, Result};
 use serde::{Deserialize, Serialize};
 use serde_json::Error as JsonError;
@@ -31,7 +31,7 @@ pub enum FormatBackend {
 ///
 /// The V3 format to support "large bucket" support where a bucket
 /// can span multiple erasure sets.
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct FormatErasureV3 {
     /// Version of 'xl' format.
     pub version: FormatErasureVersion,
@@ -88,7 +88,7 @@ pub enum DistributionAlgoVersion {
 ///
 /// Ideally we will never have a situation where we will have to change the
 /// fields of this struct and deal with related migration.
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct FormatV3 {
     /// Version of the format config.
     pub version: FormatMetaVersion,
@@ -103,8 +103,8 @@ pub struct FormatV3 {
     pub erasure: FormatErasureV3,
     // /// DiskInfo is an extended type which returns current
     // /// disk usage per path.
-    // #[serde(skip)]
-    // pub disk_info: Option<data_types::DeskInfo>,
+    #[serde(skip)]
+    pub disk_info: Option<DiskInfo>,
 }
 
 impl TryFrom<&[u8]> for FormatV3 {
@@ -146,7 +146,7 @@ impl FormatV3 {
             format,
             id: Uuid::new_v4(),
             erasure,
-            // disk_info: None,
+            disk_info: None,
         }
     }
 

ecstore/src/disk/local.rs

@@ -6,10 +6,11 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
     os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions,
     FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-    UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+    UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
 };
 use crate::bitrot::bitrot_verify;
-use crate::cache_value::cache::{Cache, Opts};
+use crate::bucket::metadata_sys::GLOBAL_BucketMetadataSys;
+use crate::cache_value::cache::{Cache, Opts, UpdateFn};
 use crate::disk::error::{
     convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir,
     map_err_not_exists, os_err_to_file_err,
@@ -18,27 +19,35 @@ use crate::disk::os::{check_path_length, is_empty_dir};
 use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
+use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, SizeSummary};
+use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
+use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
+use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
+use crate::heal::heal_commands::{HealScanMode, HealingTracker};
+use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
+use crate::new_object_layer_fn;
 use crate::set_disk::{
     conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
     CHECK_PART_VOLUME_NOT_FOUND,
 };
-use crate::store_api::BitrotAlgorithm;
+use crate::store_api::{BitrotAlgorithm, StorageAPI};
 use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
 use crate::utils::os::get_info;
-use crate::utils::path::{clean, has_suffix, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
+use crate::utils::path::{clean, has_suffix, path_join, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
 use crate::{
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
     utils,
 };
+use common::defer;
 use path_absolutize::Absolutize;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::io::Cursor;
 use std::os::unix::fs::MetadataExt;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, SystemTime};
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
@@ -46,7 +55,7 @@ use std::{
 use time::OffsetDateTime;
 use tokio::fs::{self, File};
 use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind};
-use tokio::runtime::Runtime;
+use tokio::sync::mpsc::Sender;
 use tokio::sync::RwLock;
 use tracing::{error, info, warn};
 use uuid::Uuid;
@@ -138,10 +147,10 @@ impl LocalDisk {
             last_check: format_last_check,
         };
         let root_clone = root.clone();
-        let disk_id = Arc::new(id.map_or("".to_string(), |id| id.to_string()));
-        let update_fn = move || {
-            let rt = Runtime::new().unwrap();
-            rt.block_on(async {
+        let update_fn: UpdateFn<DiskInfo> = Box::new(move || {
+            let disk_id = id.map_or("".to_string(), |id| id.to_string());
+            let root = root_clone.clone();
+            Box::pin(async move {
                 match get_disk_info(root.clone()).await {
                     Ok((info, root)) => {
                         let disk_info = DiskInfo {
@@ -167,14 +176,14 @@ impl LocalDisk {
                     Err(err) => Err(err),
                 }
             })
-        };
+        });
 
-        let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default());
+        let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
 
         // TODO: DIRECT suport
         // TODD: DiskInfo
         let mut disk = Self {
-            root: root_clone.clone(),
+            root: root.clone(),
             endpoint: ep.clone(),
             format_path,
             format_info: RwLock::new(format_info),
@@ -190,7 +199,7 @@ impl LocalDisk {
             // format_data: Mutex::new(format_data),
             // format_last_check: Mutex::new(format_last_check),
         };
-        let (info, _root) = get_disk_info(root_clone).await?;
+        let (info, _root) = get_disk_info(root).await?;
         disk.major = info.major;
         disk.minor = info.minor;
         disk.fstype = info.fstype;
@@ -932,7 +941,7 @@ impl DiskAPI for LocalDisk {
     }
 
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
-        let volume_dir = self.get_bucket_path(&volume)?;
+        let volume_dir = self.get_bucket_path(volume)?;
         check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?;
         let mut resp = CheckPartsResp {
             results: vec![0; fi.parts.len()],
@@ -958,22 +967,16 @@ impl DiskAPI for LocalDisk {
                     resp.results[i] = CHECK_PART_SUCCESS;
                 }
                 Err(err) => {
-                    match os_err_to_file_err(err).downcast_ref() {
-                        Some(DiskError::FileNotFound) => {
-                            if !skip_access_checks(volume) {
-                                if let Err(err) = access(&volume_dir).await {
-                                    match err.kind() {
-                                        ErrorKind::NotFound => {
-                                            resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
-                                            continue;
-                                        }
-                                        _ => {}
-                                    }
+                    if let Some(DiskError::FileNotFound) = os_err_to_file_err(err).downcast_ref() {
+                        if !skip_access_checks(volume) {
+                            if let Err(err) = access(&volume_dir).await {
+                                if err.kind() == ErrorKind::NotFound {
+                                    resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
+                                    continue;
                                 }
                             }
-                            resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
                         }
-                        _ => {}
+                        resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
                     }
                     continue;
                 }
@@ -1881,6 +1884,162 @@ impl DiskAPI for LocalDisk {
 
         Ok(info)
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        self.scanning.fetch_add(1, Ordering::SeqCst);
+        defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) });
+
+        // Check if the current bucket has replication configuration
+        if let Ok((rcfg, _)) = GLOBAL_BucketMetadataSys
+            .read()
+            .await
+            .get_replication_config(&cache.info.name)
+            .await
+        {
+            if has_active_rules(&rcfg, "", true) {
+                // TODO: globalBucketTargetSys
+            }
+        }
+
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::msg("errServerNotInitialized")),
+        };
+        let loc = self.get_disk_location();
+        let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
+        let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?);
+        let disk_clone = disk.clone();
+        let mut cache = cache.clone();
+        cache.info.updates = Some(updates.clone());
+        let mut data_usage_info = scan_data_folder(
+            &disks,
+            disk,
+            &cache,
+            Box::new(move |item: &ScannerItem| {
+                let mut item = item.clone();
+                let disk = disk_clone.clone();
+                Box::pin(async move {
+                    if item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) {
+                        return Err(Error::from_string(ERR_SKIP_FILE));
+                    }
+                    let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
+                    let mut res = HashMap::new();
+                    let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await;
+                    let buf = match disk.read_metadata(item.path.clone()).await {
+                        Ok(buf) => buf,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    done_sz(buf.len() as u64).await;
+                    res.insert("metasize".to_string(), buf.len().to_string());
+                    item.transform_meda_dir();
+                    let meta_cache = MetaCacheEntry {
+                        name: item.object_path().to_string_lossy().to_string(),
+                        metadata: buf,
+                        ..Default::default()
+                    };
+                    let fivs = match meta_cache.file_info_versions(&item.bucket) {
+                        Ok(fivs) => fivs,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    let mut size_s = SizeSummary::default();
+                    let done = ScannerMetrics::time(ScannerMetric::ApplyAll);
+                    let obj_infos = match item.apply_versions_actions(&fivs.versions).await {
+                        Ok(obj_infos) => obj_infos,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+
+                    let versioned = false;
+                    let mut obj_deleted = false;
+                    for info in obj_infos.iter() {
+                        let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);
+                        let sz: usize;
+                        (obj_deleted, sz) = item.apply_actions(info, &size_s).await;
+                        done().await;
+
+                        if obj_deleted {
+                            break;
+                        }
+
+                        let actual_sz = match info.get_actual_size() {
+                            Ok(size) => size,
+                            Err(_) => continue,
+                        };
+
+                        if info.delete_marker {
+                            size_s.delete_markers += 1;
+                        }
+
+                        if info.version_id.is_some() && sz == actual_sz {
+                            size_s.versions += 1;
+                        }
+
+                        size_s.total_size += sz;
+
+                        if info.delete_marker {
+                            continue;
+                        }
+                    }
+
+                    for frer_version in fivs.free_versions.iter() {
+                        let _obj_info =
+                            frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned);
+                        let done = ScannerMetrics::time(ScannerMetric::TierObjSweep);
+                        done().await;
+                    }
+
+                    // todo: global trace
+                    if obj_deleted {
+                        return Err(Error::from_string(ERR_IGNORE_FILE_CONTRIB));
+                    }
+                    done().await;
+                    Ok(size_s)
+                })
+            }),
+            scan_mode,
+        )
+        .await?;
+        data_usage_info.info.last_update = Some(SystemTime::now());
+        Ok(data_usage_info)
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        let healing_file = path_join(&[
+            self.path(),
+            PathBuf::from(RUSTFS_META_BUCKET),
+            PathBuf::from(BUCKET_META_PREFIX),
+            PathBuf::from(HEALING_TRACKER_FILENAME),
+        ]);
+        let b = match fs::read(healing_file).await {
+            Ok(b) => b,
+            Err(_) => return None,
+        };
+        if b.is_empty() {
+            return None;
+        }
+        match HealingTracker::unmarshal_msg(&b) {
+            Ok(h) => Some(h),
+            Err(_) => Some(HealingTracker::default()),
+        }
+    }
 }
 
 async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
@@ -1893,10 +2052,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
         if root_disk_threshold > 0 {
             disk_info.total <= root_disk_threshold
         } else {
-            match is_root_disk(&drive_path, SLASH_SEPARATOR) {
-                Ok(result) => result,
-                Err(_) => false,
-            }
+            is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default()
         }
     } else {
         false

ecstore/src/disk/mod.rs

@@ -1,7 +1,7 @@
 pub mod endpoint;
 pub mod error;
 pub mod format;
-mod local;
+pub mod local;
 pub mod os;
 pub mod remote;
 
@@ -14,9 +14,13 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 const STORAGE_FORMAT_FILE: &str = "xl.meta";
 
 use crate::{
-    erasure::{ReadAt, Writer},
+    erasure::Writer,
     error::{Error, Result},
     file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
+    heal::{
+        data_usage_cache::{DataUsageCache, DataUsageEntry},
+        heal_commands::{HealScanMode, HealingTracker},
+    },
     store_api::{FileInfo, RawFileInfo},
 };
 use endpoint::Endpoint;
@@ -35,7 +39,6 @@ use std::{
     io::{Cursor, SeekFrom},
     path::PathBuf,
     sync::Arc,
-    usize,
 };
 use time::OffsetDateTime;
 use tokio::{
@@ -53,8 +56,8 @@ pub type DiskStore = Arc<Disk>;
 
 #[derive(Debug)]
 pub enum Disk {
-    Local(LocalDisk),
-    Remote(RemoteDisk),
+    Local(Box<LocalDisk>),
+    Remote(Box<RemoteDisk>),
 }
 
 #[async_trait::async_trait]
@@ -338,15 +341,34 @@ impl DiskAPI for Disk {
             Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await,
         }
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        match self {
+            Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await,
+            Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await,
+        }
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        match self {
+            Disk::Local(local_disk) => local_disk.healing().await,
+            Disk::Remote(remote_disk) => remote_disk.healing().await,
+        }
+    }
 }
 
 pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskStore> {
     if ep.is_local {
         let s = local::LocalDisk::new(ep, opt.cleanup).await?;
-        Ok(Arc::new(Disk::Local(s)))
+        Ok(Arc::new(Disk::Local(Box::new(s))))
     } else {
         let remote_disk = remote::RemoteDisk::new(ep, opt).await?;
-        Ok(Arc::new(Disk::Remote(remote_disk)))
+        Ok(Arc::new(Disk::Remote(Box::new(remote_disk))))
     }
 }
 
@@ -436,6 +458,13 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache>;
+    async fn healing(&self) -> Option<HealingTracker>;
 }
 
 #[derive(Debug, Default, Serialize, Deserialize)]
@@ -467,7 +496,7 @@ pub struct DiskInfoOptions {
     pub noop: bool,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DiskInfo {
     pub total: u64,
     pub free: u64,
@@ -489,7 +518,7 @@ pub struct DiskInfo {
     pub error: String,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DiskMetrics {
     api_calls: HashMap<String, u64>,
     total_waiting: u32,
@@ -653,7 +682,7 @@ impl MetaCacheEntry {
         let mut fm = FileMeta::new();
         fm.unmarshal_msg(&self.metadata)?;
 
-        Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?)
+        fm.into_file_info_versions(bucket, self.name.as_str(), false)
     }
 
     pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option<MetaCacheEntry>, bool)> {
@@ -812,7 +841,7 @@ impl MetaCacheEntries {
         selected = Some(MetaCacheEntry {
             name: selected.as_ref().unwrap().name.clone(),
             cached: Some(FileMeta {
-                meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver.clone(),
+                meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver,
                 ..Default::default()
             }),
             _reusable: true,
@@ -835,7 +864,7 @@ impl MetaCacheEntries {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Clone, Debug, Default)]
 pub struct DiskOption {
     pub cleanup: bool,
     pub health_check: bool,
@@ -1281,10 +1310,10 @@ impl Reader for RemoteFileReader {
             Err(Error::from_string(error_info))
         }
     }
-    async fn seek(&mut self, offset: usize) -> Result<()> {
+    async fn seek(&mut self, _offset: usize) -> Result<()> {
         unimplemented!()
     }
-    async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize> {
+    async fn read_exact(&mut self, _buf: &mut [u8]) -> Result<usize> {
         unimplemented!()
     }
 }

ecstore/src/disk/os.rs

@@ -56,7 +56,7 @@ pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
         return Ok(false);
     }
 
-    Ok(same_disk(disk_path, root_disk)?)
+    same_disk(disk_path, root_disk)
 }
 
 pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {

ecstore/src/disk/remote.rs

@@ -5,11 +5,13 @@ use protos::{
     node_service_time_out_client,
     proto_gen::node_service::{
         CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
-        DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
-        ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest,
-        UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
+        DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest,
+        ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst,
+        StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
     },
 };
+use tokio::sync::mpsc::{self, Sender};
+use tokio_stream::{wrappers::ReceiverStream, StreamExt};
 use tonic::Request;
 use tracing::info;
 use uuid::Uuid;
@@ -22,6 +24,10 @@ use super::{
 use crate::{
     disk::error::DiskError,
     error::{Error, Result},
+    heal::{
+        data_usage_cache::{DataUsageCache, DataUsageEntry},
+        heal_commands::{HealScanMode, HealingTracker},
+    },
     store_api::{FileInfo, RawFileInfo},
 };
 use protos::proto_gen::node_service::RenamePartRequst;
@@ -747,4 +753,48 @@ impl DiskAPI for RemoteDisk {
 
         Ok(disk_info)
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        info!("ns_scanner");
+        let cache = serde_json::to_string(cache)?;
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?;
+
+        let (tx, rx) = mpsc::channel(10);
+        let in_stream = ReceiverStream::new(rx);
+        let mut response = client.ns_scanner(in_stream).await?.into_inner();
+        let request = NsScannerRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            cache,
+            scan_mode: scan_mode as u64,
+        };
+        tx.send(request).await?;
+
+        loop {
+            match response.next().await {
+                Some(Ok(resp)) => {
+                    if !resp.update.is_empty() {
+                        let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
+                        let _ = updates.send(data_usage_cache).await;
+                    } else if !resp.data_usage_cache.is_empty() {
+                        let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
+                        return Ok(data_usage_cache);
+                    } else {
+                        return Err(Error::from_string("scan was interrupted"));
+                    }
+                }
+                _ => return Err(Error::from_string("scan was interrupted")),
+            }
+        }
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        None
+    }
 }