temp(4)

Signed-off-by: junxiang Mu <1948535941@qq.com>

ecstore/src/disk/local.rs

@@ -19,15 +19,17 @@ use crate::disk::os::{check_path_length, is_empty_dir};
 use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
-use crate::heal::data_scanner::has_active_rules;
-use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
+use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, SizeSummary};
+use crate::heal::data_scanner_metric::{globalScannerMetrics, ScannerMetric, ScannerMetrics};
+use crate::heal::data_usage_cache::{self, DataUsageCache, DataUsageEntry};
+use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
 use crate::heal::heal_commands::HealScanMode;
 use crate::new_object_layer_fn;
 use crate::set_disk::{
     conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
     CHECK_PART_VOLUME_NOT_FOUND,
 };
-use crate::store_api::BitrotAlgorithm;
+use crate::store_api::{BitrotAlgorithm, StorageAPI};
 use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
 use crate::utils::os::get_info;
 use crate::utils::path::{clean, has_suffix, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
@@ -39,13 +41,13 @@ use crate::{
 use common::defer;
 use path_absolutize::Absolutize;
 use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::io::Cursor;
 use std::os::unix::fs::MetadataExt;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, SystemTime};
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
@@ -1878,7 +1880,7 @@ impl DiskAPI for LocalDisk {
     }
 
     async fn ns_scanner(
-        &self,
+        self: Arc<Self>,
         cache: &DataUsageCache,
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,
@@ -1904,7 +1906,112 @@ impl DiskAPI for LocalDisk {
             Some(s) => s,
             None => return Err(Error::msg("errServerNotInitialized")),
         };
-        todo!()
+        let loc = self.get_disk_location();
+        let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
+        let disk = self.clone();
+        let disk_clone = disk.clone();
+        let mut cache = cache.clone();
+        cache.info.updates = Some(updates.clone());
+        let mut data_usage_info = scan_data_folder(
+            &disks,
+            disk,
+            &cache,
+            Box::new(move |item: &ScannerItem| {
+                let mut item = item.clone();
+                let disk = disk_clone.clone();
+                Box::pin(async move {
+                    if item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) {
+                        return Err(Error::from_string(ERR_SKIP_FILE));
+                    }
+                    let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
+                    let mut res = HashMap::new();
+                    let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata);
+                    let buf = match disk.read_metadata(item.path.clone()).await {
+                        Ok(buf) => buf,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    done_sz(buf.len() as u64).await;
+                    res.insert("metasize".to_string(), buf.len().to_string());
+                    item.transform_meda_dir();
+                    let meta_cache = MetaCacheEntry {
+                        name: item.object_path().to_string_lossy().to_string(),
+                        metadata: buf,
+                        ..Default::default()
+                    };
+                    let fivs = match meta_cache.file_info_versions(&item.bucket) {
+                        Ok(fivs) => fivs,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    let mut size_s = SizeSummary::default();
+                    let done = ScannerMetrics::time(ScannerMetric::ApplyAll);
+                    let obj_infos = match item.apply_versions_actions(&fivs.versions).await {
+                        Ok(obj_infos) => obj_infos,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+
+                    let versioned = false;
+                    let mut obj_deleted = false;
+                    for info in obj_infos.iter() {
+                        let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);
+                        let mut sz = 0;
+                        (obj_deleted, sz) = item.apply_actions(&info, &size_s).await;
+                        done().await;
+
+                        if obj_deleted {
+                            break;
+                        }
+
+                        let actual_sz = match info.get_actual_size() {
+                            Ok(size) => size,
+                            Err(_) => continue,
+                        };
+
+                        if info.delete_marker {
+                            size_s.delete_markers += 1;
+                        }
+
+                        if info.version_id.is_some() && sz == actual_sz {
+                            size_s.versions += 1;
+                        }
+
+                        size_s.total_size += sz;
+
+                        if info.delete_marker {
+                            continue;
+                        }
+                    }
+
+                    for frer_version in fivs.free_versions.iter() {
+                        let _obj_info = frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned);
+                        let done = ScannerMetrics::time(ScannerMetric::TierObjSweep);
+                        done().await;
+                    }
+
+                    // todo: global trace
+                    if obj_deleted {
+                        return Err(Error::from_string(ERR_IGNORE_FILE_CONTRIB));
+                    }
+                    done().await;
+                    Ok(size_s)
+                })
+            }),
+            scan_mode,
+        )
+        .await?;
+        data_usage_info.info.last_update = Some(SystemTime::now());
+        Ok(data_usage_info)
     }
 }
 

ecstore/src/disk/mod.rs

@@ -16,7 +16,7 @@ const STORAGE_FORMAT_FILE: &str = "xl.meta";
 use crate::{
     erasure::Writer,
     error::{Error, Result},
-    file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
+    file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion, FileMetaVersion},
     heal::{
         data_usage_cache::{DataUsageCache, DataUsageEntry},
         heal_commands::HealScanMode,
@@ -344,14 +344,14 @@ impl DiskAPI for Disk {
     }
 
     async fn ns_scanner(
-        &self,
+        self: Arc<Self>,
         cache: &DataUsageCache,
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,
     ) -> Result<DataUsageCache> {
-        match self {
-            Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await,
-            Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await,
+        match &*self {
+            Disk::Local(local_disk) => Arc::new(local_disk).ns_scanner(cache, updates, scan_mode).await,
+            Disk::Remote(remote_disk) => Arc::new(remote_disk).ns_scanner(cache, updates, scan_mode).await,
         }
     }
 }
@@ -453,7 +453,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
     async fn ns_scanner(
-        &self,
+        self: Arc<Self>,
         cache: &DataUsageCache,
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,

ecstore/src/disk/remote.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::{path::PathBuf, sync::Arc};
 
 use futures::lock::Mutex;
 use protos::{
@@ -736,7 +736,7 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn ns_scanner(
-        &self,
+        self: Arc<Self>,
         cache: &DataUsageCache,
         updates: Sender<DataUsageEntry>,
         scan_mode: HealScanMode,