merge main

ecstore/src/config/storageclass.rs

@@ -189,13 +189,7 @@ pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
 
     validate_parity_inner(standard.parity, rrs.parity, set_drive_count)?;
 
-    let optimize = {
-        if let Ok(ev) = env::var(OPTIMIZE_ENV) {
-            Some(ev)
-        } else {
-            None
-        }
-    };
+    let optimize = { env::var(OPTIMIZE_ENV).ok() };
 
     let inline_block = {
         if let Ok(ev) = env::var(INLINE_BLOCK_ENV) {

ecstore/src/disk/error.rs

@@ -1,4 +1,5 @@
 use std::io::{self, ErrorKind};
+use std::path::PathBuf;
 
 use tracing::error;
 
@@ -345,6 +346,19 @@ pub fn os_err_to_file_err(e: io::Error) -> Error {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub struct FileAccessDeniedWithContext {
+    pub path: PathBuf,
+    #[source]
+    pub source: std::io::Error,
+}
+
+impl std::fmt::Display for FileAccessDeniedWithContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "访问文件 '{}' 被拒绝: {}", self.path.display(), self.source)
+    }
+}
+
 pub fn is_unformatted_disk(err: &Error) -> bool {
     matches!(err.downcast_ref::<DiskError>(), Some(DiskError::UnformattedDisk))
 }

ecstore/src/disk/local.rs

@@ -11,16 +11,20 @@ use super::{
 };
 use crate::bitrot::bitrot_verify;
 use crate::bucket::metadata_sys::{self};
+use crate::bucket::versioning::VersioningApi;
+use crate::bucket::versioning_sys::BucketVersioningSys;
 use crate::cache_value::cache::{Cache, Opts, UpdateFn};
 use crate::disk::error::{
     convert_access_error, is_err_os_not_exist, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir,
-    is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err,
+    is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err, FileAccessDeniedWithContext,
 };
 use crate::disk::os::{check_path_length, is_empty_dir};
 use crate::disk::STORAGE_FORMAT_FILE;
 use crate::file_meta::{get_file_info, read_xl_meta_no_data, FileInfoOpts};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
-use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary};
+use crate::heal::data_scanner::{
+    lc_has_active_rules, rep_has_active_rules, scan_data_folder, ScannerItem, ShouldSleepFn, SizeSummary,
+};
 use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
 use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
 use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
@@ -51,7 +55,6 @@ use path_absolutize::Absolutize;
 use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::io::SeekFrom;
-use std::os::unix::fs::MetadataExt;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};
@@ -308,7 +311,19 @@ impl LocalDisk {
     //     })
     // }
 
+    #[allow(unreachable_code)]
+    #[allow(unused_variables)]
     pub async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> {
+        if recursive {
+            remove_all(delete_path).await?;
+        } else {
+            remove(delete_path).await?;
+        }
+
+        return Ok(());
+
+        // TODO: 异步通知 检测硬盘空间 清空回收站
+
         let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
         if let Some(parent) = trash_path.parent() {
             if !parent.exists() {
@@ -347,7 +362,6 @@ impl LocalDisk {
             return Ok(());
         }
 
-        // TODO: 异步通知 检测硬盘空间 清空回收站
         Ok(())
     }
 
@@ -383,7 +397,10 @@ impl LocalDisk {
                     kind => {
                         if kind.to_string() != "directory not empty" {
                             warn!("delete_file remove_dir {:?} err {}", &delete_path, kind.to_string());
-                            return Err(Error::from(err));
+                            return Err(Error::new(FileAccessDeniedWithContext {
+                                path: delete_path.clone(),
+                                source: err,
+                            }));
                         }
                     }
                 }
@@ -395,7 +412,10 @@ impl LocalDisk {
                 ErrorKind::NotFound => (),
                 _ => {
                     warn!("delete_file remove_file {:?}  err {:?}", &delete_path, &err);
-                    return Err(Error::from(err));
+                    return Err(Error::new(FileAccessDeniedWithContext {
+                        path: delete_path.clone(),
+                        source: err,
+                    }));
                 }
             }
         }
@@ -472,13 +492,14 @@ impl LocalDisk {
         let meta = f.metadata().await?;
 
         if meta.is_dir() {
-            return Err(Error::new(DiskError::FileNotFound));
+            // fix use io::Error
+            return Err(std::io::Error::new(ErrorKind::NotFound, "is dir").into());
         }
 
         let meta = f.metadata().await.map_err(os_err_to_file_err)?;
 
         if meta.is_dir() {
-            return Err(Error::new(DiskError::FileNotFound));
+            return Err(std::io::Error::new(ErrorKind::NotFound, "is dir").into());
         }
 
         let size = meta.len() as usize;
@@ -743,12 +764,10 @@ impl LocalDisk {
             .await
             .map_err(os_err_to_file_err)?;
 
-        // let mut data = Vec::new();
-        // let n = file.read_to_end(&mut data).await?;
-
         let meta = file.metadata().await?;
+        let file_size = meta.len() as usize;
 
-        bitrot_verify(Box::new(file), meta.size() as usize, part_size, algo, sum.to_vec(), shard_size).await
+        bitrot_verify(Box::new(file), file_size, part_size, algo, sum.to_vec(), shard_size).await
     }
 
     async fn scan_dir<W: AsyncWrite + Unpin>(
@@ -816,8 +835,6 @@ impl LocalDisk {
 
         // 第一层过滤
         for item in entries.iter_mut() {
-            // warn!("walk_dir  get entry {:?}", &entry);
-
             let entry = item.clone();
             // check limit
             if opts.limit > 0 && *objs_returned >= opts.limit {
@@ -857,7 +874,10 @@ impl LocalDisk {
                 let metadata = self
                     .read_metadata(self.get_object_path(bucket, format!("{}/{}", &current, &entry).as_str())?)
                     .await?;
-                let name = entry.trim_end_matches(STORAGE_FORMAT_FILE).trim_end_matches(SLASH_SEPARATOR);
+
+                // 用strip_suffix只删除一次
+                let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
+                let name = entry.trim_end_matches(SLASH_SEPARATOR);
                 let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
 
                 out.write_obj(&MetaCacheEntry {
@@ -887,7 +907,6 @@ impl LocalDisk {
         let mut dir_stack: Vec<String> = Vec::with_capacity(5);
 
         for entry in entries.iter() {
-            //
             if opts.limit > 0 && *objs_returned >= opts.limit {
                 return Ok(());
             }
@@ -1217,7 +1236,7 @@ impl DiskAPI for LocalDisk {
                 .join(path)
                 .join(fi.data_dir.map_or("".to_string(), |dir| dir.to_string()))
                 .join(format!("part.{}", part.number));
-            let err = match self
+            let err = (self
                 .bitrot_verify(
                     &part_path,
                     erasure.shard_file_size(part.size),
@@ -1225,11 +1244,8 @@ impl DiskAPI for LocalDisk {
                     &checksum_info.hash,
                     erasure.shard_size(erasure.block_size),
                 )
-                .await
-            {
-                Ok(_) => None,
-                Err(err) => Some(err),
-            };
+                .await)
+                .err();
             resp.results[i] = conv_part_err_to_int(&err);
             if resp.results[i] == CHECK_PART_UNKNOWN {
                 if let Some(err) = err {
@@ -2283,6 +2299,7 @@ impl DiskAPI for LocalDisk {
         Ok(info)
     }
 
+    #[tracing::instrument(level = "info", skip_all)]
     async fn ns_scanner(
         &self,
         cache: &DataUsageCache,
@@ -2296,18 +2313,30 @@ impl DiskAPI for LocalDisk {
         // must befor metadata_sys
         let Some(store) = new_object_layer_fn() else { return Err(Error::msg("errServerNotInitialized")) };
 
+        let mut cache = cache.clone();
+        // Check if the current bucket has a configured lifecycle policy
+        if let Ok((lc, _)) = metadata_sys::get_lifecycle_config(&cache.info.name).await {
+            if lc_has_active_rules(&lc, "") {
+                cache.info.life_cycle = Some(lc);
+            }
+        }
+
         // Check if the current bucket has replication configuration
         if let Ok((rcfg, _)) = metadata_sys::get_replication_config(&cache.info.name).await {
-            if has_active_rules(&rcfg, "", true) {
+            if rep_has_active_rules(&rcfg, "", true) {
                 // TODO: globalBucketTargetSys
             }
         }
 
+        let vcfg = match BucketVersioningSys::get(&cache.info.name).await {
+            Ok(vcfg) => Some(vcfg),
+            Err(_) => None,
+        };
+
         let loc = self.get_disk_location();
         let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
         let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?);
         let disk_clone = disk.clone();
-        let mut cache = cache.clone();
         cache.info.updates = Some(updates.clone());
         let mut data_usage_info = scan_data_folder(
             &disks,
@@ -2316,8 +2345,9 @@ impl DiskAPI for LocalDisk {
             Box::new(move |item: &ScannerItem| {
                 let mut item = item.clone();
                 let disk = disk_clone.clone();
+                let vcfg = vcfg.clone();
                 Box::pin(async move {
-                    if item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) {
+                    if !item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) {
                         return Err(Error::from_string(ERR_SKIP_FILE));
                     }
                     let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
@@ -2358,7 +2388,12 @@ impl DiskAPI for LocalDisk {
                         }
                     };
 
-                    let versioned = false;
+                    let versioned = if let Some(vcfg) = vcfg.as_ref() {
+                        vcfg.versioned(item.object_path().to_str().unwrap_or_default())
+                    } else {
+                        false
+                    };
+
                     let mut obj_deleted = false;
                     for info in obj_infos.iter() {
                         let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);

ecstore/src/endpoints.rs

@@ -1,4 +1,4 @@
-use tracing::warn;
+use tracing::{instrument, warn};
 
 use crate::{
     disk::endpoint::{Endpoint, EndpointType},
@@ -407,7 +407,7 @@ pub struct PoolEndpoints {
     pub platform: String,
 }
 
-/// list of list of endpoints
+/// list of endpoints
 #[derive(Debug, Clone, Default)]
 pub struct EndpointServerPools(pub Vec<PoolEndpoints>);
 
@@ -532,6 +532,8 @@ impl EndpointServerPools {
 
         nodes
     }
+
+    #[instrument]
     pub fn hosts_sorted(&self) -> Vec<Option<XHost>> {
         let (mut peers, local) = self.peers();
 
@@ -604,7 +606,6 @@ impl EndpointServerPools {
 
 #[cfg(test)]
 mod test {
-
     use super::*;
     use std::path::Path;
 

ecstore/src/global.rs

@@ -61,7 +61,7 @@ pub fn get_global_deployment_id() -> Option<String> {
 pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
     GLOBAL_Endpoints
         .set(EndpointServerPools::from(eps))
-        .expect("GLOBAL_Endpoints set faild")
+        .expect("GLOBAL_Endpoints set failed")
 }
 
 pub fn get_global_endpoints() -> EndpointServerPools {

ecstore/src/heal/data_scanner.rs

@@ -18,7 +18,10 @@ use super::{
     data_usage_cache::{DataUsageCache, DataUsageEntry, DataUsageHash},
     heal_commands::{HealScanMode, HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN},
 };
-use crate::heal::data_usage::DATA_USAGE_ROOT;
+use crate::{
+    bucket::{versioning::VersioningApi, versioning_sys::BucketVersioningSys},
+    heal::data_usage::DATA_USAGE_ROOT,
+};
 use crate::{
     cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
     config::{
@@ -49,7 +52,7 @@ use common::error::{Error, Result};
 use lazy_static::lazy_static;
 use rand::Rng;
 use rmp_serde::{Deserializer, Serializer};
-use s3s::dto::{ReplicationConfiguration, ReplicationRuleStatus};
+use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
 use serde::{Deserialize, Serialize};
 use tokio::{
     sync::{
@@ -352,7 +355,7 @@ impl CurrentScannerCycle {
 
             let str_len = rmp::decode::read_str_len(&mut cur)?;
 
-            // ！！！ Vec::with_capacity(str_len) 失败，vec!正常
+            // ！！！Vec::with_capacity(str_len) 失败，vec! 正常
             let mut field_buff = vec![0u8; str_len as usize];
 
             cur.read_exact(&mut field_buff)?;
@@ -412,7 +415,7 @@ pub struct ScannerItem {
     pub prefix: String,
     pub object_name: String,
     pub replication: Option<ReplicationConfiguration>,
-    // todo: lifecycle
+    pub lifecycle: Option<BucketLifecycleConfiguration>,
     // typ: fs::Permissions,
     pub heal: Heal,
     pub debug: bool,
@@ -435,7 +438,7 @@ impl ScannerItem {
 
     pub async fn apply_versions_actions(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
         let obj_infos = self.apply_newer_noncurrent_version_limit(fivs).await?;
-        if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst).try_into().unwrap() {
+        if obj_infos.len() >= SCANNER_EXCESS_OBJECT_VERSIONS.load(Ordering::SeqCst) as usize {
             // todo
         }
 
@@ -444,12 +447,7 @@ impl ScannerItem {
             cumulative_size += obj_info.size;
         }
 
-        if cumulative_size
-            >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE
-                .load(Ordering::SeqCst)
-                .try_into()
-                .unwrap()
-        {
+        if cumulative_size >= SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL_SIZE.load(Ordering::SeqCst) as usize {
             //todo
         }
 
@@ -457,22 +455,31 @@ impl ScannerItem {
     }
 
     pub async fn apply_newer_noncurrent_version_limit(&self, fivs: &[FileInfo]) -> Result<Vec<ObjectInfo>> {
-        let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
-        let mut object_infos = Vec::new();
-        for info in fivs.iter() {
-            object_infos.push(info.to_object_info(&self.bucket, &self.object_path().to_string_lossy(), false));
+        // let done = ScannerMetrics::time(ScannerMetric::ApplyNonCurrent);
+        let versioned = match BucketVersioningSys::get(&self.bucket).await {
+            Ok(vcfg) => vcfg.versioned(self.object_path().to_str().unwrap_or_default()),
+            Err(_) => false,
+        };
+        let mut object_infos = Vec::with_capacity(fivs.len());
+
+        if self.lifecycle.is_none() {
+            for info in fivs.iter() {
+                object_infos.push(info.to_object_info(&self.bucket, &self.object_path().to_string_lossy(), versioned));
+            }
+            return Ok(object_infos);
         }
-        done().await;
+
+        // done().await;
 
         Ok(object_infos)
     }
 
-    pub async fn apply_actions(&self, _oi: &ObjectInfo, _size_s: &SizeSummary) -> (bool, usize) {
+    pub async fn apply_actions(&self, oi: &ObjectInfo, _size_s: &SizeSummary) -> (bool, usize) {
         let done = ScannerMetrics::time(ScannerMetric::Ilm);
         //todo: lifecycle
         done().await;
 
-        (false, 0)
+        (false, oi.size)
     }
 }
 
@@ -551,6 +558,7 @@ impl FolderScanner {
         true
     }
 
+    #[tracing::instrument(level = "info", skip_all)]
     async fn scan_folder(&mut self, folder: &CachedFolder, into: &mut DataUsageEntry) -> Result<()> {
         let this_hash = hash_path(&folder.name);
         let was_compacted = into.compacted;
@@ -564,8 +572,18 @@ impl FolderScanner {
 
             let (_, prefix) = path_to_bucket_object_with_base_path(&self.root, &folder.name);
             // Todo: lifeCycle
+            let active_life_cycle = if let Some(lc) = self.old_cache.info.life_cycle.as_ref() {
+                if lc_has_active_rules(lc, &prefix) {
+                    self.old_cache.info.life_cycle.clone()
+                } else {
+                    None
+                }
+            } else {
+                None
+            };
+
             let replication_cfg = if self.old_cache.info.replication.is_some()
-                && has_active_rules(self.old_cache.info.replication.as_ref().unwrap(), &prefix, true)
+                && rep_has_active_rules(self.old_cache.info.replication.as_ref().unwrap(), &prefix, true)
             {
                 self.old_cache.info.replication.clone()
             } else {
@@ -596,7 +614,7 @@ impl FolderScanner {
                         continue;
                     }
 
-                    if !sub_path.is_dir() {
+                    if sub_path.is_dir() {
                         let h = hash_path(ent_name.to_str().unwrap());
                         if h == this_hash {
                             continue;
@@ -641,6 +659,7 @@ impl FolderScanner {
                             .unwrap_or_default(),
                         debug: self.data_usage_scanner_debug,
                         replication: replication_cfg.clone(),
+                        lifecycle: active_life_cycle.clone(),
                         heal: Heal::default(),
                     };
 
@@ -684,16 +703,11 @@ impl FolderScanner {
             }
 
             let should_compact = self.new_cache.info.name != folder.name
-                && existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS.try_into().unwrap()
-                || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap();
+                && (existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS as usize
+                    || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS as usize);
 
             let total_folders = existing_folders.len() + new_folders.len();
-            if total_folders
-                > SCANNER_EXCESS_FOLDERS
-                    .load(std::sync::atomic::Ordering::SeqCst)
-                    .try_into()
-                    .unwrap()
-            {
+            if total_folders > SCANNER_EXCESS_FOLDERS.load(Ordering::SeqCst) as usize {
                 let _prefix_name = format!("{}/", folder.name.trim_end_matches('/'));
                 // todo: notification
             }
@@ -941,7 +955,7 @@ impl FolderScanner {
                 let _ = list_path_raw(rx, lopts).await;
 
                 if *found_objs.read().await {
-                    let this = CachedFolder {
+                    let this: CachedFolder = CachedFolder {
                         name: k.clone(),
                         parent: this_hash.clone(),
                         object_heal_prob_div: 1,
@@ -957,7 +971,7 @@ impl FolderScanner {
         if !into.compacted && self.new_cache.info.name != folder.name {
             let mut flat = self.new_cache.size_recursive(&this_hash.key()).unwrap_or_default();
             flat.compacted = true;
-            let compact = if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT.try_into().unwrap() {
+            let compact = if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT as usize {
                 true
             } else {
                 // Compact if we only have objects as children...
@@ -1002,6 +1016,7 @@ impl FolderScanner {
         Ok(())
     }
 
+    #[tracing::instrument(level = "info", skip_all)]
     async fn send_update(&mut self) {
         if SystemTime::now().duration_since(self.last_update).unwrap() < Duration::from_secs(60) {
             return;
@@ -1013,8 +1028,9 @@ impl FolderScanner {
     }
 }
 
+#[tracing::instrument(level = "info", skip(into, folder_scanner))]
 async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner: &mut FolderScanner) {
-    let mut dst = if !into.compacted {
+    let mut dst = if into.compacted {
         DataUsageEntry::default()
     } else {
         into.clone()
@@ -1034,7 +1050,74 @@ async fn scan(folder: &CachedFolder, into: &mut DataUsageEntry, folder_scanner:
     }
 }
 
-pub fn has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
+fn lc_get_prefix(rule: &LifecycleRule) -> String {
+    if let Some(p) = &rule.prefix {
+        return p.to_string();
+    } else if let Some(filter) = &rule.filter {
+        if let Some(p) = &filter.prefix {
+            return p.to_string();
+        } else if let Some(and) = &filter.and {
+            if let Some(p) = &and.prefix {
+                return p.to_string();
+            }
+        }
+    }
+
+    "".into()
+}
+
+pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool {
+    if config.rules.is_empty() {
+        return false;
+    }
+
+    for rule in config.rules.iter() {
+        if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) {
+            continue;
+        }
+        let rule_prefix = lc_get_prefix(rule);
+        if !prefix.is_empty() && !rule_prefix.is_empty() {
+            if !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix) {
+                continue;
+            }
+        }
+
+        if let Some(e) = &rule.noncurrent_version_expiration {
+            if let Some(true) = e.noncurrent_days.map(|d| d > 0) {
+                return true;
+            }
+            if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
+                return true;
+            }
+        }
+
+        if rule.noncurrent_version_transitions.is_some() {
+            return true;
+        }
+        if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) {
+            return true;
+        }
+
+        if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) {
+            return true;
+        }
+
+        if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker.map(|m| m)) {
+            return true;
+        }
+
+        if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) {
+            return true;
+        }
+
+        if rule.transitions.is_some() {
+            return true;
+        }
+    }
+    false
+}
+
+pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
     if config.rules.is_empty() {
         return false;
     }
@@ -1092,8 +1175,14 @@ pub async fn scan_data_folder(
         root: base_path,
         get_size: get_size_fn,
         old_cache: cache.clone(),
-        new_cache: DataUsageCache::default(),
-        update_cache: DataUsageCache::default(),
+        new_cache: DataUsageCache {
+            info: cache.info.clone(),
+            ..Default::default()
+        },
+        update_cache: DataUsageCache {
+            info: cache.info.clone(),
+            ..Default::default()
+        },
         data_usage_scanner_debug: false,
         heal_object_select: 0,
         scan_mode: heal_scan_mode,
@@ -1121,8 +1210,7 @@ pub async fn scan_data_folder(
     if s.scan_folder(&folder, &mut root).await.is_err() {
         close_disk().await;
     }
-    s.new_cache
-        .force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN.try_into().unwrap());
+    s.new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN as usize);
     s.new_cache.info.last_update = Some(SystemTime::now());
     s.new_cache.info.next_cycle = cache.info.next_cycle;
     close_disk().await;

ecstore/src/heal/data_usage_cache.rs

@@ -10,7 +10,7 @@ use http::HeaderMap;
 use path_clean::PathClean;
 use rand::Rng;
 use rmp_serde::Serializer;
-use s3s::dto::ReplicationConfiguration;
+use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::hash::{DefaultHasher, Hash, Hasher};
@@ -347,7 +347,8 @@ pub struct DataUsageCacheInfo {
     pub next_cycle: u32,
     pub last_update: Option<SystemTime>,
     pub skip_healing: bool,
-    // todo: life_cycle
+    #[serde(skip)]
+    pub life_cycle: Option<BucketLifecycleConfiguration>,
     // pub life_cycle:
     #[serde(skip)]
     pub updates: Option<Sender<DataUsageEntry>>,
@@ -380,7 +381,7 @@ impl DataUsageCache {
         let mut retries = 0;
         while retries < 5 {
             let path = Path::new(BUCKET_META_PREFIX).join(name);
-            warn!("Loading data usage cache from backend: {}", path.display());
+            // warn!("Loading data usage cache from backend: {}", path.display());
             match store
                 .get_object_reader(
                     RUSTFS_META_BUCKET,
@@ -588,7 +589,7 @@ impl DataUsageCache {
             Some(e) => e,
             None => return,
         };
-        if top_e.children.len() > DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap() {
+        if top_e.children.len() > <u64 as TryInto<usize>>::try_into(DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS).unwrap() {
             self.reduce_children_of(&hash_path(&self.info.name), limit, true);
         }
         if self.cache.len() <= limit {

ecstore/src/heal/heal_ops.rs

@@ -406,10 +406,7 @@ impl HealSequence {
 
     async fn traverse_and_heal(h: Arc<HealSequence>) {
         let buckets_only = false;
-        let result = match Self::heal_items(h.clone(), buckets_only).await {
-            Ok(_) => None,
-            Err(err) => Some(err),
-        };
+        let result = (Self::heal_items(h.clone(), buckets_only).await).err();
         let _ = h.traverse_and_heal_done_tx.read().await.send(result).await;
     }
 

ecstore/src/io.rs

@@ -140,10 +140,14 @@ impl<R> EtagReader<R> {
 
 impl<R: AsyncRead + Unpin> AsyncRead for EtagReader<R> {
     fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<tokio::io::Result<()>> {
+        let befor_size = buf.filled().len();
+
         match Pin::new(&mut self.inner).poll_read(cx, buf) {
             Poll::Ready(Ok(())) => {
-                let bytes = buf.filled();
-                self.md5.update(bytes);
+                if buf.filled().len() > befor_size {
+                    let bytes = &buf.filled()[befor_size..];
+                    self.md5.update(bytes);
+                }
 
                 Poll::Ready(Ok(()))
             }

ecstore/src/peer.rs

@@ -80,12 +80,7 @@ impl S3PeerSys {
         let mut futures = Vec::with_capacity(self.clients.len());
         for client in self.clients.iter() {
             // client_clon
-            futures.push(async move {
-                match client.get_bucket_info(bucket, &BucketOptions::default()).await {
-                    Ok(_) => None,
-                    Err(err) => Some(err),
-                }
-            });
+            futures.push(async move { (client.get_bucket_info(bucket, &BucketOptions::default()).await).err() });
         }
         let errs = join_all(futures).await;
 

ecstore/src/set_disk.rs

@@ -1235,7 +1235,7 @@ impl SetDisks {
 
         let shallow_versions: Vec<Vec<FileMetaShallowVersion>> = metadata_shallow_versions.iter().flatten().cloned().collect();
 
-        let read_quorum = (fileinfos.len() + 1) / 2;
+        let read_quorum = fileinfos.len().div_ceil(2);
         let versions = merge_file_meta_versions(read_quorum, false, 1, &shallow_versions);
         let meta = FileMeta {
             versions,
@@ -3020,7 +3020,7 @@ impl SetDisks {
                             });
                             // Calc usage
                             let before = cache.info.last_update;
-                            let cache = match disk.clone().ns_scanner(&cache, tx, heal_scan_mode, None).await {
+                            let mut cache = match disk.clone().ns_scanner(&cache, tx, heal_scan_mode, None).await {
                                 Ok(cache) => cache,
                                 Err(_) => {
                                     if cache.info.last_update > before {
@@ -3030,6 +3030,9 @@ impl SetDisks {
                                     continue;
                                 }
                             };
+
+                            cache.info.updates = None;
+                            let _ = task.await;
                             let mut root = DataUsageEntry::default();
                             if let Some(r) = cache.root() {
                                 root = cache.flatten(&r);
@@ -3046,7 +3049,6 @@ impl SetDisks {
                                     entry: root,
                                 })
                                 .await;
-                            let _ = task.await;
                             let _ = cache.save(&cache_name.to_string_lossy()).await;
                         }
                     }
@@ -5090,7 +5092,7 @@ fn is_object_dang_ling(
     });
 
     if !valid_meta.is_valid() {
-        let data_blocks = (meta_arr.len() + 1) / 2;
+        let data_blocks = meta_arr.len().div_ceil(2);
         if not_found_parts_errs > data_blocks {
             return Ok(valid_meta);
         }
@@ -5103,7 +5105,7 @@ fn is_object_dang_ling(
     }
 
     if valid_meta.deleted {
-        let data_blocks = (errs.len() + 1) / 2;
+        let data_blocks = errs.len().div_ceil(2);
         if not_found_meta_errs > data_blocks {
             return Ok(valid_meta);
         }
@@ -5318,7 +5320,7 @@ async fn disks_with_all_parts(
             if let Some(data) = &meta.data {
                 let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number);
                 let data_len = data.len();
-                let verify_err = match bitrot_verify(
+                let verify_err = (bitrot_verify(
                     Box::new(Cursor::new(data.clone())),
                     data_len,
                     meta.erasure.shard_file_size(meta.size),
@@ -5326,11 +5328,8 @@ async fn disks_with_all_parts(
                     checksum_info.hash,
                     meta.erasure.shard_size(meta.erasure.block_size),
                 )
-                .await
-                {
-                    Ok(_) => None,
-                    Err(err) => Some(err),
-                };
+                .await)
+                    .err();
 
                 if let Some(vec) = data_errs_by_part.get_mut(&0) {
                     if index < vec.len() {

ecstore/src/store.rs

@@ -608,7 +608,7 @@ impl ECStore {
 
         let mut ress = Vec::new();
 
-        // join_all结果跟输入顺序一致
+        // join_all 结果跟输入顺序一致
         for (i, res) in results.into_iter().enumerate() {
             let index = i;
 
@@ -740,7 +740,7 @@ impl ECStore {
                 let cancel_clone = cancel.clone();
                 let all_buckets_clone = all_buckets.clone();
                 futures.push(async move {
-                    let (tx, mut rx) = mpsc::channel(100);
+                    let (tx, mut rx) = mpsc::channel(1);
                     let task = tokio::spawn(async move {
                         loop {
                             match rx.recv().await {
@@ -951,6 +951,7 @@ impl ECStore {
     }
 }
 
+#[tracing::instrument(level = "info", skip(all_buckets, updates))]
 async fn update_scan(
     all_merged: Arc<RwLock<DataUsageCache>>,
     results: Arc<RwLock<Vec<DataUsageCache>>>,
@@ -972,7 +973,7 @@ async fn update_scan(
         }
         w.merge(info);
     }
-    if w.info.last_update > *last_update && w.root().is_none() {
+    if (last_update.is_none() || w.info.last_update > *last_update) && w.root().is_some() {
         let _ = updates.send(w.dui(&w.info.name, &all_buckets)).await;
         *last_update = w.info.last_update;
     }
@@ -1010,7 +1011,7 @@ pub async fn all_local_disk() -> Vec<DiskStore> {
         .collect()
 }
 
-// init_local_disks 初始化本地磁盘，server启动前必须初始化成功
+// init_local_disks 初始化本地磁盘，server 启动前必须初始化成功
 pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
     let opt = &DiskOption {
         cleanup: true,
@@ -1279,7 +1280,7 @@ impl StorageAPI for ECStore {
 
         // TODO: replication opts.srdelete_op
 
-        // 删除meta
+        // 删除 meta
         self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str())
             .await?;
         Ok(())
@@ -1483,7 +1484,7 @@ impl StorageAPI for ECStore {
         //     results.push(jh.await.unwrap());
         // }
 
-        // 记录pool Index 对应的objects pool_idx -> objects idx
+        // 记录 pool Index 对应的 objects pool_idx -> objects idx
         let mut pool_obj_idx_map = HashMap::new();
         let mut orig_index_map = HashMap::new();
 
@@ -1533,9 +1534,9 @@ impl StorageAPI for ECStore {
 
         if !pool_obj_idx_map.is_empty() {
             for (i, sets) in self.pools.iter().enumerate() {
-                //  取pool idx 对应的 objects index
+                //  取 pool idx 对应的 objects index
                 if let Some(objs) = pool_obj_idx_map.get(&i) {
-                    //  取对应obj,理论上不会none
+                    //  取对应 obj，理论上不会 none
                     // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
 
                     if objs.is_empty() {
@@ -1544,10 +1545,10 @@ impl StorageAPI for ECStore {
 
                     let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;
 
-                    // 同时存入不可能为none
+                    // 同时存入不可能为 none
                     let org_indexes = orig_index_map.get(&i).unwrap();
 
-                    // perrs的顺序理论上跟obj_idxs顺序一致
+                    // perrs 的顺序理论上跟 obj_idxs 顺序一致
                     for (i, err) in perrs.into_iter().enumerate() {
                         let obj_idx = org_indexes[i];
 
@@ -1580,7 +1581,7 @@ impl StorageAPI for ECStore {
         let object = utils::path::encode_dir_object(object);
         let object = object.as_str();
 
-        // 查询在哪个pool
+        // 查询在哪个 pool
         let (mut pinfo, errs) = self
             .get_pool_info_existing_with_opts(bucket, object, &opts)
             .await

ecstore/src/store_list_objects.rs

@@ -25,7 +25,7 @@ use std::io::ErrorKind;
 use std::sync::Arc;
 use tokio::sync::broadcast::{self, Receiver as B_Receiver};
 use tokio::sync::mpsc::{self, Receiver, Sender};
-use tracing::{error, warn};
+use tracing::error;
 use uuid::Uuid;
 
 const MAX_OBJECT_LIST: i32 = 1000;
@@ -248,7 +248,7 @@ impl ECStore {
             ..Default::default()
         };
 
-        warn!("list_objects_generic opts {:?}", &opts);
+        // warn!("list_objects_generic opts {:?}", &opts);
 
         // use get
         if !opts.prefix.is_empty() && opts.limit == 1 && opts.marker.is_none() {
@@ -698,7 +698,7 @@ impl ECStore {
                 futures.push(async move {
                     let mut ask_disks = get_list_quorum(&opts.ask_disks, set.set_drive_count as i32);
                     if ask_disks == -1 {
-                        let new_disks = get_quorum_disks(&disks, &infos, (disks.len() + 1) / 2);
+                        let new_disks = get_quorum_disks(&disks, &infos, disks.len().div_ceil(2));
                         if !new_disks.is_empty() {
                             disks = new_disks;
                         } else {
@@ -1156,13 +1156,7 @@ async fn merge_entry_channels(
             if let Some(entry) = &best {
                 let mut versions = Vec::with_capacity(to_merge.len() + 1);
 
-                let mut has_xl = {
-                    if let Ok(meta) = entry.clone().xl_meta() {
-                        Some(meta)
-                    } else {
-                        None
-                    }
-                };
+                let mut has_xl = { entry.clone().xl_meta().ok() };
 
                 if let Some(x) = &has_xl {
                     versions.push(x.versions.clone());
@@ -1229,7 +1223,7 @@ impl SetDisks {
 
         let mut ask_disks = get_list_quorum(&opts.ask_disks, self.set_drive_count as i32);
         if ask_disks == -1 {
-            let new_disks = get_quorum_disks(&disks, &infos, (disks.len() + 1) / 2);
+            let new_disks = get_quorum_disks(&disks, &infos, disks.len().div_ceil(2));
             if !new_disks.is_empty() {
                 disks = new_disks;
                 ask_disks = 1;

ecstore/src/utils/os/linux.rs

@@ -151,7 +151,7 @@ fn read_drive_stats(stats_file: &str) -> Result<IOStats> {
 fn read_stat(file_name: &str) -> Result<Vec<u64>> {
     // 打开文件
     let path = Path::new(file_name);
-    let file = File::open(&path)?;
+    let file = File::open(path)?;
 
     // 创建一个 BufReader
     let reader = io::BufReader::new(file);
@@ -161,7 +161,8 @@ fn read_stat(file_name: &str) -> Result<Vec<u64>> {
     if let Some(line) = reader.lines().next() {
         let line = line?;
         // 分割行并解析为 u64
-        for token in line.trim().split_whitespace() {
+        // https://rust-lang.github.io/rust-clippy/master/index.html#trim_split_whitespace
+        for token in line.split_whitespace() {
             let ui64: u64 = token.parse()?;
             stats.push(ui64);
         }

ecstore/src/utils/os/windows.rs

@@ -1,7 +1,8 @@
 #![allow(unsafe_code)] // TODO: audit unsafe code
 
 use super::IOStats;
-use crate::{disk::Info, error::Result};
+use crate::disk::Info;
+use common::error::Result;
 use std::io::{Error, ErrorKind};
 use std::mem;
 use std::os::windows::ffi::OsStrExt;