merge heal

Cargo.lock

@@ -427,7 +427,9 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 name = "common"
 version = "0.0.1"
 dependencies = [
+ "async-trait",
  "lazy_static",
+ "scopeguard",
  "tokio",
  "tonic",
  "tracing-error",
@@ -655,6 +657,7 @@ dependencies = [
  "url",
  "uuid",
  "winapi",
+ "workers",
  "xxhash-rust",
 ]
 
@@ -3207,6 +3210,14 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "workers"
+version = "0.0.1"
+dependencies = [
+ "common",
+ "tokio",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"

Cargo.toml

@@ -9,7 +9,7 @@ members = [
     "common/protos",
     "api/admin",
     "reader",
-    "router",
+    "router", "common/workers",
 ]
 
 [workspace.package]
@@ -23,6 +23,7 @@ version = "0.0.1"
 async-trait = "0.1.83"
 backon = "1.2.0"
 bytes = "1.8.0"
+bytesize = "1.3.0"
 clap = { version = "4.5.20", features = ["derive"] }
 ecstore = { path = "./ecstore" }
 flatbuffers = "24.3.25"
@@ -84,4 +85,4 @@ uuid = { version = "1.11.0", features = [
 log = "0.4.22"
 axum = "0.7.7"
 md-5 = "0.10.6"
-bytesize = "1.3.0"
+workers = { path = "./common/workers" }

common/common/Cargo.toml

@@ -4,7 +4,9 @@ version.workspace = true
 edition.workspace = true
 
 [dependencies]
+async-trait.workspace = true
 lazy_static.workspace = true
+scopeguard = "1.2.0"
 tokio.workspace = true
 tonic.workspace = true
-tracing-error.workspace = true
+tracing-error.workspace = true

common/common/src/last_minute.rs

@@ -0,0 +1,112 @@
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Clone, Debug, Default)]
+pub struct AccElem {
+    pub total: u64,
+    pub size: u64,
+    pub n: u64,
+}
+
+impl AccElem {
+    pub fn add(&mut self, dur: &Duration) {
+        let dur = dur.as_secs();
+        self.total += dur;
+        self.n += 1;
+    }
+
+    pub fn merge(&mut self, b: &AccElem) {
+        self.n += b.n;
+        self.total += b.total;
+        self.size += b.size;
+    }
+
+    pub fn avg(&self) -> Duration {
+        if self.n >= 1 && self.total > 0 {
+            return Duration::from_secs(self.total / self.n);
+        }
+        Duration::from_secs(0)
+    }
+}
+
+#[derive(Clone)]
+pub struct LastMinuteLatency {
+    pub totals: Vec<AccElem>,
+    pub last_sec: u64,
+}
+
+impl Default for LastMinuteLatency {
+    fn default() -> Self {
+        Self {
+            totals: vec![AccElem::default(); 60],
+            last_sec: Default::default(),
+        }
+    }
+}
+
+impl LastMinuteLatency {
+    pub fn merge(&mut self, o: &mut LastMinuteLatency) -> LastMinuteLatency {
+        let mut merged = LastMinuteLatency::default();
+        if self.last_sec > o.last_sec {
+            o.forward_to(self.last_sec);
+            merged.last_sec = self.last_sec;
+        } else {
+            self.forward_to(o.last_sec);
+            merged.last_sec = o.last_sec;
+        }
+
+        for i in 0..merged.totals.len() {
+            merged.totals[i] = AccElem {
+                total: self.totals[i].total + o.totals[i].total,
+                n: self.totals[i].n + o.totals[i].n,
+                size: self.totals[i].size + o.totals[i].size,
+            }
+        }
+        merged
+    }
+
+    pub fn add(&mut self, t: &Duration) {
+        let sec = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        self.forward_to(sec);
+        let win_idx = sec % 60;
+        self.totals[win_idx as usize].add(t);
+        self.last_sec = sec;
+    }
+
+    pub fn add_all(&mut self, sec: u64, a: &AccElem) {
+        self.forward_to(sec);
+        let win_idx = sec % 60;
+        self.totals[win_idx as usize].merge(a);
+        self.last_sec = sec;
+    }
+
+    pub fn get_total(&mut self) -> AccElem {
+        let mut res = AccElem::default();
+        let sec = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        self.forward_to(sec);
+        for elem in self.totals.iter() {
+            res.merge(elem);
+        }
+        res
+    }
+
+    pub fn forward_to(&mut self, t: u64) {
+        if self.last_sec >= t {
+            return;
+        }
+        if t - self.last_sec >= 60 {
+            self.totals = vec![AccElem::default(); 60];
+            return;
+        }
+        while self.last_sec != t {
+            let idx = (self.last_sec + 1) % 60;
+            self.totals[idx as usize] = AccElem::default();
+            self.last_sec += 1;
+        }
+    }
+}

common/common/src/lib.rs

@@ -1,2 +1,23 @@
 pub mod error;
 pub mod globals;
+pub mod last_minute;
+
+/// Defers evaluation of a block of code until the end of the scope.
+#[macro_export]
+macro_rules! defer {
+    ($($body:tt)*) => {
+        let _guard = {
+            pub struct Guard<F: FnOnce()>(Option<F>);
+
+            impl<F: FnOnce()> Drop for Guard<F> {
+                fn drop(&mut self) {
+                    (self.0).take().map(|f| f());
+                }
+            }
+
+            Guard(Some(|| {
+                let _ = { $($body)* };
+            }))
+        };
+    };
+}

common/lock/src/lrwmutex.rs

@@ -135,14 +135,14 @@ mod test {
         let id = "foo";
         let source = "dandan";
         let timeout = Duration::from_secs(5);
-        assert_eq!(true, l_rw_lock.get_lock(id, source, &timeout).await);
+        assert!(l_rw_lock.get_lock(id, source, &timeout).await);
         l_rw_lock.un_lock().await;
 
         l_rw_lock.lock().await;
 
-        assert_eq!(false, l_rw_lock.get_r_lock(id, source, &timeout).await);
+        assert!(!(l_rw_lock.get_r_lock(id, source, &timeout).await));
         l_rw_lock.un_lock().await;
-        assert_eq!(true, l_rw_lock.get_r_lock(id, source, &timeout).await);
+        assert!(l_rw_lock.get_r_lock(id, source, &timeout).await);
 
         Ok(())
     }
@@ -156,7 +156,7 @@ mod test {
         let one_fn = async {
             let one = Arc::clone(&l_rw_lock);
             let timeout = Duration::from_secs(1);
-            assert_eq!(true, one.get_lock(id, source, &timeout).await);
+            assert!(one.get_lock(id, source, &timeout).await);
             sleep(Duration::from_secs(5)).await;
             l_rw_lock.un_lock().await;
         };
@@ -164,9 +164,9 @@ mod test {
         let two_fn = async {
             let two = Arc::clone(&l_rw_lock);
             let timeout = Duration::from_secs(2);
-            assert_eq!(false, two.get_r_lock(id, source, &timeout).await);
+            assert!(!(two.get_r_lock(id, source, &timeout).await));
             sleep(Duration::from_secs(5)).await;
-            assert_eq!(true, two.get_r_lock(id, source, &timeout).await);
+            assert!(two.get_r_lock(id, source, &timeout).await);
             two.un_r_lock().await;
         };
 

common/lock/src/namespace_lock.rs

@@ -287,7 +287,7 @@ mod test {
                 })
                 .await?;
 
-        assert_eq!(result, true);
+        assert!(result);
         Ok(())
     }
 }

common/protos/src/generated/proto_gen/node_service.rs

@@ -15,6 +15,20 @@ pub struct PingResponse {
     pub body: ::prost::alloc::vec::Vec<u8>,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
+pub struct HealBucketRequest {
+    #[prost(string, tag = "1")]
+    pub bucket: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub options: ::prost::alloc::string::String,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct HealBucketResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, optional, tag = "2")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
 pub struct ListBucketRequest {
     #[prost(string, tag = "1")]
     pub options: ::prost::alloc::string::String,
@@ -565,6 +579,26 @@ pub struct DiskInfoResponse {
     #[prost(string, optional, tag = "3")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NsScannerRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub cache: ::prost::alloc::string::String,
+    #[prost(uint64, tag = "3")]
+    pub scan_mode: u64,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct NsScannerResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(string, tag = "2")]
+    pub update: ::prost::alloc::string::String,
+    #[prost(string, tag = "3")]
+    pub data_usage_cache: ::prost::alloc::string::String,
+    #[prost(string, optional, tag = "4")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
 /// lock api have same argument type
 #[derive(Clone, PartialEq, ::prost::Message)]
 pub struct GenerallyLockRequest {
@@ -673,6 +707,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "Ping"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn heal_bucket(
+            &mut self,
+            request: impl tonic::IntoRequest<super::HealBucketRequest>,
+        ) -> std::result::Result<tonic::Response<super::HealBucketResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/HealBucket");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "HealBucket"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn list_bucket(
             &mut self,
             request: impl tonic::IntoRequest<super::ListBucketRequest>,
@@ -1139,6 +1188,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "DiskInfo"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn ns_scanner(
+            &mut self,
+            request: impl tonic::IntoStreamingRequest<Message = super::NsScannerRequest>,
+        ) -> std::result::Result<tonic::Response<tonic::codec::Streaming<super::NsScannerResponse>>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic::codec::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/NsScanner");
+            let mut req = request.into_streaming_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "NsScanner"));
+            self.inner.streaming(req, path, codec).await
+        }
         pub async fn lock(
             &mut self,
             request: impl tonic::IntoRequest<super::GenerallyLockRequest>,
@@ -1243,6 +1307,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::PingRequest>,
         ) -> std::result::Result<tonic::Response<super::PingResponse>, tonic::Status>;
+        async fn heal_bucket(
+            &self,
+            request: tonic::Request<super::HealBucketRequest>,
+        ) -> std::result::Result<tonic::Response<super::HealBucketResponse>, tonic::Status>;
         async fn list_bucket(
             &self,
             request: tonic::Request<super::ListBucketRequest>,
@@ -1376,6 +1444,14 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::DiskInfoRequest>,
         ) -> std::result::Result<tonic::Response<super::DiskInfoResponse>, tonic::Status>;
+        /// Server streaming response type for the NsScanner method.
+        type NsScannerStream: tonic::codegen::tokio_stream::Stream<Item = std::result::Result<super::NsScannerResponse, tonic::Status>>
+            + std::marker::Send
+            + 'static;
+        async fn ns_scanner(
+            &self,
+            request: tonic::Request<tonic::Streaming<super::NsScannerRequest>>,
+        ) -> std::result::Result<tonic::Response<Self::NsScannerStream>, tonic::Status>;
         async fn lock(
             &self,
             request: tonic::Request<super::GenerallyLockRequest>,
@@ -1499,6 +1575,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/HealBucket" => {
+                    #[allow(non_camel_case_types)]
+                    struct HealBucketSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::HealBucketRequest> for HealBucketSvc<T> {
+                        type Response = super::HealBucketResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::HealBucketRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::heal_bucket(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = HealBucketSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/ListBucket" => {
                     #[allow(non_camel_case_types)]
                     struct ListBucketSvc<T: NodeService>(pub Arc<T>);
@@ -2369,6 +2473,35 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/NsScanner" => {
+                    #[allow(non_camel_case_types)]
+                    struct NsScannerSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::StreamingService<super::NsScannerRequest> for NsScannerSvc<T> {
+                        type Response = super::NsScannerResponse;
+                        type ResponseStream = T::NsScannerStream;
+                        type Future = BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<tonic::Streaming<super::NsScannerRequest>>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::ns_scanner(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = NsScannerSvc(inner);
+                        let codec = tonic::codec::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.streaming(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/Lock" => {
                     #[allow(non_camel_case_types)]
                     struct LockSvc<T: NodeService>(pub Arc<T>);

common/protos/src/node.proto

@@ -12,6 +12,16 @@ message PingResponse {
   bytes body = 2;
 }
 
+message HealBucketRequest {
+  string bucket = 1;
+  string options = 2;
+}
+
+message HealBucketResponse {
+  bool success = 1;
+  optional string error_info = 2;
+}
+
 message ListBucketRequest {
   string options = 1;
 }
@@ -382,6 +392,19 @@ message DiskInfoResponse {
   optional string error_info = 3;
 }
 
+message NsScannerRequest {
+  string disk = 1;
+  string cache = 2;
+  uint64 scan_mode = 3;
+}
+
+message NsScannerResponse {
+  bool success = 1;
+  string update = 2;
+  string data_usage_cache = 3;
+  optional string error_info = 4;
+}
+
 // lock api have same argument type
 message GenerallyLockRequest {
     string args = 1;
@@ -397,6 +420,7 @@ message GenerallyLockResponse {
 service NodeService {
 /* -------------------------------meta service-------------------------- */
   rpc Ping(PingRequest) returns (PingResponse) {};
+  rpc HealBucket(HealBucketRequest) returns (HealBucketResponse) {};
   rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {};
   rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {};
   rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {};
@@ -432,6 +456,7 @@ service NodeService {
   rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {};
   rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {};
   rpc DiskInfo(DiskInfoRequest) returns (DiskInfoResponse) {};
+  rpc NsScanner(stream NsScannerRequest) returns (stream NsScannerResponse) {};
 
 /* -------------------------------lock service-------------------------- */
 

common/workers/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "workers"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+common.workspace = true
+tokio.workspace = true

common/workers/src/lib.rs

@@ -0,0 +1 @@
+pub mod workers;

common/workers/src/workers.rs

@@ -0,0 +1,87 @@
+use std::sync::Arc;
+use tokio::sync::{Mutex, Notify};
+
+pub struct Workers {
+    available: Mutex<usize>, // 可用的工作槽
+    notify: Notify,          // 用于通知等待的任务
+    limit: usize,            // 最大并发工作数
+}
+
+impl Workers {
+    // 创建 Workers 对象，允许最多 n 个作业并发执行
+    pub fn new(n: usize) -> Result<Arc<Workers>, &'static str> {
+        if n == 0 {
+            return Err("n must be > 0");
+        }
+
+        Ok(Arc::new(Workers {
+            available: Mutex::new(n),
+            notify: Notify::new(),
+            limit: n,
+        }))
+    }
+
+    // 让一个作业获得执行的机会
+    pub async fn take(&self) {
+        let mut available = self.available.lock().await;
+        while *available == 0 {
+            // 等待直到有可用槽
+            self.notify.notified().await;
+            available = self.available.lock().await;
+        }
+        *available -= 1; // 减少可用槽
+    }
+
+    // 让一个作业释放其机会
+    pub async fn give(&self) {
+        let mut available = self.available.lock().await;
+        *available += 1; // 增加可用槽
+        self.notify.notify_one(); // 通知一个等待的任务
+    }
+
+    // 等待所有并发作业完成
+    pub async fn wait(&self) {
+        loop {
+            {
+                let available = self.available.lock().await;
+                if *available == self.limit {
+                    break;
+                }
+            }
+            // 等待直到所有槽都被释放
+            self.notify.notified().await;
+        }
+    }
+
+    pub async fn available(&self) -> usize {
+        *self.available.lock().await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::Duration;
+    use tokio::time::sleep;
+
+    #[tokio::test]
+    async fn test_workers() {
+        let workers = Arc::new(Workers::new(5).unwrap());
+
+        for _ in 0..4 {
+            let workers = workers.clone();
+            tokio::spawn(async move {
+                workers.take().await;
+                sleep(Duration::from_secs(3)).await;
+                workers.give().await;
+            });
+        }
+
+        // Sleep: wait for spawn task started
+        sleep(Duration::from_secs(1)).await;
+        workers.wait().await;
+        if workers.available().await != workers.limit {
+            assert!(false);
+        }
+    }
+}

e2e_test/src/reliant/lock.rs

@@ -33,13 +33,13 @@ async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
     let response = client.lock(request).await?.into_inner();
     println!("request ended");
     if let Some(error_info) = response.error_info {
-        assert!(false, "can not get lock: {}", error_info);
+        panic!("can not get lock: {}", error_info);
     }
 
     let request = Request::new(GenerallyLockRequest { args });
     let response = client.un_lock(request).await?.into_inner();
     if let Some(error_info) = response.error_info {
-        assert!(false, "can not get un_lock: {}", error_info);
+        panic!("can not get un_lock: {}", error_info);
     }
 
     Ok(())
@@ -58,17 +58,16 @@ async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
         vec![locker],
     )
     .await;
-    assert_eq!(
-        ns.0.write()
-            .await
-            .get_lock(&Options {
-                timeout: Duration::from_secs(5),
-                retry_interval: Duration::from_secs(1),
-            })
-            .await
-            .unwrap(),
-        true
-    );
+    assert!(ns
+        .0
+        .write()
+        .await
+        .get_lock(&Options {
+            timeout: Duration::from_secs(5),
+            retry_interval: Duration::from_secs(1),
+        })
+        .await
+        .unwrap());
 
     ns.0.write().await.un_lock().await.unwrap();
     Ok(())

ecstore/Cargo.toml

@@ -60,6 +60,7 @@ s3s-policy.workspace = true
 rand.workspace = true
 pin-project-lite.workspace = true
 md-5.workspace = true
+workers.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
 

ecstore/src/bitrot.rs

@@ -103,7 +103,7 @@ impl Hasher {
 }
 
 impl BitrotAlgorithm {
-    pub fn new(&self) -> Hasher {
+    pub fn new_hasher(&self) -> Hasher {
         match self {
             BitrotAlgorithm::SHA256 => Hasher::SHA256(Sha256::new()),
             BitrotAlgorithm::HighwayHash256 | BitrotAlgorithm::HighwayHash256S => {
@@ -186,10 +186,8 @@ pub fn new_bitrot_reader(
 }
 
 pub async fn close_bitrot_writers(writers: &mut [Option<BitrotWriter>]) -> Result<()> {
-    for w in writers.into_iter() {
-        if let Some(w) = w {
-            let _ = w.close().await?;
-        }
+    for w in writers.iter_mut().flatten() {
+        w.close().await?;
     }
 
     Ok(())
@@ -207,7 +205,7 @@ pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: BitrotAlgori
     if algo != BitrotAlgorithm::HighwayHash256S {
         return size;
     }
-    size.div_ceil(shard_size) * algo.new().size() + size
+    size.div_ceil(shard_size) * algo.new_hasher().size() + size
 }
 
 pub fn bitrot_verify(
@@ -219,7 +217,7 @@ pub fn bitrot_verify(
     mut shard_size: usize,
 ) -> Result<()> {
     if algo != BitrotAlgorithm::HighwayHash256S {
-        let mut h = algo.new();
+        let mut h = algo.new_hasher();
         h.update(r.get_ref());
         if h.finalize() != want {
             return Err(Error::new(DiskError::FileCorrupt));
@@ -227,7 +225,7 @@ pub fn bitrot_verify(
 
         return Ok(());
     }
-    let mut h = algo.new();
+    let mut h = algo.new_hasher();
     let mut hash_buf = vec![0; h.size()];
     let mut left = want_size;
 
@@ -271,7 +269,7 @@ impl WholeBitrotWriter {
             volume: volume.to_string(),
             file_path: file_path.to_string(),
             _shard_size: shard_size,
-            hash: algo.new(),
+            hash: algo.new_hasher(),
         }
     }
 }
@@ -353,7 +351,7 @@ impl StreamingBitrotWriter {
         algo: BitrotAlgorithm,
         shard_size: usize,
     ) -> Result<Self> {
-        let hasher = algo.new();
+        let hasher = algo.new_hasher();
         let (tx, mut rx) = mpsc::channel::<Option<Vec<u8>>>(10);
 
         let total_file_size = length.div_ceil(shard_size) * hasher.size() + length;
@@ -362,7 +360,7 @@ impl StreamingBitrotWriter {
         let task = spawn(async move {
             loop {
                 if let Some(Some(buf)) = rx.recv().await {
-                    let _ = writer.write(&buf).await.unwrap();
+                    writer.write(&buf).await.unwrap();
                     continue;
                 }
 
@@ -389,7 +387,7 @@ impl Writer for StreamingBitrotWriter {
             return Ok(());
         }
         self.hasher.reset();
-        self.hasher.update(&buf);
+        self.hasher.update(buf);
         let hash_bytes = self.hasher.clone().finalize();
         let _ = self.tx.send(Some(hash_bytes)).await?;
         let _ = self.tx.send(Some(buf.to_vec())).await?;
@@ -430,7 +428,7 @@ impl StreamingBitrotReader {
         till_offset: usize,
         shard_size: usize,
     ) -> Self {
-        let hasher = algo.new();
+        let hasher = algo.new_hasher();
         Self {
             disk,
             _data: data.to_vec(),
@@ -489,7 +487,7 @@ pub struct BitrotFileWriter {
 
 impl BitrotFileWriter {
     pub fn new(inner: FileWriter, algo: BitrotAlgorithm, _shard_size: usize) -> Self {
-        let hasher = algo.new();
+        let hasher = algo.new_hasher();
         Self {
             inner,
             hasher,
@@ -513,7 +511,7 @@ impl Writer for BitrotFileWriter {
             return Ok(());
         }
         self.hasher.reset();
-        self.hasher.update(&buf);
+        self.hasher.update(buf);
         let hash_bytes = self.hasher.clone().finalize();
         let _ = self.inner.write(&hash_bytes).await?;
         let _ = self.inner.write(buf).await?;
@@ -544,7 +542,7 @@ struct BitrotFileReader {
 
 impl BitrotFileReader {
     pub fn new(inner: FileReader, algo: BitrotAlgorithm, _till_offset: usize, shard_size: usize) -> Self {
-        let hasher = algo.new();
+        let hasher = algo.new_hasher();
         Self {
             inner,
             // till_offset: ceil(till_offset, shard_size) * hasher.size() + till_offset,
@@ -648,7 +646,7 @@ mod test {
             }
             let checksum = decode_to_vec(checksums.get(algo).unwrap()).unwrap();
 
-            let mut h = algo.new();
+            let mut h = algo.new_hasher();
             let mut msg = Vec::with_capacity(h.size() * h.block_size());
             let mut sum = Vec::with_capacity(h.size());
 
@@ -656,7 +654,7 @@ mod test {
                 h.update(&msg);
                 sum = h.finalize();
                 msg.extend(sum.clone());
-                h = algo.new();
+                h = algo.new_hasher();
             }
 
             if checksum != sum {

ecstore/src/bucket/metadata.rs

@@ -9,7 +9,6 @@ use s3s::dto::{
     BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
     ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
-use s3s::xml;
 use serde::Serializer;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;

ecstore/src/bucket/metadata_sys.rs

@@ -28,7 +28,7 @@ use super::target::BucketTargets;
 use lazy_static::lazy_static;
 
 lazy_static! {
-    static ref GLOBAL_BucketMetadataSys: Arc<RwLock<BucketMetadataSys>> = Arc::new(RwLock::new(BucketMetadataSys::new()));
+    pub static ref GLOBAL_BucketMetadataSys: Arc<RwLock<BucketMetadataSys>> = Arc::new(RwLock::new(BucketMetadataSys::new()));
 }
 
 pub async fn init_bucket_metadata_sys(api: ECStore, buckets: Vec<String>) {

ecstore/src/bucket/policy/bucket_policy.rs

@@ -2,7 +2,6 @@ use crate::error::{Error, Result};
 // use rmp_serde::Serializer as rmpSerializer;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
-use std::collections::HashSet;
 
 use super::{
     action::{Action, ActionSet, IAMActionConditionKeyMap},

ecstore/src/bucket/utils.rs

@@ -39,10 +39,8 @@ pub fn check_bucket_name_common(bucket_name: &str, strict: bool) -> Result<(), E
         if !VALID_BUCKET_NAME_STRICT.is_match(bucket_name_trimmed) {
             return Err(Error::msg("Bucket name contains invalid characters"));
         }
-    } else {
-        if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) {
-            return Err(Error::msg("Bucket name contains invalid characters"));
-        }
+    } else if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) {
+        return Err(Error::msg("Bucket name contains invalid characters"));
     }
     Ok(())
 }

ecstore/src/bucket/versioning/mod.rs

@@ -40,10 +40,8 @@ impl VersioningApi for VersioningConfiguration {
         }
 
         if let Some(status) = self.status.as_ref() {
-            if status.as_str() == BucketVersioningStatus::ENABLED {
-                if prefix.is_empty() {
-                    return false;
-                }
+            if status.as_str() == BucketVersioningStatus::ENABLED && prefix.is_empty() {
+                return false;
 
                 // TODO: ExcludeFolders
             }

ecstore/src/cache_value/cache.rs

@@ -1,5 +1,7 @@
 use std::{
     fmt::Debug,
+    future::Future,
+    pin::Pin,
     ptr,
     sync::{
         atomic::{AtomicPtr, AtomicU64, Ordering},
@@ -12,7 +14,7 @@ use tokio::{spawn, sync::Mutex};
 
 use crate::error::Result;
 
-type UpdateFn<T> = Box<dyn Fn() -> Result<T> + Send + Sync>;
+pub type UpdateFn<T> = Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<T>> + Send>> + Send + Sync + 'static>;
 
 #[derive(Clone, Debug, Default)]
 pub struct Opts {
@@ -54,8 +56,10 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
             .duration_since(UNIX_EPOCH)
             .expect("Time went backwards")
             .as_secs();
-        if v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() {
-            return Ok(v.unwrap());
+        if now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() {
+            if let Some(v) = v {
+                return Ok(v);
+            }
         }
 
         if self.opts.no_wait && v.is_some() && now - self.last_update_ms.load(Ordering::SeqCst) < self.ttl.as_secs() * 2 {
@@ -94,7 +98,7 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
     }
 
     async fn update(&self) -> Result<()> {
-        match (self.update_fn)() {
+        match (self.update_fn)().await {
             Ok(val) => {
                 self.val.store(Box::into_raw(Box::new(val)), Ordering::SeqCst);
                 let now = SystemTime::now()
@@ -110,7 +114,7 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
                     return Ok(());
                 }
 
-                return Err(err);
+                Err(err)
             }
         }
     }

ecstore/src/cache_value/metacache_set.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{future::Future, pin::Pin, sync::Arc};
 
 use tokio::{
     spawn,
@@ -14,6 +14,10 @@ use crate::{
     error::{Error, Result},
 };
 
+type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+type PartialFn = Box<dyn Fn(MetaCacheEntries, &[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+type FinishedFn = Box<dyn Fn(&[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
+
 #[derive(Default)]
 pub struct ListPathRawOptions {
     pub disks: Vec<Option<DiskStore>>,
@@ -26,9 +30,12 @@ pub struct ListPathRawOptions {
     pub min_disks: usize,
     pub report_not_found: bool,
     pub per_disk_limit: i32,
-    pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
-    pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
-    pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
+    pub agreed: Option<AgreedFn>,
+    pub partial: Option<PartialFn>,
+    pub finished: Option<FinishedFn>,
+    // pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
+    // pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
+    // pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
 }
 
 impl Clone for ListPathRawOptions {
@@ -38,12 +45,12 @@ impl Clone for ListPathRawOptions {
             fallback_disks: self.fallback_disks.clone(),
             bucket: self.bucket.clone(),
             path: self.path.clone(),
-            recursice: self.recursice.clone(),
+            recursice: self.recursice,
             filter_prefix: self.filter_prefix.clone(),
             forward_to: self.forward_to.clone(),
-            min_disks: self.min_disks.clone(),
-            report_not_found: self.report_not_found.clone(),
-            per_disk_limit: self.per_disk_limit.clone(),
+            min_disks: self.min_disks,
+            report_not_found: self.report_not_found,
+            per_disk_limit: self.per_disk_limit,
             ..Default::default()
         }
     }
@@ -73,7 +80,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
                     .walk_dir(WalkDirOptions {
                         bucket: opts_clone.bucket.clone(),
                         base_dir: opts_clone.path.clone(),
-                        recursive: opts_clone.recursice.clone(),
+                        recursive: opts_clone.recursice,
                         report_notfound: opts_clone.report_not_found,
                         filter_prefix: opts_clone.filter_prefix.clone(),
                         forward_to: opts_clone.forward_to.clone(),
@@ -185,8 +192,8 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
         }
 
         if has_err > 0 && has_err > opts.disks.len() - opts.min_disks {
-            if let Some(finished_fn) = opts.finished.clone() {
-                finished_fn(&errs);
+            if let Some(finished_fn) = opts.finished.as_ref() {
+                finished_fn(&errs).await;
             }
             let mut combined_err = Vec::new();
             errs.iter().zip(opts.disks.iter()).for_each(|(err, disk)| match (err, disk) {
@@ -203,24 +210,22 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
         }
 
         // Break if all at EOF or error.
-        if at_eof + has_err == readers.len() {
-            if has_err > 0 {
-                if let Some(finished_fn) = opts.finished.clone() {
-                    finished_fn(&errs);
-                }
-                break;
+        if at_eof + has_err == readers.len() && has_err > 0 {
+            if let Some(finished_fn) = opts.finished.as_ref() {
+                finished_fn(&errs).await;
             }
+            break;
         }
 
         if agree == readers.len() {
-            if let Some(agreed_fn) = opts.agreed.clone() {
-                agreed_fn(current);
+            if let Some(agreed_fn) = opts.agreed.as_ref() {
+                agreed_fn(current).await;
             }
             continue;
         }
 
-        if let Some(partial_fn) = opts.partial.clone() {
-            partial_fn(MetaCacheEntries(top_entries), &errs);
+        if let Some(partial_fn) = opts.partial.as_ref() {
+            partial_fn(MetaCacheEntries(top_entries), &errs).await;
         }
     }
 

ecstore/src/chunk_stream.rs

@@ -1,256 +1,256 @@
-use crate::error::StdError;
-use bytes::Bytes;
-use futures::pin_mut;
-use futures::stream::{Stream, StreamExt};
-use std::future::Future;
-use std::pin::Pin;
-use std::task::{Context, Poll};
-use transform_stream::AsyncTryStream;
+// use crate::error::StdError;
+// use bytes::Bytes;
+// use futures::pin_mut;
+// use futures::stream::{Stream, StreamExt};
+// use std::future::Future;
+// use std::pin::Pin;
+// use std::task::{Context, Poll};
+// use transform_stream::AsyncTryStream;
 
-pub type SyncBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + Sync + 'a>>;
+// pub type SyncBoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + Sync + 'a>>;
 
-pub struct ChunkedStream<'a> {
-    /// inner
-    inner: AsyncTryStream<Bytes, StdError, SyncBoxFuture<'a, Result<(), StdError>>>,
+// pub struct ChunkedStream<'a> {
+//     /// inner
+//     inner: AsyncTryStream<Bytes, StdError, SyncBoxFuture<'a, Result<(), StdError>>>,
 
-    remaining_length: usize,
-}
+//     remaining_length: usize,
+// }
 
-impl<'a> ChunkedStream<'a> {
-    pub fn new<S>(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self
-    where
-        S: Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'a,
-    {
-        let inner = AsyncTryStream::<_, _, SyncBoxFuture<'a, Result<(), StdError>>>::new(|mut y| {
-            #[allow(clippy::shadow_same)] // necessary for `pin_mut!`
-            Box::pin(async move {
-                pin_mut!(body);
-                // 上一次没用完的数据
-                let mut prev_bytes = Bytes::new();
-                let mut readed_size = 0;
+// impl<'a> ChunkedStream<'a> {
+//     pub fn new<S>(body: S, content_length: usize, chunk_size: usize, need_padding: bool) -> Self
+//     where
+//         S: Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'a,
+//     {
+//         let inner = AsyncTryStream::<_, _, SyncBoxFuture<'a, Result<(), StdError>>>::new(|mut y| {
+//             #[allow(clippy::shadow_same)] // necessary for `pin_mut!`
+//             Box::pin(async move {
+//                 pin_mut!(body);
+//                 // 上一次没用完的数据
+//                 let mut prev_bytes = Bytes::new();
+//                 let mut readed_size = 0;
 
-                loop {
-                    let data: Vec<Bytes> = {
-                        // 读固定大小的数据
-                        match Self::read_data(body.as_mut(), prev_bytes, chunk_size).await {
-                            None => break,
-                            Some(Err(e)) => return Err(e),
-                            Some(Ok((data, remaining_bytes))) => {
-                                // debug!(
-                                //     "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ",
-                                //     content_length,
-                                //     readed_size,
-                                //     data.len(),
-                                //     remaining_bytes.len()
-                                // );
+//                 loop {
+//                     let data: Vec<Bytes> = {
+//                         // 读固定大小的数据
+//                         match Self::read_data(body.as_mut(), prev_bytes, chunk_size).await {
+//                             None => break,
+//                             Some(Err(e)) => return Err(e),
+//                             Some(Ok((data, remaining_bytes))) => {
+//                                 // debug!(
+//                                 //     "content_length:{},readed_size:{}, read_data data:{}, remaining_bytes: {} ",
+//                                 //     content_length,
+//                                 //     readed_size,
+//                                 //     data.len(),
+//                                 //     remaining_bytes.len()
+//                                 // );
 
-                                prev_bytes = remaining_bytes;
-                                data
-                            }
-                        }
-                    };
+//                                 prev_bytes = remaining_bytes;
+//                                 data
+//                             }
+//                         }
+//                     };
 
-                    for bytes in data {
-                        readed_size += bytes.len();
-                        // debug!("readed_size {}, content_length {}", readed_size, content_length,);
-                        y.yield_ok(bytes).await;
-                    }
+//                     for bytes in data {
+//                         readed_size += bytes.len();
+//                         // debug!("readed_size {}, content_length {}", readed_size, content_length,);
+//                         y.yield_ok(bytes).await;
+//                     }
 
-                    if readed_size + prev_bytes.len() >= content_length {
-                        // debug!(
-                        //     "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}",
-                        //     readed_size,
-                        //     prev_bytes.len(),
-                        //     content_length,
-                        // );
+//                     if readed_size + prev_bytes.len() >= content_length {
+//                         // debug!(
+//                         //     "读完了 readed_size:{} + prev_bytes.len({}) == content_length {}",
+//                         //     readed_size,
+//                         //     prev_bytes.len(),
+//                         //     content_length,
+//                         // );
 
-                        // 填充0？
-                        if !need_padding {
-                            y.yield_ok(prev_bytes).await;
-                            break;
-                        }
+//                         // 填充0？
+//                         if !need_padding {
+//                             y.yield_ok(prev_bytes).await;
+//                             break;
+//                         }
 
-                        let mut bytes = vec![0u8; chunk_size];
-                        let (left, _) = bytes.split_at_mut(prev_bytes.len());
-                        left.copy_from_slice(&prev_bytes);
+//                         let mut bytes = vec![0u8; chunk_size];
+//                         let (left, _) = bytes.split_at_mut(prev_bytes.len());
+//                         left.copy_from_slice(&prev_bytes);
 
-                        y.yield_ok(Bytes::from(bytes)).await;
+//                         y.yield_ok(Bytes::from(bytes)).await;
 
-                        break;
-                    }
-                }
+//                         break;
+//                     }
+//                 }
 
-                // debug!("chunked stream exit");
+//                 // debug!("chunked stream exit");
 
-                Ok(())
-            })
-        });
-        Self {
-            inner,
-            remaining_length: content_length,
-        }
-    }
-    /// read data and return remaining bytes
-    async fn read_data<S>(
-        mut body: Pin<&mut S>,
-        prev_bytes: Bytes,
-        data_size: usize,
-    ) -> Option<Result<(Vec<Bytes>, Bytes), StdError>>
-    where
-        S: Stream<Item = Result<Bytes, StdError>> + Send,
-    {
-        let mut bytes_buffer = Vec::new();
+//                 Ok(())
+//             })
+//         });
+//         Self {
+//             inner,
+//             remaining_length: content_length,
+//         }
+//     }
+//     /// read data and return remaining bytes
+//     async fn read_data<S>(
+//         mut body: Pin<&mut S>,
+//         prev_bytes: Bytes,
+//         data_size: usize,
+//     ) -> Option<Result<(Vec<Bytes>, Bytes), StdError>>
+//     where
+//         S: Stream<Item = Result<Bytes, StdError>> + Send,
+//     {
+//         let mut bytes_buffer = Vec::new();
 
-        // 只执行一次
-        let mut push_data_bytes = |mut bytes: Bytes| {
-            // debug!("read from body {} split per {}, prev_bytes: {}", bytes.len(), data_size, prev_bytes.len());
+//         // 只执行一次
+//         let mut push_data_bytes = |mut bytes: Bytes| {
+//             // debug!("read from body {} split per {}, prev_bytes: {}", bytes.len(), data_size, prev_bytes.len());
 
-            if bytes.is_empty() {
-                return None;
-            }
+//             if bytes.is_empty() {
+//                 return None;
+//             }
 
-            if data_size == 0 {
-                return Some(bytes);
-            }
+//             if data_size == 0 {
+//                 return Some(bytes);
+//             }
 
-            // 合并上一次数据
-            if !prev_bytes.is_empty() {
-                let need_size = data_size.wrapping_sub(prev_bytes.len());
-                // debug!(
-                //     " 上一次有剩余{},从这一次中取{},共：{}",
-                //     prev_bytes.len(),
-                //     need_size,
-                //     prev_bytes.len() + need_size
-                // );
-                if bytes.len() >= need_size {
-                    let data = bytes.split_to(need_size);
-                    let mut combined = Vec::new();
-                    combined.extend_from_slice(&prev_bytes);
-                    combined.extend_from_slice(&data);
+//             // 合并上一次数据
+//             if !prev_bytes.is_empty() {
+//                 let need_size = data_size.wrapping_sub(prev_bytes.len());
+//                 // debug!(
+//                 //     " 上一次有剩余{},从这一次中取{},共：{}",
+//                 //     prev_bytes.len(),
+//                 //     need_size,
+//                 //     prev_bytes.len() + need_size
+//                 // );
+//                 if bytes.len() >= need_size {
+//                     let data = bytes.split_to(need_size);
+//                     let mut combined = Vec::new();
+//                     combined.extend_from_slice(&prev_bytes);
+//                     combined.extend_from_slice(&data);
 
-                    // debug!(
-                    //     "取到的长度大于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}",
-                    //     need_size,
-                    //     combined.len(),
-                    //     bytes.len(),
-                    // );
+//                     // debug!(
+//                     //     "取到的长度大于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}",
+//                     //     need_size,
+//                     //     combined.len(),
+//                     //     bytes.len(),
+//                     // );
 
-                    bytes_buffer.push(Bytes::from(combined));
-                } else {
-                    let mut combined = Vec::new();
-                    combined.extend_from_slice(&prev_bytes);
-                    combined.extend_from_slice(&bytes);
+//                     bytes_buffer.push(Bytes::from(combined));
+//                 } else {
+//                     let mut combined = Vec::new();
+//                     combined.extend_from_slice(&prev_bytes);
+//                     combined.extend_from_slice(&bytes);
 
-                    // debug!(
-                    //     "取到的长度小于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}，直接返回",
-                    //     need_size,
-                    //     combined.len(),
-                    //     bytes.len(),
-                    // );
+//                     // debug!(
+//                     //     "取到的长度小于所需，取出需要的长度：{},与上一次合并得到：{}，bytes剩余：{}，直接返回",
+//                     //     need_size,
+//                     //     combined.len(),
+//                     //     bytes.len(),
+//                     // );
 
-                    return Some(Bytes::from(combined));
-                }
-            }
+//                     return Some(Bytes::from(combined));
+//                 }
+//             }
 
-            // 取到的数据比需要的块大，从bytes中截取需要的块大小
-            if data_size <= bytes.len() {
-                let n = bytes.len() / data_size;
+//             // 取到的数据比需要的块大，从bytes中截取需要的块大小
+//             if data_size <= bytes.len() {
+//                 let n = bytes.len() / data_size;
 
-                for _ in 0..n {
-                    let data = bytes.split_to(data_size);
+//                 for _ in 0..n {
+//                     let data = bytes.split_to(data_size);
 
-                    // println!("bytes_buffer.push: {}， 剩余：{}", data.len(), bytes.len());
-                    bytes_buffer.push(data);
-                }
+//                     // println!("bytes_buffer.push: {}， 剩余：{}", data.len(), bytes.len());
+//                     bytes_buffer.push(data);
+//                 }
 
-                Some(bytes)
-            } else {
-                // 不够
-                Some(bytes)
-            }
-        };
+//                 Some(bytes)
+//             } else {
+//                 // 不够
+//                 Some(bytes)
+//             }
+//         };
 
-        // 剩余数据
-        let remaining_bytes = 'outer: {
-            // // 如果上一次数据足够，跳出
-            // if let Some(remaining_bytes) = push_data_bytes(prev_bytes) {
-            //     println!("从剩下的取");
-            //     break 'outer remaining_bytes;
-            // }
+//         // 剩余数据
+//         let remaining_bytes = 'outer: {
+//             // // 如果上一次数据足够，跳出
+//             // if let Some(remaining_bytes) = push_data_bytes(prev_bytes) {
+//             //     println!("从剩下的取");
+//             //     break 'outer remaining_bytes;
+//             // }
 
-            loop {
-                match body.next().await? {
-                    Err(e) => return Some(Err(e)),
-                    Ok(bytes) => {
-                        if let Some(remaining_bytes) = push_data_bytes(bytes) {
-                            break 'outer remaining_bytes;
-                        }
-                    }
-                }
-            }
-        };
+//             loop {
+//                 match body.next().await? {
+//                     Err(e) => return Some(Err(e)),
+//                     Ok(bytes) => {
+//                         if let Some(remaining_bytes) = push_data_bytes(bytes) {
+//                             break 'outer remaining_bytes;
+//                         }
+//                     }
+//                 }
+//             }
+//         };
 
-        Some(Ok((bytes_buffer, remaining_bytes)))
-    }
+//         Some(Ok((bytes_buffer, remaining_bytes)))
+//     }
 
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes, StdError>>> {
-        let ans = Pin::new(&mut self.inner).poll_next(cx);
-        if let Poll::Ready(Some(Ok(ref bytes))) = ans {
-            self.remaining_length = self.remaining_length.saturating_sub(bytes.len());
-        }
-        ans
-    }
+//     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Bytes, StdError>>> {
+//         let ans = Pin::new(&mut self.inner).poll_next(cx);
+//         if let Poll::Ready(Some(Ok(ref bytes))) = ans {
+//             self.remaining_length = self.remaining_length.saturating_sub(bytes.len());
+//         }
+//         ans
+//     }
 
-    // pub fn exact_remaining_length(&self) -> usize {
-    //     self.remaining_length
-    // }
-}
+//     // pub fn exact_remaining_length(&self) -> usize {
+//     //     self.remaining_length
+//     // }
+// }
 
-impl Stream for ChunkedStream<'_> {
-    type Item = Result<Bytes, StdError>;
+// impl Stream for ChunkedStream<'_> {
+//     type Item = Result<Bytes, StdError>;
 
-    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        self.poll(cx)
-    }
+//     fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+//         self.poll(cx)
+//     }
 
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (0, None)
-    }
-}
+//     fn size_hint(&self) -> (usize, Option<usize>) {
+//         (0, None)
+//     }
+// }
 
-#[cfg(test)]
-mod test {
+// #[cfg(test)]
+// mod test {
 
-    use super::*;
+//     use super::*;
 
-    #[tokio::test]
-    async fn test_chunked_stream() {
-        let chunk_size = 4;
+//     #[tokio::test]
+//     async fn test_chunked_stream() {
+//         let chunk_size = 4;
 
-        let data1 = vec![1u8; 7777]; // 65536
-        let data2 = vec![1u8; 7777]; // 65536
+//         let data1 = vec![1u8; 7777]; // 65536
+//         let data2 = vec![1u8; 7777]; // 65536
 
-        let content_length = data1.len() + data2.len();
+//         let content_length = data1.len() + data2.len();
 
-        let chunk1 = Bytes::from(data1);
-        let chunk2 = Bytes::from(data2);
+//         let chunk1 = Bytes::from(data1);
+//         let chunk2 = Bytes::from(data2);
 
-        let chunk_results: Vec<Result<Bytes, _>> = vec![Ok(chunk1), Ok(chunk2)];
+//         let chunk_results: Vec<Result<Bytes, _>> = vec![Ok(chunk1), Ok(chunk2)];
 
-        let stream = futures::stream::iter(chunk_results);
+//         let stream = futures::stream::iter(chunk_results);
 
-        let mut chunked_stream = ChunkedStream::new(stream, content_length, chunk_size, true);
+//         let mut chunked_stream = ChunkedStream::new(stream, content_length, chunk_size, true);
 
-        loop {
-            let ans1 = chunked_stream.next().await;
-            if ans1.is_none() {
-                break;
-            }
+//         loop {
+//             let ans1 = chunked_stream.next().await;
+//             if ans1.is_none() {
+//                 break;
+//             }
 
-            let bytes = ans1.unwrap().unwrap();
-            assert!(bytes.len() == chunk_size)
-        }
+//             let bytes = ans1.unwrap().unwrap();
+//             assert!(bytes.len() == chunk_size)
+//         }
 
-        // assert_eq!(ans1.unwrap(), chunk1_data.as_slice());
-    }
-}
+//         // assert_eq!(ans1.unwrap(), chunk1_data.as_slice());
+//     }
+// }

ecstore/src/config/common.rs

@@ -15,7 +15,7 @@ use s3s::dto::StreamingBlob;
 use s3s::Body;
 use tracing::error;
 
-const CONFIG_PREFIX: &str = "config";
+pub const CONFIG_PREFIX: &str = "config";
 const CONFIG_FILE: &str = "config.json";
 
 pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
@@ -161,32 +161,29 @@ async fn apply_dynamic_config(cfg: &mut Config, api: &ECStore) -> Result<()> {
     Ok(())
 }
 
-async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: &ECStore, subsys: &String) -> Result<()> {
+async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: &ECStore, subsys: &str) -> Result<()> {
     let set_drive_counts = api.set_drive_counts();
-    match subsys.as_str() {
-        STORAGE_CLASS_SUB_SYS => {
-            let kvs = match cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY) {
-                Some(res) => res,
-                None => KVS::new(),
-            };
+    if subsys == STORAGE_CLASS_SUB_SYS {
+        let kvs = match cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY) {
+            Some(res) => res,
+            None => KVS::new(),
+        };
 
-            for (i, count) in set_drive_counts.iter().enumerate() {
-                match storageclass::lookup_config(&kvs, *count) {
-                    Ok(res) => {
-                        if i == 0 && GLOBAL_StorageClass.get().is_none() {
-                            if let Err(r) = GLOBAL_StorageClass.set(res) {
-                                error!("GLOBAL_StorageClass.set failed {:?}", r);
-                            }
+        for (i, count) in set_drive_counts.iter().enumerate() {
+            match storageclass::lookup_config(&kvs, *count) {
+                Ok(res) => {
+                    if i == 0 && GLOBAL_StorageClass.get().is_none() {
+                        if let Err(r) = GLOBAL_StorageClass.set(res) {
+                            error!("GLOBAL_StorageClass.set failed {:?}", r);
                         }
                     }
-                    Err(err) => {
-                        error!("init storageclass err:{:?}", &err);
-                        break;
-                    }
+                }
+                Err(err) => {
+                    error!("init storageclass err:{:?}", &err);
+                    break;
                 }
             }
         }
-        _ => {}
     }
 
     Ok(())

ecstore/src/config/heal.rs

@@ -0,0 +1,65 @@
+use std::time::Duration;
+
+use crate::{
+    error::{Error, Result},
+    utils::bool_flag::parse_bool,
+};
+
+#[derive(Debug, Default)]
+pub struct Config {
+    pub bitrot: String,
+    pub sleep: Duration,
+    pub io_count: usize,
+    pub drive_workers: usize,
+    pub cache: Duration,
+}
+
+impl Config {
+    pub fn bitrot_scan_cycle(&self) -> Duration {
+        self.cache
+    }
+
+    pub fn get_workers(&self) -> usize {
+        self.drive_workers
+    }
+
+    pub fn update(&mut self, nopts: &Config) {
+        self.bitrot = nopts.bitrot.clone();
+        self.io_count = nopts.io_count;
+        self.sleep = nopts.sleep;
+        self.drive_workers = nopts.drive_workers;
+    }
+}
+
+const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
+
+fn parse_bitrot_config(s: &str) -> Result<Duration> {
+    match parse_bool(s) {
+        Ok(enabled) => {
+            if enabled {
+                Ok(Duration::from_secs_f64(0.0))
+            } else {
+                Ok(Duration::from_secs_f64(-1.0))
+            }
+        }
+        Err(_) => {
+            if !s.ends_with("m") {
+                return Err(Error::from_string("unknown format"));
+            }
+
+            match s.trim_end_matches('m').parse::<u64>() {
+                Ok(months) => {
+                    if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
+                        return Err(Error::from_string(format!(
+                            "minimum bitrot cycle is {} month(s)",
+                            RUSTFS_BITROT_CYCLE_IN_MONTHS
+                        )));
+                    }
+
+                    Ok(Duration::from_secs(months * 30 * 24 * 60))
+                }
+                Err(err) => Err(err.into()),
+            }
+        }
+    }
+}

ecstore/src/config/mod.rs

@@ -1,5 +1,7 @@
 pub mod common;
 pub mod error;
+#[allow(dead_code)]
+pub mod heal;
 pub mod storageclass;
 
 use crate::error::Result;
@@ -17,8 +19,16 @@ lazy_static! {
     pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
 }
 
+pub static RUSTFS_CONFIG_PREFIX: &str = "config";
+
 pub struct ConfigSys {}
 
+impl Default for ConfigSys {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl ConfigSys {
     pub fn new() -> Self {
         Self {}
@@ -44,6 +54,12 @@ pub struct KV {
 #[derive(Debug, Deserialize, Serialize, Clone)]
 pub struct KVS(Vec<KV>);
 
+impl Default for KVS {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl KVS {
     pub fn new() -> Self {
         KVS(Vec::new())
@@ -69,6 +85,12 @@ impl KVS {
 #[derive(Debug, Clone)]
 pub struct Config(HashMap<String, HashMap<String, KVS>>);
 
+impl Default for Config {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Config {
     pub fn new() -> Self {
         let mut cfg = Config(HashMap::new());

ecstore/src/config/storageclass.rs

@@ -228,7 +228,7 @@ pub fn parse_storage_class(env: &str) -> Result<StorageClass> {
 
     // only two elements allowed in the string - "scheme" and "number of parity drives"
     if s.len() != 2 {
-        return Err(Error::msg(&format!(
+        return Err(Error::msg(format!(
             "Invalid storage class format: {}. Expected 'Scheme:Number of parity drives'.",
             env
         )));
@@ -236,13 +236,13 @@ pub fn parse_storage_class(env: &str) -> Result<StorageClass> {
 
     // only allowed scheme is "EC"
     if s[0] != SCHEME_PREFIX {
-        return Err(Error::msg(&format!("Unsupported scheme {}. Supported scheme is EC.", s[0])));
+        return Err(Error::msg(format!("Unsupported scheme {}. Supported scheme is EC.", s[0])));
     }
 
     // Number of parity drives should be integer
     let parity_drives: usize = match s[1].parse() {
         Ok(num) => num,
-        Err(_) => return Err(Error::msg(&format!("Failed to parse parity value: {}.", s[1]))),
+        Err(_) => return Err(Error::msg(format!("Failed to parse parity value: {}.", s[1]))),
     };
 
     Ok(StorageClass { parity: parity_drives })

ecstore/src/disk/endpoint.rs

@@ -2,6 +2,7 @@ use crate::error::{Error, Result};
 use crate::utils::net;
 use path_absolutize::Absolutize;
 use path_clean::PathClean;
+use std::fs;
 use std::{fmt::Display, path::Path};
 use url::{ParseError, Url};
 
@@ -29,7 +30,13 @@ pub struct Endpoint {
 impl Display for Endpoint {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if self.url.scheme() == "file" {
-            write!(f, "{}", self.url.path())
+            write!(
+                f,
+                "{}",
+                fs::canonicalize(self.url.path())
+                    .map_err(|_| std::fmt::Error)?
+                    .to_string_lossy()
+            )
         } else {
             write!(f, "{}", self.url)
         }

ecstore/src/disk/error.rs

@@ -106,6 +106,9 @@ pub enum DiskError {
 
     #[error("part missing or corrupt")]
     PartMissingOrCorrupt,
+
+    #[error("No healing is required")]
+    NoHealRequired,
 }
 
 impl DiskError {
@@ -210,6 +213,7 @@ pub fn clone_disk_err(e: &DiskError) -> Error {
         DiskError::MoreData => Error::new(DiskError::MoreData),
         DiskError::OutdatedXLMeta => Error::new(DiskError::OutdatedXLMeta),
         DiskError::PartMissingOrCorrupt => Error::new(DiskError::PartMissingOrCorrupt),
+        DiskError::NoHealRequired => Error::new(DiskError::NoHealRequired),
     }
 }
 
@@ -261,14 +265,7 @@ pub fn os_err_to_file_err(e: io::Error) -> Error {
 }
 
 pub fn is_err_file_not_found(err: &Error) -> bool {
-    if let Some(e) = err.downcast_ref::<DiskError>() {
-        match e {
-            DiskError::FileNotFound => true,
-            _ => false,
-        }
-    } else {
-        false
-    }
+    matches!(err.downcast_ref::<DiskError>(), Some(DiskError::FileNotFound))
 }
 
 pub fn is_sys_err_no_space(e: &io::Error) -> bool {
@@ -422,18 +419,32 @@ pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error {
 }
 
 pub fn is_all_not_found(errs: &[Option<Error>]) -> bool {
-    for err in errs.iter() {
-        if let Some(err) = err {
-            if let Some(err) = err.downcast_ref::<DiskError>() {
-                match err {
-                    DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
-                        continue;
-                    }
-                    _ => return false,
+    for err in errs.iter().flatten() {
+        if let Some(err) = err.downcast_ref::<DiskError>() {
+            match err {
+                DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => {
+                    continue;
                 }
+                _ => return false,
             }
         }
     }
 
     !errs.is_empty()
 }
+
+pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
+    if errs.is_empty() {
+        return false;
+    }
+    let mut not_found_count = 0;
+    for err in errs.iter().flatten() {
+        match err.downcast_ref() {
+            Some(DiskError::VolumeNotFound) | Some(DiskError::DiskNotFound) => {
+                not_found_count += 1;
+            }
+            _ => {}
+        }
+    }
+    errs.len() == not_found_count
+}

ecstore/src/disk/format.rs

@@ -1,4 +1,4 @@
-use super::error::DiskError;
+use super::{error::DiskError, DiskInfo};
 use crate::error::{Error, Result};
 use serde::{Deserialize, Serialize};
 use serde_json::Error as JsonError;
@@ -31,7 +31,7 @@ pub enum FormatBackend {
 ///
 /// The V3 format to support "large bucket" support where a bucket
 /// can span multiple erasure sets.
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct FormatErasureV3 {
     /// Version of 'xl' format.
     pub version: FormatErasureVersion,
@@ -88,7 +88,7 @@ pub enum DistributionAlgoVersion {
 ///
 /// Ideally we will never have a situation where we will have to change the
 /// fields of this struct and deal with related migration.
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct FormatV3 {
     /// Version of the format config.
     pub version: FormatMetaVersion,
@@ -103,8 +103,8 @@ pub struct FormatV3 {
     pub erasure: FormatErasureV3,
     // /// DiskInfo is an extended type which returns current
     // /// disk usage per path.
-    // #[serde(skip)]
-    // pub disk_info: Option<data_types::DeskInfo>,
+    #[serde(skip)]
+    pub disk_info: Option<DiskInfo>,
 }
 
 impl TryFrom<&[u8]> for FormatV3 {
@@ -146,7 +146,7 @@ impl FormatV3 {
             format,
             id: Uuid::new_v4(),
             erasure,
-            // disk_info: None,
+            disk_info: None,
         }
     }
 

ecstore/src/disk/local.rs

@@ -6,10 +6,11 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
 use super::{
     os, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, FileInfoVersions,
     FileReader, FileWriter, Info, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
-    UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
+    UpdateMetadataOpts, VolumeInfo, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET,
 };
 use crate::bitrot::bitrot_verify;
-use crate::cache_value::cache::{Cache, Opts};
+use crate::bucket::metadata_sys::GLOBAL_BucketMetadataSys;
+use crate::cache_value::cache::{Cache, Opts, UpdateFn};
 use crate::disk::error::{
     convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir,
     map_err_not_exists, os_err_to_file_err,
@@ -18,27 +19,35 @@ use crate::disk::os::{check_path_length, is_empty_dir};
 use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
 use crate::error::{Error, Result};
 use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
+use crate::heal::data_scanner::{has_active_rules, scan_data_folder, ScannerItem, SizeSummary};
+use crate::heal::data_scanner_metric::{ScannerMetric, ScannerMetrics};
+use crate::heal::data_usage_cache::{DataUsageCache, DataUsageEntry};
+use crate::heal::error::{ERR_IGNORE_FILE_CONTRIB, ERR_SKIP_FILE};
+use crate::heal::heal_commands::{HealScanMode, HealingTracker};
+use crate::heal::heal_ops::HEALING_TRACKER_FILENAME;
+use crate::new_object_layer_fn;
 use crate::set_disk::{
     conv_part_err_to_int, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
     CHECK_PART_VOLUME_NOT_FOUND,
 };
-use crate::store_api::BitrotAlgorithm;
+use crate::store_api::{BitrotAlgorithm, StorageAPI};
 use crate::utils::fs::{access, lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY};
 use crate::utils::os::get_info;
-use crate::utils::path::{clean, has_suffix, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
+use crate::utils::path::{clean, has_suffix, path_join, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR};
 use crate::{
     file_meta::FileMeta,
     store_api::{FileInfo, RawFileInfo},
     utils,
 };
+use common::defer;
 use path_absolutize::Absolutize;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::io::Cursor;
 use std::os::unix::fs::MetadataExt;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, SystemTime};
 use std::{
     fs::Metadata,
     path::{Path, PathBuf},
@@ -46,7 +55,7 @@ use std::{
 use time::OffsetDateTime;
 use tokio::fs::{self, File};
 use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind};
-use tokio::runtime::Runtime;
+use tokio::sync::mpsc::Sender;
 use tokio::sync::RwLock;
 use tracing::{error, info, warn};
 use uuid::Uuid;
@@ -138,10 +147,10 @@ impl LocalDisk {
             last_check: format_last_check,
         };
         let root_clone = root.clone();
-        let disk_id = Arc::new(id.map_or("".to_string(), |id| id.to_string()));
-        let update_fn = move || {
-            let rt = Runtime::new().unwrap();
-            rt.block_on(async {
+        let update_fn: UpdateFn<DiskInfo> = Box::new(move || {
+            let disk_id = id.map_or("".to_string(), |id| id.to_string());
+            let root = root_clone.clone();
+            Box::pin(async move {
                 match get_disk_info(root.clone()).await {
                     Ok((info, root)) => {
                         let disk_info = DiskInfo {
@@ -167,14 +176,14 @@ impl LocalDisk {
                     Err(err) => Err(err),
                 }
             })
-        };
+        });
 
-        let cache = Cache::new(Box::new(update_fn), Duration::from_secs(1), Opts::default());
+        let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
 
         // TODO: DIRECT suport
         // TODD: DiskInfo
         let mut disk = Self {
-            root: root_clone.clone(),
+            root: root.clone(),
             endpoint: ep.clone(),
             format_path,
             format_info: RwLock::new(format_info),
@@ -190,7 +199,7 @@ impl LocalDisk {
             // format_data: Mutex::new(format_data),
             // format_last_check: Mutex::new(format_last_check),
         };
-        let (info, _root) = get_disk_info(root_clone).await?;
+        let (info, _root) = get_disk_info(root).await?;
         disk.major = info.major;
         disk.minor = info.minor;
         disk.fstype = info.fstype;
@@ -932,7 +941,7 @@ impl DiskAPI for LocalDisk {
     }
 
     async fn check_parts(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
-        let volume_dir = self.get_bucket_path(&volume)?;
+        let volume_dir = self.get_bucket_path(volume)?;
         check_path_length(volume_dir.join(path).to_string_lossy().as_ref())?;
         let mut resp = CheckPartsResp {
             results: vec![0; fi.parts.len()],
@@ -958,22 +967,16 @@ impl DiskAPI for LocalDisk {
                     resp.results[i] = CHECK_PART_SUCCESS;
                 }
                 Err(err) => {
-                    match os_err_to_file_err(err).downcast_ref() {
-                        Some(DiskError::FileNotFound) => {
-                            if !skip_access_checks(volume) {
-                                if let Err(err) = access(&volume_dir).await {
-                                    match err.kind() {
-                                        ErrorKind::NotFound => {
-                                            resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
-                                            continue;
-                                        }
-                                        _ => {}
-                                    }
+                    if let Some(DiskError::FileNotFound) = os_err_to_file_err(err).downcast_ref() {
+                        if !skip_access_checks(volume) {
+                            if let Err(err) = access(&volume_dir).await {
+                                if err.kind() == ErrorKind::NotFound {
+                                    resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
+                                    continue;
                                 }
                             }
-                            resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
                         }
-                        _ => {}
+                        resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
                     }
                     continue;
                 }
@@ -1881,6 +1884,162 @@ impl DiskAPI for LocalDisk {
 
         Ok(info)
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        self.scanning.fetch_add(1, Ordering::SeqCst);
+        defer!(|| { self.scanning.fetch_sub(1, Ordering::SeqCst) });
+
+        // Check if the current bucket has replication configuration
+        if let Ok((rcfg, _)) = GLOBAL_BucketMetadataSys
+            .read()
+            .await
+            .get_replication_config(&cache.info.name)
+            .await
+        {
+            if has_active_rules(&rcfg, "", true) {
+                // TODO: globalBucketTargetSys
+            }
+        }
+
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::msg("errServerNotInitialized")),
+        };
+        let loc = self.get_disk_location();
+        let disks = store.get_disks(loc.pool_idx.unwrap(), loc.disk_idx.unwrap()).await?;
+        let disk = Arc::new(LocalDisk::new(&self.endpoint(), false).await?);
+        let disk_clone = disk.clone();
+        let mut cache = cache.clone();
+        cache.info.updates = Some(updates.clone());
+        let mut data_usage_info = scan_data_folder(
+            &disks,
+            disk,
+            &cache,
+            Box::new(move |item: &ScannerItem| {
+                let mut item = item.clone();
+                let disk = disk_clone.clone();
+                Box::pin(async move {
+                    if item.path.ends_with(&format!("{}{}", SLASH_SEPARATOR, STORAGE_FORMAT_FILE)) {
+                        return Err(Error::from_string(ERR_SKIP_FILE));
+                    }
+                    let stop_fn = ScannerMetrics::log(ScannerMetric::ScanObject);
+                    let mut res = HashMap::new();
+                    let done_sz = ScannerMetrics::time_size(ScannerMetric::ReadMetadata).await;
+                    let buf = match disk.read_metadata(item.path.clone()).await {
+                        Ok(buf) => buf,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    done_sz(buf.len() as u64).await;
+                    res.insert("metasize".to_string(), buf.len().to_string());
+                    item.transform_meda_dir();
+                    let meta_cache = MetaCacheEntry {
+                        name: item.object_path().to_string_lossy().to_string(),
+                        metadata: buf,
+                        ..Default::default()
+                    };
+                    let fivs = match meta_cache.file_info_versions(&item.bucket) {
+                        Ok(fivs) => fivs,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+                    let mut size_s = SizeSummary::default();
+                    let done = ScannerMetrics::time(ScannerMetric::ApplyAll);
+                    let obj_infos = match item.apply_versions_actions(&fivs.versions).await {
+                        Ok(obj_infos) => obj_infos,
+                        Err(err) => {
+                            res.insert("err".to_string(), err.to_string());
+                            stop_fn(&res).await;
+                            return Err(Error::from_string(ERR_SKIP_FILE));
+                        }
+                    };
+
+                    let versioned = false;
+                    let mut obj_deleted = false;
+                    for info in obj_infos.iter() {
+                        let done = ScannerMetrics::time(ScannerMetric::ApplyVersion);
+                        let sz: usize;
+                        (obj_deleted, sz) = item.apply_actions(info, &size_s).await;
+                        done().await;
+
+                        if obj_deleted {
+                            break;
+                        }
+
+                        let actual_sz = match info.get_actual_size() {
+                            Ok(size) => size,
+                            Err(_) => continue,
+                        };
+
+                        if info.delete_marker {
+                            size_s.delete_markers += 1;
+                        }
+
+                        if info.version_id.is_some() && sz == actual_sz {
+                            size_s.versions += 1;
+                        }
+
+                        size_s.total_size += sz;
+
+                        if info.delete_marker {
+                            continue;
+                        }
+                    }
+
+                    for frer_version in fivs.free_versions.iter() {
+                        let _obj_info =
+                            frer_version.to_object_info(&item.bucket, &item.object_path().to_string_lossy(), versioned);
+                        let done = ScannerMetrics::time(ScannerMetric::TierObjSweep);
+                        done().await;
+                    }
+
+                    // todo: global trace
+                    if obj_deleted {
+                        return Err(Error::from_string(ERR_IGNORE_FILE_CONTRIB));
+                    }
+                    done().await;
+                    Ok(size_s)
+                })
+            }),
+            scan_mode,
+        )
+        .await?;
+        data_usage_info.info.last_update = Some(SystemTime::now());
+        Ok(data_usage_info)
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        let healing_file = path_join(&[
+            self.path(),
+            PathBuf::from(RUSTFS_META_BUCKET),
+            PathBuf::from(BUCKET_META_PREFIX),
+            PathBuf::from(HEALING_TRACKER_FILENAME),
+        ]);
+        let b = match fs::read(healing_file).await {
+            Ok(b) => b,
+            Err(_) => return None,
+        };
+        if b.is_empty() {
+            return None;
+        }
+        match HealingTracker::unmarshal_msg(&b) {
+            Ok(h) => Some(h),
+            Err(_) => Some(HealingTracker::default()),
+        }
+    }
 }
 
 async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
@@ -1893,10 +2052,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> {
         if root_disk_threshold > 0 {
             disk_info.total <= root_disk_threshold
         } else {
-            match is_root_disk(&drive_path, SLASH_SEPARATOR) {
-                Ok(result) => result,
-                Err(_) => false,
-            }
+            is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default()
         }
     } else {
         false

ecstore/src/disk/mod.rs

@@ -1,7 +1,7 @@
 pub mod endpoint;
 pub mod error;
 pub mod format;
-mod local;
+pub mod local;
 pub mod os;
 pub mod remote;
 
@@ -14,9 +14,13 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 const STORAGE_FORMAT_FILE: &str = "xl.meta";
 
 use crate::{
-    erasure::{ReadAt, Writer},
+    erasure::Writer,
     error::{Error, Result},
     file_meta::{merge_file_meta_versions, FileMeta, FileMetaShallowVersion},
+    heal::{
+        data_usage_cache::{DataUsageCache, DataUsageEntry},
+        heal_commands::{HealScanMode, HealingTracker},
+    },
     store_api::{FileInfo, RawFileInfo},
 };
 use endpoint::Endpoint;
@@ -35,7 +39,6 @@ use std::{
     io::{Cursor, SeekFrom},
     path::PathBuf,
     sync::Arc,
-    usize,
 };
 use time::OffsetDateTime;
 use tokio::{
@@ -53,8 +56,8 @@ pub type DiskStore = Arc<Disk>;
 
 #[derive(Debug)]
 pub enum Disk {
-    Local(LocalDisk),
-    Remote(RemoteDisk),
+    Local(Box<LocalDisk>),
+    Remote(Box<RemoteDisk>),
 }
 
 #[async_trait::async_trait]
@@ -338,15 +341,34 @@ impl DiskAPI for Disk {
             Disk::Remote(remote_disk) => remote_disk.disk_info(opts).await,
         }
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        match self {
+            Disk::Local(local_disk) => local_disk.ns_scanner(cache, updates, scan_mode).await,
+            Disk::Remote(remote_disk) => remote_disk.ns_scanner(cache, updates, scan_mode).await,
+        }
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        match self {
+            Disk::Local(local_disk) => local_disk.healing().await,
+            Disk::Remote(remote_disk) => remote_disk.healing().await,
+        }
+    }
 }
 
 pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result<DiskStore> {
     if ep.is_local {
         let s = local::LocalDisk::new(ep, opt.cleanup).await?;
-        Ok(Arc::new(Disk::Local(s)))
+        Ok(Arc::new(Disk::Local(Box::new(s))))
     } else {
         let remote_disk = remote::RemoteDisk::new(ep, opt).await?;
-        Ok(Arc::new(Disk::Remote(remote_disk)))
+        Ok(Arc::new(Disk::Remote(Box::new(remote_disk))))
     }
 }
 
@@ -436,6 +458,13 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn write_all(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()>;
     async fn read_all(&self, volume: &str, path: &str) -> Result<Vec<u8>>;
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo>;
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache>;
+    async fn healing(&self) -> Option<HealingTracker>;
 }
 
 #[derive(Debug, Default, Serialize, Deserialize)]
@@ -467,7 +496,7 @@ pub struct DiskInfoOptions {
     pub noop: bool,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DiskInfo {
     pub total: u64,
     pub free: u64,
@@ -489,7 +518,7 @@ pub struct DiskInfo {
     pub error: String,
 }
 
-#[derive(Clone, Debug, Default, Serialize, Deserialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct DiskMetrics {
     api_calls: HashMap<String, u64>,
     total_waiting: u32,
@@ -653,7 +682,7 @@ impl MetaCacheEntry {
         let mut fm = FileMeta::new();
         fm.unmarshal_msg(&self.metadata)?;
 
-        Ok(fm.into_file_info_versions(bucket, self.name.as_str(), false)?)
+        fm.into_file_info_versions(bucket, self.name.as_str(), false)
     }
 
     pub fn matches(&self, other: &MetaCacheEntry, strict: bool) -> Result<(Option<MetaCacheEntry>, bool)> {
@@ -812,7 +841,7 @@ impl MetaCacheEntries {
         selected = Some(MetaCacheEntry {
             name: selected.as_ref().unwrap().name.clone(),
             cached: Some(FileMeta {
-                meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver.clone(),
+                meta_ver: selected.as_ref().unwrap().cached.as_ref().unwrap().meta_ver,
                 ..Default::default()
             }),
             _reusable: true,
@@ -835,7 +864,7 @@ impl MetaCacheEntries {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Clone, Debug, Default)]
 pub struct DiskOption {
     pub cleanup: bool,
     pub health_check: bool,
@@ -1281,10 +1310,10 @@ impl Reader for RemoteFileReader {
             Err(Error::from_string(error_info))
         }
     }
-    async fn seek(&mut self, offset: usize) -> Result<()> {
+    async fn seek(&mut self, _offset: usize) -> Result<()> {
         unimplemented!()
     }
-    async fn read_exact(&mut self, buf: &mut [u8]) -> Result<usize> {
+    async fn read_exact(&mut self, _buf: &mut [u8]) -> Result<usize> {
         unimplemented!()
     }
 }

ecstore/src/disk/os.rs

@@ -56,7 +56,7 @@ pub fn is_root_disk(disk_path: &str, root_disk: &str) -> Result<bool> {
         return Ok(false);
     }
 
-    Ok(same_disk(disk_path, root_disk)?)
+    same_disk(disk_path, root_disk)
 }
 
 pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> Result<()> {

ecstore/src/disk/remote.rs

@@ -5,11 +5,13 @@ use protos::{
     node_service_time_out_client,
     proto_gen::node_service::{
         CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
-        DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest,
-        ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest,
-        UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
+        DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, NsScannerRequest,
+        ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst,
+        StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest,
     },
 };
+use tokio::sync::mpsc::{self, Sender};
+use tokio_stream::{wrappers::ReceiverStream, StreamExt};
 use tonic::Request;
 use tracing::info;
 use uuid::Uuid;
@@ -22,6 +24,10 @@ use super::{
 use crate::{
     disk::error::DiskError,
     error::{Error, Result},
+    heal::{
+        data_usage_cache::{DataUsageCache, DataUsageEntry},
+        heal_commands::{HealScanMode, HealingTracker},
+    },
     store_api::{FileInfo, RawFileInfo},
 };
 use protos::proto_gen::node_service::RenamePartRequst;
@@ -747,4 +753,48 @@ impl DiskAPI for RemoteDisk {
 
         Ok(disk_info)
     }
+
+    async fn ns_scanner(
+        &self,
+        cache: &DataUsageCache,
+        updates: Sender<DataUsageEntry>,
+        scan_mode: HealScanMode,
+    ) -> Result<DataUsageCache> {
+        info!("ns_scanner");
+        let cache = serde_json::to_string(cache)?;
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?;
+
+        let (tx, rx) = mpsc::channel(10);
+        let in_stream = ReceiverStream::new(rx);
+        let mut response = client.ns_scanner(in_stream).await?.into_inner();
+        let request = NsScannerRequest {
+            disk: self.root.to_string_lossy().to_string(),
+            cache,
+            scan_mode: scan_mode as u64,
+        };
+        tx.send(request).await?;
+
+        loop {
+            match response.next().await {
+                Some(Ok(resp)) => {
+                    if !resp.update.is_empty() {
+                        let data_usage_cache = serde_json::from_str::<DataUsageEntry>(&resp.update)?;
+                        let _ = updates.send(data_usage_cache).await;
+                    } else if !resp.data_usage_cache.is_empty() {
+                        let data_usage_cache = serde_json::from_str::<DataUsageCache>(&resp.data_usage_cache)?;
+                        return Ok(data_usage_cache);
+                    } else {
+                        return Err(Error::from_string("scan was interrupted"));
+                    }
+                }
+                _ => return Err(Error::from_string("scan was interrupted")),
+            }
+        }
+    }
+
+    async fn healing(&self) -> Option<HealingTracker> {
+        None
+    }
 }

ecstore/src/endpoints.rs

@@ -109,6 +109,24 @@ impl Endpoints {
     pub fn into_inner(self) -> Vec<Endpoint> {
         self.0
     }
+
+    pub fn into_ref(&self) -> &Vec<Endpoint> {
+        &self.0
+    }
+
+    // GetString - returns endpoint string of i-th endpoint (0-based),
+    // and empty string for invalid indexes.
+    pub fn get_string(&self, i: usize) -> String {
+        if i >= self.0.len() {
+            return "".to_string();
+        }
+
+        self.0[i].to_string()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
 }
 
 #[derive(Debug)]

ecstore/src/erasure.rs

@@ -19,6 +19,7 @@ use uuid::Uuid;
 // use crate::chunk_stream::ChunkedStream;
 use crate::disk::error::DiskError;
 
+#[derive(Default)]
 pub struct Erasure {
     data_shards: usize,
     parity_shards: usize,
@@ -417,6 +418,54 @@ impl Erasure {
 
         till_offset
     }
+
+    pub async fn heal(
+        &self,
+        writers: &mut [Option<BitrotWriter>],
+        readers: Vec<Option<BitrotReader>>,
+        total_length: usize,
+        _prefer: &[bool],
+    ) -> Result<()> {
+        if writers.len() != self.parity_shards + self.data_shards {
+            return Err(Error::from_string("invalid argument"));
+        }
+        let mut reader = ShardReader::new(readers, self, 0, total_length);
+
+        let start_block = 0;
+        let mut end_block = total_length / self.block_size;
+        if total_length % self.block_size != 0 {
+            end_block += 1;
+        }
+
+        let mut errs = Vec::new();
+        for _ in start_block..=end_block {
+            let mut bufs = reader.read().await?;
+
+            if self.parity_shards > 0 {
+                self.encoder.as_ref().unwrap().reconstruct(&mut bufs)?;
+            }
+
+            let shards = bufs.into_iter().flatten().collect::<Vec<_>>();
+            if shards.len() != self.parity_shards + self.data_shards {
+                return Err(Error::from_string("can not reconstruct data"));
+            }
+
+            for (i, w) in writers.iter_mut().enumerate() {
+                if w.is_none() {
+                    continue;
+                }
+                match w.as_mut().unwrap().write(shards[i].as_ref()).await {
+                    Ok(_) => {}
+                    Err(e) => errs.push(e),
+                }
+            }
+        }
+        if !errs.is_empty() {
+            return Err(errs[0].clone());
+        }
+
+        Ok(())
+    }
 }
 
 #[async_trait::async_trait]

ecstore/src/global.rs

@@ -6,6 +6,7 @@ use uuid::Uuid;
 use crate::{
     disk::DiskStore,
     endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
+    heal::{background_heal_ops::HealRoutine, heal_ops::AllHealState},
     store::ECStore,
 };
 
@@ -24,6 +25,8 @@ lazy_static! {
     pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc<RwLock<TypeLocalDiskSetDrives>> = Arc::new(RwLock::new(Vec::new()));
     pub static ref GLOBAL_Endpoints: RwLock<EndpointServerPools> = RwLock::new(EndpointServerPools(Vec::new()));
     pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
+    pub static ref GLOBAL_BackgroundHealRoutine: Arc<RwLock<HealRoutine>> = HealRoutine::new();
+    pub static ref GLOBAL_BackgroundHealState: Arc<RwLock<AllHealState>> = AllHealState::new(false);
     static ref globalDeploymentIDPtr: RwLock<Uuid> = RwLock::new(Uuid::nil());
 }
 
@@ -34,7 +37,7 @@ pub async fn set_global_deployment_id(id: Uuid) {
 pub async fn get_global_deployment_id() -> Uuid {
     let id_ptr = globalDeploymentIDPtr.read().await;
 
-    id_ptr.clone()
+    *id_ptr
 }
 
 pub async fn set_global_endpoints(eps: Vec<PoolEndpoints>) {

ecstore/src/heal/background_heal_ops.rs

@@ -1,76 +1,424 @@
-use std::sync::Arc;
-
+use s3s::{S3Error, S3ErrorCode};
+use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
 use tokio::{
-    select,
     sync::{
-        broadcast::Receiver as B_Receiver,
         mpsc::{self, Receiver, Sender},
+        RwLock,
     },
+    time::interval,
 };
-
-use crate::{error::Error, heal::heal_ops::NOP_HEAL, utils::path::SLASH_SEPARATOR};
+use tracing::{error, info};
+use uuid::Uuid;
 
 use super::{
     heal_commands::{HealOpts, HealResultItem},
-    heal_ops::HealSequence,
+    heal_ops::{new_bg_heal_sequence, HealSequence},
+};
+use crate::heal::error::ERR_RETRY_HEALING;
+use crate::{
+    config::RUSTFS_CONFIG_PREFIX,
+    disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
+    error::{Error, Result},
+    global::{GLOBAL_BackgroundHealRoutine, GLOBAL_BackgroundHealState, GLOBAL_LOCAL_DISK_MAP},
+    heal::{
+        data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT},
+        data_usage_cache::DataUsageCache,
+        heal_commands::{init_healing_tracker, load_healing_tracker},
+        heal_ops::NOP_HEAL,
+    },
+    new_object_layer_fn,
+    store::get_disk_via_endpoint,
+    store_api::{BucketInfo, BucketOptions, StorageAPI},
+    utils::path::{path_join, SLASH_SEPARATOR},
 };
 
-#[derive(Clone, Debug)]
+pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10);
+
+pub async fn init_auto_heal() {
+    init_background_healing().await;
+    if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
+        if v == "on" {
+            GLOBAL_BackgroundHealState
+                .write()
+                .await
+                .push_heal_local_disks(&get_local_disks_to_heal().await)
+                .await;
+            tokio::spawn(async {
+                monitor_local_disks_and_heal().await;
+            });
+        }
+    }
+}
+
+async fn init_background_healing() {
+    let bg_seq = Arc::new(RwLock::new(new_bg_heal_sequence()));
+    for _ in 0..GLOBAL_BackgroundHealRoutine.read().await.workers {
+        let bg_seq_clone = bg_seq.clone();
+        tokio::spawn(async {
+            GLOBAL_BackgroundHealRoutine.write().await.add_worker(bg_seq_clone).await;
+        });
+    }
+    let _ = GLOBAL_BackgroundHealState
+        .write()
+        .await
+        .launch_new_heal_sequence(bg_seq)
+        .await;
+}
+
+async fn get_local_disks_to_heal() -> Vec<Endpoint> {
+    let mut disks_to_heal = Vec::new();
+    for (_, disk) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
+        if let Some(disk) = disk {
+            if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+                if let Some(DiskError::UnformattedDisk) = err.downcast_ref() {
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+            let h = disk.healing().await;
+            if let Some(h) = h {
+                if !h.finished {
+                    disks_to_heal.push(disk.endpoint());
+                }
+            }
+        }
+    }
+
+    // todo
+    // if disks_to_heal.len() == GLOBAL_Endpoints.read().await.n {
+
+    // }
+    disks_to_heal
+}
+
+async fn monitor_local_disks_and_heal() {
+    let mut interval = interval(DEFAULT_MONITOR_NEW_DISK_INTERVAL);
+
+    loop {
+        interval.tick().await;
+        let heal_disks = GLOBAL_BackgroundHealState.read().await.get_heal_local_disk_endpoints().await;
+        if heal_disks.is_empty() {
+            interval.reset();
+            continue;
+        }
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = lock.as_ref().expect("errServerNotInitialized");
+        if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
+            if let Some(DiskError::NoHealRequired) = err.downcast_ref() {
+            } else {
+                info!("heal format err: {}", err.to_string());
+                interval.reset();
+                continue;
+            }
+        }
+
+        for disk in heal_disks.into_ref().iter() {
+            let disk_clone = disk.clone();
+            tokio::spawn(async move {
+                GLOBAL_BackgroundHealState
+                    .write()
+                    .await
+                    .set_disk_healing_status(disk_clone.clone(), true)
+                    .await;
+                if heal_fresh_disk(&disk_clone).await.is_err() {
+                    GLOBAL_BackgroundHealState
+                        .write()
+                        .await
+                        .set_disk_healing_status(disk_clone.clone(), false)
+                        .await;
+                    return;
+                }
+                GLOBAL_BackgroundHealState
+                    .write()
+                    .await
+                    .pop_heal_local_disks(&[disk_clone])
+                    .await;
+            });
+        }
+        interval.reset();
+    }
+}
+
+async fn heal_fresh_disk(endpoint: &Endpoint) -> Result<()> {
+    let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.disk_idx as usize);
+    let disk = match get_disk_via_endpoint(endpoint).await {
+        Some(disk) => disk,
+        None => {
+            return Err(Error::from_string(format!(
+                "Unexpected error disk must be initialized by now after formatting: {}",
+                endpoint
+            )))
+        }
+    };
+
+    if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
+        match err.downcast_ref() {
+            Some(DiskError::DriveIsRoot) => {
+                return Ok(());
+            }
+            Some(DiskError::UnformattedDisk) => {}
+            _ => {
+                return Err(err);
+            }
+        }
+    }
+
+    let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
+        Ok(tracker) => tracker,
+        Err(err) => {
+            match err.downcast_ref() {
+                Some(DiskError::FileNotFound) => {
+                    return Ok(());
+                }
+                _ => {
+                    info!(
+                        "Unable to load healing tracker on '{}': {}, re-initializing..",
+                        disk.to_string(),
+                        err.to_string()
+                    );
+                }
+            }
+            init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string()).await?
+        }
+    };
+
+    info!(
+        "Healing drive '{}' - 'mc admin heal alias/ --verbose' to check the current status.",
+        endpoint.to_string()
+    );
+
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => return Err(Error::msg("errServerNotInitialized")),
+    };
+    let mut buckets = store.list_bucket(&BucketOptions::default()).await?;
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)])
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    });
+    buckets.push(BucketInfo {
+        name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)])
+            .to_string_lossy()
+            .to_string(),
+        ..Default::default()
+    });
+
+    buckets.sort_by(|a, b| {
+        let a_has_prefix = a.name.starts_with(RUSTFS_META_BUCKET);
+        let b_has_prefix = b.name.starts_with(RUSTFS_META_BUCKET);
+
+        match (a_has_prefix, b_has_prefix) {
+            (true, false) => Ordering::Less,
+            (false, true) => Ordering::Greater,
+            _ => b.created.cmp(&a.created),
+        }
+    });
+
+    if let Ok(cache) = DataUsageCache::load(&store.pools[pool_idx].disk_set[set_idx], DATA_USAGE_CACHE_NAME).await {
+        let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::new());
+        tracker.objects_total_count = data_usage_info.objects_total_count;
+        tracker.objects_total_size = data_usage_info.objects_total_size;
+    };
+
+    tracker.set_queue_buckets(&buckets).await;
+    tracker.save().await?;
+
+    let tracker = Arc::new(RwLock::new(tracker));
+    let qb = tracker.read().await.queue_buckets.clone();
+    store.pools[pool_idx].disk_set[set_idx]
+        .clone()
+        .heal_erasure_set(&qb, tracker.clone())
+        .await?;
+    let mut tracker_w = tracker.write().await;
+    if tracker_w.items_failed > 0 && tracker_w.retry_attempts < 4 {
+        tracker_w.retry_attempts += 1;
+        tracker_w.reset_healing().await;
+        if let Err(err) = tracker_w.update().await {
+            info!("update tracker failed: {}", err.to_string());
+        }
+        return Err(Error::from_string(ERR_RETRY_HEALING));
+    }
+
+    if tracker_w.items_failed > 0 {
+        info!(
+            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}, failed: {}).",
+            disk.to_string(),
+            tracker_w.retry_attempts,
+            tracker_w.items_healed,
+            tracker_w.item_skipped,
+            tracker_w.items_failed
+        );
+    } else if tracker_w.retry_attempts > 0 {
+        info!(
+            "Healing of drive '{}' is incomplete, retried {} times (healed: {}, skipped: {}).",
+            disk.to_string(),
+            tracker_w.retry_attempts,
+            tracker_w.items_healed,
+            tracker_w.item_skipped
+        );
+    } else {
+        info!(
+            "Healing of drive '{}' is finished (healed: {}, skipped: {}).",
+            disk.to_string(),
+            tracker_w.items_healed,
+            tracker_w.item_skipped
+        );
+    }
+
+    if tracker_w.heal_id.is_empty() {
+        if let Err(err) = tracker_w.delete().await {
+            error!("delete tracker failed: {}", err.to_string());
+        }
+    }
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
+    };
+    let disks = store.get_disks(pool_idx, set_idx).await?;
+    for disk in disks.into_iter() {
+        if disk.is_none() {
+            continue;
+        }
+        let mut tracker = match load_healing_tracker(&disk).await {
+            Ok(tracker) => tracker,
+            Err(err) => {
+                match err.downcast_ref() {
+                    Some(DiskError::FileNotFound) => {}
+                    _ => {
+                        info!("Unable to load healing tracker on '{:?}': {}, re-initializing..", disk, err.to_string());
+                    }
+                }
+                continue;
+            }
+        };
+        if tracker.heal_id == tracker_w.heal_id {
+            tracker.finished = true;
+            tracker.update().await?;
+        }
+    }
+    Ok(())
+}
+
+#[derive(Debug)]
 pub struct HealTask {
     pub bucket: String,
     pub object: String,
     pub version_id: String,
     pub opts: HealOpts,
-    pub resp_tx: Arc<Sender<HealResult>>,
-    pub resp_rx: Arc<Receiver<HealResult>>,
+    pub resp_tx: Option<Sender<HealResult>>,
+    pub resp_rx: Option<Receiver<HealResult>>,
 }
 
 impl HealTask {
     pub fn new(bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Self {
-        let (tx, rx) = mpsc::channel(10);
         Self {
             bucket: bucket.to_string(),
             object: object.to_string(),
             version_id: version_id.to_string(),
-            opts: opts.clone(),
-            resp_tx: tx.into(),
-            resp_rx: rx.into(),
+            opts: *opts,
+            resp_tx: None,
+            resp_rx: None,
         }
     }
 }
 
 pub struct HealResult {
     pub result: HealResultItem,
-    err: Error,
+    pub err: Option<Error>,
 }
 
 pub struct HealRoutine {
-    tasks_tx: Sender<HealTask>,
+    pub tasks_tx: Sender<HealTask>,
     tasks_rx: Receiver<HealTask>,
     workers: usize,
 }
 
 impl HealRoutine {
-    pub async fn add_worker(&mut self, mut ctx: B_Receiver<bool>, bgseq: &HealSequence) {
+    pub fn new() -> Arc<RwLock<Self>> {
+        let mut workers = num_cpus::get() / 2;
+        if let Ok(env_heal_workers) = env::var("_RUSTFS_HEAL_WORKERS") {
+            if let Ok(num_healers) = env_heal_workers.parse::<usize>() {
+                workers = num_healers;
+            }
+        }
+
+        if workers == 0 {
+            workers = 4;
+        }
+
+        let (tx, rx) = mpsc::channel(100);
+        Arc::new(RwLock::new(Self {
+            tasks_tx: tx,
+            tasks_rx: rx,
+            workers,
+        }))
+    }
+
+    pub async fn add_worker(&mut self, bgseq: Arc<RwLock<HealSequence>>) {
         loop {
-            select! {
-                task = self.tasks_rx.recv() => {
-                    let mut res = HealResultItem::default();
-                    let mut err: Error;
-                    match task {
-                        Some(task) => {
-                            if task.bucket == NOP_HEAL {
-                                err = Error::from_string("skip file");
-                            } else if task.bucket == SLASH_SEPARATOR {
-                                (res, err) = heal_disk_format(task.opts).await;
+            let mut d_res = HealResultItem::default();
+            let d_err: Option<Error>;
+            match self.tasks_rx.recv().await {
+                Some(task) => {
+                    if task.bucket == NOP_HEAL {
+                        d_err = Some(Error::from_string("skip file"));
+                    } else if task.bucket == SLASH_SEPARATOR {
+                        match heal_disk_format(task.opts).await {
+                            Ok((res, err)) => {
+                                d_res = res;
+                                d_err = err;
                             }
-                        },
-                        None => return,
+                            Err(err) => d_err = Some(err),
+                        }
+                    } else {
+                        let layer = new_object_layer_fn();
+                        let lock = layer.read().await;
+                        let store = lock.as_ref().expect("Not init");
+                        if task.object.is_empty() {
+                            match store.heal_bucket(&task.bucket, &task.opts).await {
+                                Ok(res) => {
+                                    d_res = res;
+                                    d_err = None;
+                                }
+                                Err(err) => d_err = Some(err),
+                            }
+                        } else {
+                            match store
+                                .heal_object(&task.bucket, &task.object, &task.version_id, &task.opts)
+                                .await
+                            {
+                                Ok((res, err)) => {
+                                    d_res = res;
+                                    d_err = err;
+                                }
+                                Err(err) => d_err = Some(err),
+                            }
+                        }
+                    }
+                    if let Some(resp_tx) = task.resp_tx {
+                        let _ = resp_tx
+                            .send(HealResult {
+                                result: d_res,
+                                err: d_err,
+                            })
+                            .await;
+                    } else {
+                        // when respCh is not set caller is not waiting but we
+                        // update the relevant metrics for them
+                        if d_err.is_none() {
+                            bgseq.write().await.count_healed(d_res.heal_item_type);
+                        } else {
+                            bgseq.write().await.count_failed(d_res.heal_item_type);
+                        }
                     }
                 }
-                _ = ctx.recv() => {
-                    return;
-                }
+                None => return,
             }
         }
     }
@@ -80,6 +428,15 @@ impl HealRoutine {
 
 // }
 
-async fn heal_disk_format(opts: HealOpts) -> (HealResultItem, Error) {
-    todo!()
+async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Error>)> {
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = lock.as_ref().expect("Not init");
+    let (res, err) = store.heal_format(opts.dry_run).await?;
+    // return any error, ignore error returned when disks have
+    // already healed.
+    if err.is_some() {
+        return Ok((HealResultItem::default(), err));
+    }
+    Ok((res, err))
 }

ecstore/src/heal/data_scanner_metric.rs

@@ -0,0 +1,223 @@
+use common::last_minute::{AccElem, LastMinuteLatency};
+use lazy_static::lazy_static;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::atomic::AtomicU64;
+use std::sync::Once;
+use std::time::{Duration, UNIX_EPOCH};
+use std::{
+    collections::HashMap,
+    sync::{
+        atomic::{AtomicU32, Ordering},
+        Arc,
+    },
+    time::SystemTime,
+};
+use tokio::sync::RwLock;
+
+use super::data_scanner::{CurrentScannerCycle, UpdateCurrentPathFn};
+
+lazy_static! {
+    pub static ref globalScannerMetrics: Arc<RwLock<ScannerMetrics>> = Arc::new(RwLock::new(ScannerMetrics::new()));
+}
+
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
+pub enum ScannerMetric {
+    // START Realtime metrics, that only to records
+    // last minute latencies and total operation count.
+    ReadMetadata = 0,
+    CheckMissing,
+    SaveUsage,
+    ApplyAll,
+    ApplyVersion,
+    TierObjSweep,
+    HealCheck,
+    Ilm,
+    CheckReplication,
+    Yield,
+    CleanAbandoned,
+    ApplyNonCurrent,
+    HealAbandonedVersion,
+
+    // START Trace metrics:
+    StartTrace,
+    ScanObject, // Scan object. All operations included.
+    HealAbandonedObject,
+
+    // END realtime metrics:
+    LastRealtime,
+
+    // Trace only metrics:
+    ScanFolder,      // Scan a folder on disk, recursively.
+    ScanCycle,       // Full cycle, cluster global.
+    ScanBucketDrive, // Single bucket on one drive.
+    CompactFolder,   // Folder compacted.
+
+    // Must be last:
+    Last,
+}
+
+static INIT: Once = Once::new();
+
+#[derive(Default)]
+pub struct LockedLastMinuteLatency {
+    cached_sec: AtomicU64,
+    cached: AccElem,
+    mu: RwLock<bool>,
+    latency: LastMinuteLatency,
+}
+
+impl Clone for LockedLastMinuteLatency {
+    fn clone(&self) -> Self {
+        Self {
+            cached_sec: AtomicU64::new(0),
+            cached: self.cached.clone(),
+            mu: RwLock::new(true),
+            latency: self.latency.clone(),
+        }
+    }
+}
+
+impl LockedLastMinuteLatency {
+    pub async fn add(&mut self, value: &Duration) {
+        self.add_size(value, 0).await;
+    }
+
+    pub async fn add_size(&mut self, value: &Duration, sz: u64) {
+        let t = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .expect("Time went backwards")
+            .as_secs();
+        INIT.call_once(|| {
+            self.cached = AccElem::default();
+            self.cached_sec.store(t, Ordering::SeqCst);
+        });
+        let last_t = self.cached_sec.load(Ordering::SeqCst);
+        if last_t != t
+            && self
+                .cached_sec
+                .compare_exchange(last_t, t, Ordering::SeqCst, Ordering::SeqCst)
+                .is_ok()
+        {
+            let old = self.cached.clone();
+            self.cached = AccElem::default();
+            let a = AccElem {
+                size: old.size,
+                total: old.total,
+                n: old.n,
+            };
+            let _ = self.mu.write().await;
+            self.latency.add_all(t - 1, &a);
+        }
+        self.cached.n += 1;
+        self.cached.total += value.as_secs();
+        self.cached.size += sz;
+    }
+
+    pub async fn total(&mut self) -> AccElem {
+        let _ = self.mu.read().await;
+        self.latency.get_total()
+    }
+}
+
+pub type LogFn = Arc<dyn Fn(&HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TimeSizeFn = Arc<dyn Fn(u64) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+
+pub struct ScannerMetrics {
+    operations: Vec<AtomicU32>,
+    latency: Vec<LockedLastMinuteLatency>,
+    cycle_info: RwLock<Option<CurrentScannerCycle>>,
+    current_paths: HashMap<String, String>,
+}
+
+impl Default for ScannerMetrics {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ScannerMetrics {
+    pub fn new() -> Self {
+        Self {
+            operations: (0..ScannerMetric::Last as usize).map(|_| AtomicU32::new(0)).collect(),
+            latency: vec![LockedLastMinuteLatency::default(); ScannerMetric::LastRealtime as usize],
+            cycle_info: RwLock::new(None),
+            current_paths: HashMap::new(),
+        }
+    }
+
+    pub async fn set_cycle(&mut self, c: Option<CurrentScannerCycle>) {
+        *self.cycle_info.write().await = c;
+    }
+
+    pub fn log(s: ScannerMetric) -> LogFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move |_custom: &HashMap<String, String>| {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add(&duration).await;
+                }
+            })
+        })
+    }
+
+    pub async fn time_size(s: ScannerMetric) -> TimeSizeFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move |sz: u64| {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add_size(&duration, sz).await;
+                }
+            })
+        })
+    }
+
+    pub fn time(s: ScannerMetric) -> TimeFn {
+        let start = SystemTime::now();
+        let s_clone = s as usize;
+        Arc::new(move || {
+            Box::pin(async move {
+                let duration = SystemTime::now().duration_since(start).unwrap_or(Duration::from_secs(0));
+                let mut sm_w = globalScannerMetrics.write().await;
+                sm_w.operations[s_clone].fetch_add(1, Ordering::SeqCst);
+                if s_clone < ScannerMetric::LastRealtime as usize {
+                    sm_w.latency[s_clone].add(&duration).await;
+                }
+            })
+        })
+    }
+}
+
+pub type CloseDiskFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
+pub fn current_path_updater(disk: &str, _initial: &str) -> (UpdateCurrentPathFn, CloseDiskFn) {
+    let disk_1 = disk.to_string();
+    let disk_2 = disk.to_string();
+    (
+        Arc::new(move |path: &str| {
+            let disk_inner = disk_1.clone();
+            let path = path.to_string();
+            Box::pin(async move {
+                globalScannerMetrics
+                    .write()
+                    .await
+                    .current_paths
+                    .insert(disk_inner, path.to_string());
+            })
+        }),
+        Arc::new(move || {
+            let disk_inner = disk_2.clone();
+            Box::pin(async move {
+                globalScannerMetrics.write().await.current_paths.remove(&disk_inner);
+            })
+        }),
+    )
+}

ecstore/src/heal/data_usage.rs

@@ -0,0 +1,139 @@
+use std::{collections::HashMap, time::SystemTime};
+
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use tokio::sync::mpsc::Receiver;
+use tracing::info;
+
+use crate::{
+    config::common::save_config,
+    disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
+    new_object_layer_fn,
+    utils::path::SLASH_SEPARATOR,
+};
+
+pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
+const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
+const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
+pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
+lazy_static! {
+    pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, BUCKET_META_PREFIX);
+    pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_OBJ_NAME);
+    pub static ref DATA_USAGE_BLOOM_NAME_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, DATA_USAGE_BLOOM_NAME);
+    pub static ref BACKGROUND_HEAL_INFO_PATH: String =
+        format!("{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, ".background-heal.json");
+}
+
+// BucketTargetUsageInfo - bucket target usage info provides
+// - replicated size for all objects sent to this target
+// - replica size for all objects received from this target
+// - replication pending size for all objects pending replication to this target
+// - replication failed size for all objects failed replication to this target
+// - replica pending count
+// - replica failed count
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketTargetUsageInfo {
+    pub replication_pending_size: u64,
+    pub replication_failed_size: u64,
+    pub replicated_size: u64,
+    pub replica_size: u64,
+    pub replication_pending_count: u64,
+    pub replication_failed_count: u64,
+    pub replicated_count: u64,
+}
+
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BucketUsageInfo {
+    pub size: u64,
+    // Following five fields suffixed with V1 are here for backward compatibility
+    // Total Size for objects that have not yet been replicated
+    pub replication_pending_size_v1: u64,
+    // Total size for objects that have witness one or more failures and will be retried
+    pub replication_failed_size_v1: u64,
+    // Total size for objects that have been replicated to destination
+    pub replicated_size_v1: u64,
+    // Total number of objects pending replication
+    pub replication_pending_count_v1: u64,
+    // Total number of objects that failed replication
+    pub replication_failed_count_v1: u64,
+
+    pub objects_count: u64,
+    pub object_size_histogram: HashMap<String, u64>,
+    pub object_versions_histogram: HashMap<String, u64>,
+    pub versions_count: u64,
+    pub delete_markers_count: u64,
+    pub replica_size: u64,
+    pub replica_count: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct DataUsageInfo {
+    pub total_capacity: u64,
+    pub total_used_capacity: u64,
+    pub total_free_capacity: u64,
+
+    // LastUpdate is the timestamp of when the data usage info was last updated.
+    // This does not indicate a full scan.
+    pub last_update: Option<SystemTime>,
+
+    // Objects total count across all buckets
+    pub objects_total_count: u64,
+    // Versions total count across all buckets
+    pub versions_total_count: u64,
+    // Delete markers total count across all buckets
+    pub delete_markers_total_count: u64,
+    // Objects total size across all buckets
+    pub objects_total_size: u64,
+    pub replication_info: HashMap<String, BucketTargetUsageInfo>,
+
+    // Total number of buckets in this cluster
+    pub buckets_count: u64,
+    // Buckets usage info provides following information across all buckets
+    // - total size of the bucket
+    // - total objects in a bucket
+    // - object size histogram per bucket
+    pub buckets_usage: HashMap<String, BucketUsageInfo>,
+    // Deprecated kept here for backward compatibility reasons.
+    pub bucket_sizes: HashMap<String, u64>,
+    // Todo: TierStats
+    // TierStats contains per-tier stats of all configured remote tiers
+}
+
+pub async fn store_data_usage_in_backend(mut rx: Receiver<DataUsageInfo>) {
+    let layer = new_object_layer_fn();
+    let lock = layer.read().await;
+    let store = match lock.as_ref() {
+        Some(s) => s,
+        None => {
+            info!("errServerNotInitialized");
+            return;
+        }
+    };
+    let mut attempts = 1;
+    loop {
+        match rx.recv().await {
+            Some(data_usage_info) => {
+                if let Ok(data) = serde_json::to_vec(&data_usage_info) {
+                    if attempts > 10 {
+                        let _ = save_config(store, &format!("{}{}", *DATA_USAGE_OBJ_NAME_PATH, ".bkp"), &data).await;
+                        attempts += 1;
+                    }
+                    let _ = save_config(store, &DATA_USAGE_OBJ_NAME_PATH, &data).await;
+                    attempts += 1;
+                } else {
+                    continue;
+                }
+            }
+            None => {
+                return;
+            }
+        }
+    }
+}

ecstore/src/heal/data_usage_cache.rs

@@ -0,0 +1,860 @@
+use crate::config::common::save_config;
+use crate::disk::error::DiskError;
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::error::{Error, Result};
+use crate::new_object_layer_fn;
+use crate::set_disk::SetDisks;
+use crate::store_api::{BucketInfo, HTTPRangeSpec, ObjectIO, ObjectOptions};
+use bytesize::ByteSize;
+use http::HeaderMap;
+use path_clean::PathClean;
+use rand::Rng;
+use rmp_serde::Serializer;
+use s3s::dto::ReplicationConfiguration;
+use s3s::{S3Error, S3ErrorCode};
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::path::Path;
+use std::time::{Duration, SystemTime};
+use tokio::sync::mpsc::Sender;
+use tokio::time::sleep;
+
+use super::data_scanner::{SizeSummary, DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS};
+use super::data_usage::{BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, DATA_USAGE_ROOT};
+
+// DATA_USAGE_BUCKET_LEN must be length of ObjectsHistogramIntervals
+pub const DATA_USAGE_BUCKET_LEN: usize = 11;
+pub const DATA_USAGE_VERSION_LEN: usize = 7;
+
+pub type DataUsageHashMap = HashSet<String>;
+
+struct ObjectHistogramInterval {
+    name: &'static str,
+    start: u64,
+    end: u64,
+}
+
+const OBJECTS_HISTOGRAM_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_BUCKET_LEN] = [
+    ObjectHistogramInterval {
+        name: "LESS_THAN_1024_B",
+        start: 0,
+        end: ByteSize::kib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024_B_AND_64_KB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::kib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_KB_AND_256_KB",
+        start: ByteSize::kib(64).as_u64(),
+        end: ByteSize::kib(256).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_256_KB_AND_512_KB",
+        start: ByteSize::kib(256).as_u64(),
+        end: ByteSize::kib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_512_KB_AND_1_MB",
+        start: ByteSize::kib(512).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1024B_AND_1_MB",
+        start: ByteSize::kib(1).as_u64(),
+        end: ByteSize::mib(1).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1_MB_AND_10_MB",
+        start: ByteSize::mib(1).as_u64(),
+        end: ByteSize::mib(10).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_MB_AND_64_MB",
+        start: ByteSize::mib(10).as_u64(),
+        end: ByteSize::mib(64).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_64_MB_AND_128_MB",
+        start: ByteSize::mib(64).as_u64(),
+        end: ByteSize::mib(128).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_128_MB_AND_512_MB",
+        start: ByteSize::mib(128).as_u64(),
+        end: ByteSize::mib(512).as_u64() - 1,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_512_MB",
+        start: ByteSize::mib(512).as_u64(),
+        end: u64::MAX,
+    },
+];
+
+const OBJECTS_VERSION_COUNT_INTERVALS: [ObjectHistogramInterval; DATA_USAGE_VERSION_LEN] = [
+    ObjectHistogramInterval {
+        name: "UNVERSIONED",
+        start: 0,
+        end: 0,
+    },
+    ObjectHistogramInterval {
+        name: "SINGLE_VERSION",
+        start: 1,
+        end: 1,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_2_AND_10",
+        start: 2,
+        end: 9,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_10_AND_100",
+        start: 10,
+        end: 99,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_100_AND_1000",
+        start: 100,
+        end: 999,
+    },
+    ObjectHistogramInterval {
+        name: "BETWEEN_1000_AND_10000",
+        start: 1000,
+        end: 9999,
+    },
+    ObjectHistogramInterval {
+        name: "GREATER_THAN_10000",
+        start: 10000,
+        end: u64::MAX,
+    },
+];
+
+// sizeHistogram is a size histogram.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct SizeHistogram(Vec<u64>);
+
+impl Default for SizeHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_BUCKET_LEN])
+    }
+}
+
+impl SizeHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_HISTOGRAM_INTERVALS.iter().enumerate() {
+            if size >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        let mut spl_count = 0;
+        for (count, oh) in self.0.iter().zip(OBJECTS_HISTOGRAM_INTERVALS.iter()) {
+            if ByteSize::kib(1).as_u64() == oh.start && oh.end == ByteSize::mib(1).as_u64() - 1 {
+                res.insert(oh.name.to_string(), spl_count);
+            } else if ByteSize::kib(1).as_u64() <= oh.start && oh.end < ByteSize::mib(1).as_u64() {
+                spl_count += count;
+                res.insert(oh.name.to_string(), *count);
+            } else {
+                res.insert(oh.name.to_string(), *count);
+            }
+        }
+        res
+    }
+}
+
+// versionsHistogram is a histogram of number of versions in an object.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct VersionsHistogram(Vec<u64>);
+
+impl Default for VersionsHistogram {
+    fn default() -> Self {
+        Self(vec![0; DATA_USAGE_VERSION_LEN])
+    }
+}
+
+impl VersionsHistogram {
+    fn add(&mut self, size: u64) {
+        for (idx, interval) in OBJECTS_VERSION_COUNT_INTERVALS.iter().enumerate() {
+            if size >= interval.start && size <= interval.end {
+                self.0[idx] += 1;
+                break;
+            }
+        }
+    }
+
+    pub fn to_map(&self) -> HashMap<String, u64> {
+        let mut res = HashMap::new();
+        for (count, ov) in self.0.iter().zip(OBJECTS_VERSION_COUNT_INTERVALS.iter()) {
+            res.insert(ov.name.to_string(), *count);
+        }
+        res
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct ReplicationStats {
+    pub pending_size: u64,
+    pub replicated_size: u64,
+    pub failed_size: u64,
+    pub failed_count: u64,
+    pub pending_count: u64,
+    pub missed_threshold_size: u64,
+    pub after_threshold_size: u64,
+    pub missed_threshold_count: u64,
+    pub after_threshold_count: u64,
+    pub replicated_count: u64,
+}
+
+impl ReplicationStats {
+    pub fn empty(&self) -> bool {
+        self.replicated_size == 0 && self.failed_size == 0 && self.failed_count == 0
+    }
+}
+
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
+pub struct ReplicationAllStats {
+    pub targets: HashMap<String, ReplicationStats>,
+    pub replica_size: u64,
+    pub replica_count: u64,
+}
+
+impl ReplicationAllStats {
+    pub fn empty(&self) -> bool {
+        if self.replica_size != 0 && self.replica_count != 0 {
+            return false;
+        }
+        for (_, v) in self.targets.iter() {
+            if !v.empty() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct DataUsageEntry {
+    pub children: DataUsageHashMap,
+    // These fields do no include any children.
+    pub size: usize,
+    pub objects: usize,
+    pub versions: usize,
+    pub delete_markers: usize,
+    pub obj_sizes: SizeHistogram,
+    pub obj_versions: VersionsHistogram,
+    pub replication_stats: Option<ReplicationAllStats>,
+    // Todo: tier
+    // pub all_tier_stats: ,
+    pub compacted: bool,
+}
+
+impl DataUsageEntry {
+    pub fn add_child(&mut self, hash: &DataUsageHash) {
+        if self.children.contains(&hash.key()) {
+            return;
+        }
+
+        self.children.insert(hash.key());
+    }
+
+    pub fn add_sizes(&mut self, summary: &SizeSummary) {
+        self.size += summary.total_size;
+        self.versions += summary.versions;
+        self.delete_markers += summary.delete_markers;
+        self.obj_sizes.add(summary.total_size as u64);
+        self.obj_versions.add(summary.versions as u64);
+
+        let replication_stats = if self.replication_stats.is_none() {
+            self.replication_stats = Some(ReplicationAllStats::default());
+            self.replication_stats.as_mut().unwrap()
+        } else {
+            self.replication_stats.as_mut().unwrap()
+        };
+        replication_stats.replica_size += summary.replica_size as u64;
+        replication_stats.replica_count += summary.replica_count as u64;
+
+        for (arn, st) in &summary.repl_target_stats {
+            let tgt_stat = replication_stats
+                .targets
+                .entry(arn.to_string())
+                .or_insert(ReplicationStats::default());
+            tgt_stat.pending_size += st.pending_size as u64;
+            tgt_stat.failed_size += st.failed_size as u64;
+            tgt_stat.replicated_size += st.replicated_size as u64;
+            tgt_stat.replicated_count += st.replicated_count as u64;
+            tgt_stat.failed_count += st.failed_count as u64;
+            tgt_stat.pending_count += st.pending_count as u64;
+        }
+        // Todo:: tiers
+    }
+
+    pub fn merge(&mut self, other: &DataUsageEntry) {
+        self.objects += other.objects;
+        self.versions += other.versions;
+        self.delete_markers += other.delete_markers;
+        self.size += other.size;
+        if let Some(o_rep) = &other.replication_stats {
+            if self.replication_stats.is_none() {
+                self.replication_stats = Some(ReplicationAllStats::default());
+            }
+            let s_rep = self.replication_stats.as_mut().unwrap();
+            s_rep.targets.clear();
+            s_rep.replica_size += o_rep.replica_size;
+            s_rep.replica_count += o_rep.replica_count;
+            for (arn, stat) in o_rep.targets.iter() {
+                let st = s_rep.targets.entry(arn.clone()).or_default();
+                *st = ReplicationStats {
+                    pending_size: stat.pending_size + st.pending_size,
+                    failed_size: stat.failed_size + st.failed_size,
+                    replicated_size: stat.replicated_size + st.replicated_size,
+                    pending_count: stat.pending_count + st.pending_count,
+                    failed_count: stat.failed_count + st.failed_count,
+                    replicated_count: stat.replicated_count + st.replicated_count,
+                    ..Default::default()
+                };
+            }
+        }
+
+        for (i, v) in other.obj_sizes.0.iter().enumerate() {
+            self.obj_sizes.0[i] += v;
+        }
+
+        for (i, v) in other.obj_versions.0.iter().enumerate() {
+            self.obj_versions.0[i] += v;
+        }
+
+        // todo: tiers
+    }
+}
+
+#[derive(Clone)]
+pub struct DataUsageEntryInfo {
+    pub name: String,
+    pub parent: String,
+    pub entry: DataUsageEntry,
+}
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct DataUsageCacheInfo {
+    pub name: String,
+    pub next_cycle: u32,
+    pub last_update: Option<SystemTime>,
+    pub skip_healing: bool,
+    // todo: life_cycle
+    // pub life_cycle:
+    #[serde(skip)]
+    pub updates: Option<Sender<DataUsageEntry>>,
+    #[serde(skip)]
+    pub replication: Option<ReplicationConfiguration>,
+}
+
+// impl Default for DataUsageCacheInfo {
+//     fn default() -> Self {
+//         Self {
+//             name: Default::default(),
+//             next_cycle: Default::default(),
+//             last_update: SystemTime::now(),
+//             skip_healing: Default::default(),
+//             updates: Default::default(),
+//             replication: Default::default(),
+//         }
+//     }
+// }
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct DataUsageCache {
+    pub info: DataUsageCacheInfo,
+    pub cache: HashMap<String, DataUsageEntry>,
+}
+
+impl DataUsageCache {
+    pub async fn load(store: &SetDisks, name: &str) -> Result<Self> {
+        let mut d = DataUsageCache::default();
+        let mut retries = 0;
+        while retries < 5 {
+            let path = Path::new(BUCKET_META_PREFIX).join(name);
+            match store
+                .get_object_reader(
+                    RUSTFS_META_BUCKET,
+                    path.to_str().unwrap(),
+                    HTTPRangeSpec::nil(),
+                    HeaderMap::new(),
+                    &ObjectOptions {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+            {
+                Ok(mut reader) => {
+                    if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                        d = info
+                    }
+                    break;
+                }
+                Err(err) => match err.downcast_ref::<DiskError>() {
+                    Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
+                        match store
+                            .get_object_reader(
+                                RUSTFS_META_BUCKET,
+                                name,
+                                HTTPRangeSpec::nil(),
+                                HeaderMap::new(),
+                                &ObjectOptions {
+                                    no_lock: true,
+                                    ..Default::default()
+                                },
+                            )
+                            .await
+                        {
+                            Ok(mut reader) => {
+                                if let Ok(info) = Self::unmarshal(&reader.read_all().await?) {
+                                    d = info
+                                }
+                                break;
+                            }
+                            Err(_) => match err.downcast_ref::<DiskError>() {
+                                Some(DiskError::FileNotFound) | Some(DiskError::VolumeNotFound) => {
+                                    break;
+                                }
+                                _ => {}
+                            },
+                        }
+                    }
+                    _ => {}
+                },
+            }
+            retries += 1;
+            let dur = {
+                let mut rng = rand::thread_rng();
+                rng.gen_range(0..1_000)
+            };
+            sleep(Duration::from_millis(dur)).await;
+        }
+        Ok(d)
+    }
+
+    pub async fn save(&self, name: &str) -> Result<()> {
+        let buf = self.marshal_msg()?;
+        let buf_clone = buf.clone();
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
+        };
+        let store_clone = store.clone();
+        let name_clone = name.to_string();
+        tokio::spawn(async move {
+            let _ = save_config(&store_clone, &format!("{}{}", &name_clone, ".bkp"), &buf_clone).await;
+        });
+        save_config(store, name, &buf).await
+    }
+
+    pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
+        let hash = hash_path(path);
+        self.cache.insert(hash.key(), e);
+        if !parent.is_empty() {
+            let phash = hash_path(parent);
+            let p = {
+                let p = self.cache.entry(phash.key()).or_default();
+                p.add_child(&hash);
+                p.clone()
+            };
+            self.cache.insert(phash.key(), p);
+        }
+    }
+
+    pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
+        self.cache.insert(hash.key(), e.clone());
+        if let Some(parent) = parent {
+            self.cache.entry(parent.key()).or_default().add_child(hash);
+        }
+    }
+
+    pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
+        self.cache.get(&hash_path(path).key()).cloned()
+    }
+
+    pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
+        self.cache.entry(h.string()).or_default().children.clone()
+    }
+
+    pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
+        let mut root = root.clone();
+        for id in root.children.clone().iter() {
+            if let Some(e) = self.cache.get(id) {
+                let mut e = e.clone();
+                if !e.children.is_empty() {
+                    e = self.flatten(&e);
+                }
+                root.merge(&e);
+            }
+        }
+        root.children.clear();
+        root
+    }
+
+    pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
+        if let Some(e) = src.cache.get(&hash.string()) {
+            self.cache.insert(hash.key(), e.clone());
+            for ch in e.children.iter() {
+                if *ch == hash.key() {
+                    return;
+                }
+                self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
+            }
+            if let Some(parent) = parent {
+                let p = self.cache.entry(parent.key()).or_default();
+                p.add_child(hash);
+            }
+        }
+    }
+
+    pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
+        let mut need_remove = Vec::new();
+        if let Some(v) = self.cache.get(&hash.string()) {
+            for child in v.children.iter() {
+                need_remove.push(child.clone());
+            }
+        }
+        self.cache.remove(&hash.string());
+        need_remove.iter().for_each(|child| {
+            self.delete_recursive(&DataUsageHash(child.to_string()));
+        });
+    }
+
+    pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
+        match self.find(path) {
+            Some(root) => {
+                if root.children.is_empty() {
+                    return None;
+                }
+                let mut flat = self.flatten(&root);
+                if flat.replication_stats.is_some() && flat.replication_stats.as_ref().unwrap().empty() {
+                    flat.replication_stats = None;
+                }
+                Some(flat)
+            }
+            None => None,
+        }
+    }
+
+    pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
+        let want = hash.key();
+        if let Some(last_index) = want.rfind('/') {
+            if let Some(v) = self.find(&want[0..last_index]) {
+                if v.children.contains(&want) {
+                    let found = hash_path(&want[0..last_index]);
+                    return Some(found);
+                }
+            }
+        }
+
+        for (k, v) in self.cache.iter() {
+            if v.children.contains(&want) {
+                let found = DataUsageHash(k.clone());
+                return Some(found);
+            }
+        }
+        None
+    }
+
+    pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
+        match self.cache.get(&hash.key()) {
+            Some(due) => due.compacted,
+            None => false,
+        }
+    }
+
+    pub fn force_compact(&mut self, limit: usize) {
+        if self.cache.len() < limit {
+            return;
+        }
+        let top = hash_path(&self.info.name).key();
+        let top_e = match self.find(&top) {
+            Some(e) => e,
+            None => return,
+        };
+        if top_e.children.len() > DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS.try_into().unwrap() {
+            self.reduce_children_of(&hash_path(&self.info.name), limit, true);
+        }
+        if self.cache.len() <= limit {
+            return;
+        }
+
+        let mut found = HashSet::new();
+        found.insert(top);
+        mark(self, &top_e, &mut found);
+        self.cache.retain(|k, _| {
+            if !found.contains(k) {
+                return false;
+            }
+            true
+        });
+    }
+
+    pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
+        let e = match self.cache.get(&path.key()) {
+            Some(e) => e,
+            None => return,
+        };
+
+        if e.compacted {
+            return;
+        }
+
+        if e.children.len() > limit && compact_self {
+            let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
+            flat.compacted = true;
+            self.delete_recursive(path);
+            self.replace_hashed(path, &None, &flat);
+            return;
+        }
+        let total = self.total_children_rec(&path.key());
+        if total < limit {
+            return;
+        }
+
+        let mut leaves = Vec::new();
+        let mut remove = total - limit;
+        add(self, path, &mut leaves);
+        leaves.sort_by(|a, b| a.objects.cmp(&b.objects));
+
+        while remove > 0 && !leaves.is_empty() {
+            let e = leaves.first().unwrap();
+            let candidate = e.path.clone();
+            if candidate == *path && !compact_self {
+                break;
+            }
+            let removing = self.total_children_rec(&candidate.key());
+            let mut flat = match self.size_recursive(&candidate.key()) {
+                Some(flat) => flat,
+                None => {
+                    leaves.remove(0);
+                    continue;
+                }
+            };
+
+            flat.compacted = true;
+            self.delete_recursive(&candidate);
+            self.replace_hashed(&candidate, &None, &flat);
+
+            remove -= removing;
+            leaves.remove(0);
+        }
+    }
+
+    pub fn total_children_rec(&self, path: &str) -> usize {
+        let root = self.find(path);
+
+        if root.is_none() {
+            return 0;
+        }
+        let root = root.unwrap();
+        if root.children.is_empty() {
+            return 0;
+        }
+
+        let mut n = root.children.len();
+        for ch in root.children.iter() {
+            n += self.total_children_rec(ch);
+        }
+        n
+    }
+
+    pub fn merge(&mut self, o: &DataUsageCache) {
+        let mut existing_root = self.root();
+        let other_root = o.root();
+        if existing_root.is_none() && other_root.is_none() {
+            return;
+        }
+        if other_root.is_none() {
+            return;
+        }
+        if existing_root.is_none() {
+            *self = o.clone();
+            return;
+        }
+        if o.info.last_update.gt(&self.info.last_update) {
+            self.info.last_update = o.info.last_update;
+        }
+
+        existing_root.as_mut().unwrap().merge(other_root.as_ref().unwrap());
+        self.cache.insert(hash_path(&self.info.name).key(), existing_root.unwrap());
+        let e_hash = self.root_hash();
+        for key in other_root.as_ref().unwrap().children.iter() {
+            let entry = &o.cache[key];
+            let flat = o.flatten(entry);
+            let mut existing = self.cache[key].clone();
+            existing.merge(&flat);
+            self.replace_hashed(&DataUsageHash(key.clone()), &Some(e_hash.clone()), &existing);
+        }
+    }
+
+    pub fn root_hash(&self) -> DataUsageHash {
+        hash_path(&self.info.name)
+    }
+
+    pub fn root(&self) -> Option<DataUsageEntry> {
+        self.find(&self.info.name)
+    }
+
+    pub fn dui(&self, path: &str, buckets: &[BucketInfo]) -> DataUsageInfo {
+        let e = match self.find(path) {
+            Some(e) => e,
+            None => return DataUsageInfo::default(),
+        };
+        let flat = self.flatten(&e);
+        DataUsageInfo {
+            last_update: self.info.last_update,
+            objects_total_count: flat.objects as u64,
+            versions_total_count: flat.versions as u64,
+            delete_markers_total_count: flat.delete_markers as u64,
+            objects_total_size: flat.size as u64,
+            buckets_count: e.children.len() as u64,
+            buckets_usage: self.buckets_usage_info(buckets),
+            ..Default::default()
+        }
+    }
+
+    pub fn buckets_usage_info(&self, buckets: &[BucketInfo]) -> HashMap<String, BucketUsageInfo> {
+        let mut dst = HashMap::new();
+        for bucket in buckets.iter() {
+            let e = match self.find(&bucket.name) {
+                Some(e) => e,
+                None => continue,
+            };
+            let flat = self.flatten(&e);
+            let mut bui = BucketUsageInfo {
+                size: flat.size as u64,
+                versions_count: flat.versions as u64,
+                objects_count: flat.objects as u64,
+                delete_markers_count: flat.delete_markers as u64,
+                object_size_histogram: flat.obj_sizes.to_map(),
+                object_versions_histogram: flat.obj_versions.to_map(),
+                ..Default::default()
+            };
+            if let Some(rs) = &flat.replication_stats {
+                bui.replica_size = rs.replica_size;
+                bui.replica_count = rs.replica_count;
+
+                for (arn, stat) in rs.targets.iter() {
+                    bui.replication_info.insert(
+                        arn.clone(),
+                        BucketTargetUsageInfo {
+                            replication_pending_size: stat.pending_size,
+                            replicated_size: stat.replicated_size,
+                            replication_failed_size: stat.failed_size,
+                            replication_pending_count: stat.pending_count,
+                            replication_failed_count: stat.failed_count,
+                            replicated_count: stat.replicated_count,
+                            ..Default::default()
+                        },
+                    );
+                }
+            }
+            dst.insert(bucket.name.clone(), bui);
+        }
+        dst
+    }
+
+    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
+        let mut buf = Vec::new();
+
+        self.serialize(&mut Serializer::new(&mut buf))?;
+
+        Ok(buf)
+    }
+
+    pub fn unmarshal(buf: &[u8]) -> Result<Self> {
+        let t: Self = rmp_serde::from_slice(buf)?;
+        Ok(t)
+    }
+}
+
+#[derive(Default, Clone)]
+struct Inner {
+    objects: usize,
+    path: DataUsageHash,
+}
+
+fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, leaves: &mut Vec<Inner>) {
+    let e = match data_usage_cache.cache.get(&path.key()) {
+        Some(e) => e,
+        None => return,
+    };
+    if !e.children.is_empty() {
+        return;
+    }
+
+    let sz = data_usage_cache.size_recursive(&path.key()).unwrap_or_default();
+    leaves.push(Inner {
+        objects: sz.objects,
+        path: path.clone(),
+    });
+    for ch in e.children.iter() {
+        add(data_usage_cache, &DataUsageHash(ch.clone()), leaves);
+    }
+}
+
+fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
+    for k in entry.children.iter() {
+        found.insert(k.to_string());
+        if let Some(ch) = duc.cache.get(k) {
+            mark(duc, ch, found);
+        }
+    }
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct DataUsageHash(pub String);
+
+impl DataUsageHash {
+    pub fn string(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn key(&self) -> String {
+        self.0.clone()
+    }
+
+    pub fn mod_(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        hash as u32 % cycles == cycle % cycles
+    }
+
+    pub fn mod_alt(&self, cycle: u32, cycles: u32) -> bool {
+        if cycles <= 1 {
+            return cycles == 1;
+        }
+
+        let hash = self.calculate_hash();
+        (hash >> 32) as u32 % cycles == cycle % cycles
+    }
+
+    fn calculate_hash(&self) -> u64 {
+        let mut hasher = DefaultHasher::new();
+        self.0.hash(&mut hasher);
+        hasher.finish()
+    }
+}
+
+pub fn hash_path(data: &str) -> DataUsageHash {
+    let mut data = data;
+    if data != DATA_USAGE_ROOT {
+        data = data.trim_matches('/');
+    }
+    DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
+}

ecstore/src/heal/error.rs

@@ -0,0 +1,5 @@
+pub const ERR_IGNORE_FILE_CONTRIB: &str = "ignore this file's contribution toward data-usage";
+pub const ERR_SKIP_FILE: &str = "skip this file";
+pub const ERR_HEAL_STOP_SIGNALLED: &str = "heal stop signaled";
+pub const ERR_HEAL_IDLE_TIMEOUT: &str = "healing results were not consumed for too long";
+pub const ERR_RETRY_HEALING: &str = "some items failed to heal, we will retry healing this drive again";

ecstore/src/heal/heal_commands.rs

@@ -1,14 +1,14 @@
-use std::{
-    path::Path,
-    time::{SystemTime, UNIX_EPOCH},
-};
+use std::{path::Path, time::SystemTime};
 
+use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
+use time::OffsetDateTime;
 use tokio::sync::RwLock;
 
 use crate::{
     disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
     error::{Error, Result},
+    global::GLOBAL_BackgroundHealState,
     heal::heal_ops::HEALING_TRACKER_FILENAME,
     new_object_layer_fn,
     store_api::{BucketInfo, StorageAPI},
@@ -37,7 +37,11 @@ pub const DRIVE_STATE_ROOT_MOUNT: &str = "root-mount";
 pub const DRIVE_STATE_UNKNOWN: &str = "unknown";
 pub const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; // only returned by disk
 
-#[derive(Clone, Copy, Debug, Default)]
+lazy_static! {
+    pub static ref TIME_SENTINEL: OffsetDateTime = OffsetDateTime::from_unix_timestamp(0).unwrap();
+}
+
+#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize)]
 pub struct HealOpts {
     pub recursive: bool,
     pub dry_run: bool,
@@ -50,7 +54,7 @@ pub struct HealOpts {
     pub set: Option<usize>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct HealDriveInfo {
     pub uuid: String,
     pub endpoint: String,
@@ -74,11 +78,21 @@ pub struct HealResultItem {
     pub object_size: usize,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct HealStartSuccess {
     pub client_token: String,
     pub client_address: String,
-    pub start_time: u64,
+    pub start_time: SystemTime,
+}
+
+impl Default for HealStartSuccess {
+    fn default() -> Self {
+        Self {
+            client_token: Default::default(),
+            client_address: Default::default(),
+            start_time: SystemTime::now(),
+        }
+    }
 }
 
 pub type HealStopSuccess = HealStartSuccess;
@@ -91,8 +105,8 @@ pub struct HealingDisk {
     pub disk_index: Option<usize>,
     pub endpoint: String,
     pub path: String,
-    pub started: u64,
-    pub last_update: u64,
+    pub started: Option<OffsetDateTime>,
+    pub last_update: Option<SystemTime>,
     pub retry_attempts: u64,
     pub objects_total_count: u64,
     pub objects_total_size: u64,
@@ -121,8 +135,8 @@ pub struct HealingTracker {
     pub disk_index: Option<usize>,
     pub path: String,
     pub endpoint: String,
-    pub started: u64,
-    pub last_update: u64,
+    pub started: Option<OffsetDateTime>,
+    pub last_update: Option<SystemTime>,
     pub objects_total_count: u64,
     pub objects_total_size: u64,
     pub items_healed: u64,
@@ -150,9 +164,7 @@ pub struct HealingTracker {
 
 impl HealingTracker {
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
-        serde_json::to_string(self)
-            .map(|s| s.as_bytes().to_vec())
-            .map_err(|err| Error::from_string(err.to_string()))
+        serde_json::to_vec(self).map_err(|err| Error::from_string(err.to_string()))
     }
 
     pub fn unmarshal_msg(data: &[u8]) -> Result<Self> {
@@ -177,7 +189,7 @@ impl HealingTracker {
         self.object = String::new();
     }
 
-    pub async fn get_last_update(&self) -> u64 {
+    pub async fn get_last_update(&self) -> Option<SystemTime> {
         let _ = self.mu.read().await;
 
         self.last_update
@@ -224,7 +236,7 @@ impl HealingTracker {
 
     pub async fn update(&mut self) -> Result<()> {
         if let Some(disk) = &self.disk {
-            if healing(&disk.path().to_string_lossy().to_string()).await?.is_none() {
+            if healing(disk.path().to_string_lossy().as_ref()).await?.is_none() {
                 return Err(Error::from_string(format!("healingTracker: drive {} is not marked as healing", self.id)));
             }
             let _ = self.mu.write().await;
@@ -252,14 +264,11 @@ impl HealingTracker {
             (self.pool_index, self.set_index, self.disk_index) = store.get_pool_and_set(&self.id).await?;
         }
 
-        self.last_update = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .expect("Time went backwards")
-            .as_secs();
+        self.last_update = Some(SystemTime::now());
 
         let htracker_bytes = self.marshal_msg()?;
 
-        // TODO: globalBackgroundHealState
+        GLOBAL_BackgroundHealState.write().await.update_heal_status(self).await;
 
         if let Some(disk) = &self.disk {
             let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
@@ -270,7 +279,7 @@ impl HealingTracker {
         Ok(())
     }
 
-    async fn delete(&self) -> Result<()> {
+    pub async fn delete(&self) -> Result<()> {
         if let Some(disk) = &self.disk {
             let file_path = Path::new(BUCKET_META_PREFIX).join(HEALING_TRACKER_FILENAME);
             return disk
@@ -289,7 +298,7 @@ impl HealingTracker {
         Ok(())
     }
 
-    async fn is_healed(&self, bucket: &str) -> bool {
+    pub async fn is_healed(&self, bucket: &str) -> bool {
         let _ = self.mu.read().await;
         for v in self.healed_buckets.iter() {
             if v == bucket {
@@ -300,7 +309,7 @@ impl HealingTracker {
         false
     }
 
-    async fn resume(&mut self) {
+    pub async fn resume(&mut self) {
         let _ = self.mu.write().await;
 
         self.items_healed = self.resume_items_healed;
@@ -311,7 +320,7 @@ impl HealingTracker {
         self.bytes_skipped = self.resume_bytes_skipped;
     }
 
-    async fn bucket_done(&mut self, bucket: &str) {
+    pub async fn bucket_done(&mut self, bucket: &str) {
         let _ = self.mu.write().await;
 
         self.resume_items_healed = self.items_healed;
@@ -325,7 +334,7 @@ impl HealingTracker {
         self.queue_buckets.retain(|x| x != bucket);
     }
 
-    async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
+    pub async fn set_queue_buckets(&mut self, buckets: &[BucketInfo]) {
         let _ = self.mu.write().await;
 
         buckets.iter().for_each(|bucket| {
@@ -373,40 +382,40 @@ impl Clone for HealingTracker {
         Self {
             disk: self.disk.clone(),
             id: self.id.clone(),
-            pool_index: self.pool_index.clone(),
-            set_index: self.set_index.clone(),
-            disk_index: self.disk_index.clone(),
+            pool_index: self.pool_index,
+            set_index: self.set_index,
+            disk_index: self.disk_index,
             path: self.path.clone(),
             endpoint: self.endpoint.clone(),
-            started: self.started.clone(),
-            last_update: self.last_update.clone(),
-            objects_total_count: self.objects_total_count.clone(),
-            objects_total_size: self.objects_total_size.clone(),
-            items_healed: self.items_healed.clone(),
-            items_failed: self.items_failed.clone(),
-            item_skipped: self.item_skipped.clone(),
-            bytes_done: self.bytes_done.clone(),
-            bytes_failed: self.bytes_failed.clone(),
-            bytes_skipped: self.bytes_skipped.clone(),
+            started: self.started,
+            last_update: self.last_update,
+            objects_total_count: self.objects_total_count,
+            objects_total_size: self.objects_total_size,
+            items_healed: self.items_healed,
+            items_failed: self.items_failed,
+            item_skipped: self.item_skipped,
+            bytes_done: self.bytes_done,
+            bytes_failed: self.bytes_failed,
+            bytes_skipped: self.bytes_skipped,
             bucket: self.bucket.clone(),
             object: self.object.clone(),
-            resume_items_healed: self.resume_items_healed.clone(),
-            resume_items_failed: self.resume_items_failed.clone(),
-            resume_items_skipped: self.resume_items_skipped.clone(),
-            resume_bytes_done: self.resume_bytes_done.clone(),
-            resume_bytes_failed: self.resume_bytes_failed.clone(),
-            resume_bytes_skipped: self.resume_bytes_skipped.clone(),
+            resume_items_healed: self.resume_items_healed,
+            resume_items_failed: self.resume_items_failed,
+            resume_items_skipped: self.resume_items_skipped,
+            resume_bytes_done: self.resume_bytes_done,
+            resume_bytes_failed: self.resume_bytes_failed,
+            resume_bytes_skipped: self.resume_bytes_skipped,
             queue_buckets: self.queue_buckets.clone(),
             healed_buckets: self.healed_buckets.clone(),
             heal_id: self.heal_id.clone(),
-            retry_attempts: self.retry_attempts.clone(),
-            finished: self.finished.clone(),
+            retry_attempts: self.retry_attempts,
+            finished: self.finished,
             mu: RwLock::new(false),
         }
     }
 }
 
-async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
+pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker> {
     if let Some(disk) = disk {
         let disk_id = disk.get_disk_id().await?;
         if let Some(disk_id) = disk_id {
@@ -430,23 +439,20 @@ async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTracker
     }
 }
 
-async fn init_healing_tracker(disk: DiskStore, heal_id: String) -> Result<HealingTracker> {
-    let mut healing_tracker = HealingTracker::default();
-    healing_tracker.id = disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string());
-    healing_tracker.heal_id = heal_id;
-    healing_tracker.path = disk.to_string();
-    healing_tracker.endpoint = disk.endpoint().to_string();
-    healing_tracker.started = SystemTime::now()
-        .duration_since(UNIX_EPOCH)
-        .expect("Time went backwards")
-        .as_secs();
+pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> Result<HealingTracker> {
     let disk_location = disk.get_disk_location();
-    healing_tracker.pool_index = disk_location.pool_idx;
-    healing_tracker.set_index = disk_location.set_idx;
-    healing_tracker.disk_index = disk_location.disk_idx;
-    healing_tracker.disk = Some(disk);
-
-    Ok(healing_tracker)
+    Ok(HealingTracker {
+        id: disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()),
+        heal_id: heal_id.to_string(),
+        path: disk.to_string(),
+        endpoint: disk.endpoint().to_string(),
+        started: Some(OffsetDateTime::now_utc()),
+        pool_index: disk_location.pool_idx,
+        set_index: disk_location.set_idx,
+        disk_index: disk_location.disk_idx,
+        disk: Some(disk),
+        ..Default::default()
+    })
 }
 
 pub async fn healing(derive_path: &str) -> Result<Option<HealingTracker>> {

ecstore/src/heal/heal_ops.rs

@@ -1,12 +1,34 @@
+use super::{
+    background_heal_ops::HealTask,
+    data_scanner::HEAL_DELETE_DANGLING,
+    error::ERR_SKIP_FILE,
+    heal_commands::{
+        HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker,
+        HEAL_ITEM_BUCKET_METADATA,
+    },
+};
+use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT};
+use crate::store_api::StorageAPI;
+use crate::{
+    config::common::CONFIG_PREFIX,
+    disk::RUSTFS_META_BUCKET,
+    global::GLOBAL_BackgroundHealRoutine,
+    heal::{
+        error::ERR_HEAL_STOP_SIGNALLED,
+        heal_commands::{HealDriveInfo, DRIVE_STATE_OK},
+    },
+};
 use crate::{
     disk::{endpoint::Endpoint, MetaCacheEntry},
     endpoints::Endpoints,
     error::{Error, Result},
     global::GLOBAL_IsDistErasure,
-    heal::heal_commands::HEAL_UNKNOWN_SCAN,
+    heal::heal_commands::{HealStartSuccess, HEAL_UNKNOWN_SCAN},
+    new_object_layer_fn,
     utils::path::has_profix,
 };
 use lazy_static::lazy_static;
+use s3s::{S3Error, S3ErrorCode};
 use std::{
     collections::HashMap,
     future::Future,
@@ -24,18 +46,13 @@ use tokio::{
     },
     time::{interval, sleep},
 };
+use tracing::info;
 use uuid::Uuid;
 
-use super::{
-    background_heal_ops::HealTask,
-    heal_commands::{HealItemType, HealOpts, HealResultItem, HealScanMode, HealStopSuccess, HealingDisk, HealingTracker},
-};
-
 type HealStatusSummary = String;
 type ItemsMap = HashMap<HealItemType, usize>;
-pub type HealObjectFn = Arc<dyn Fn(&str, &str, &str, HealScanMode) -> Result<()> + Send + Sync>;
 pub type HealEntryFn =
-    Box<dyn Fn(String, MetaCacheEntry, HealScanMode) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send>;
+    Arc<dyn Fn(String, MetaCacheEntry, HealScanMode) -> Pin<Box<dyn Future<Output = Result<()>> + Send>> + Send + Sync + 'static>;
 
 pub const BG_HEALING_UUID: &str = "0000-0000-0000-0000";
 pub const HEALING_TRACKER_FILENAME: &str = ".healing.bin";
@@ -45,6 +62,10 @@ const HEAL_RUNNING_STATUS: &str = "running";
 const HEAL_STOPPED_STATUS: &str = "stopped";
 const HEAL_FINISHED_STATUS: &str = "finished";
 
+pub const RUESTFS_RESERVED_BUCKET: &str = "rustfs";
+pub const RUESTFS_RESERVED_BUCKET_PATH: &str = "/rustfs";
+pub const LOGIN_PATH_PREFIX: &str = "/login";
+
 const MAX_UNCONSUMED_HEAL_RESULT_ITEMS: usize = 1000;
 const HEAL_UNCONSUMED_TIMEOUT: std::time::Duration = Duration::from_secs(24 * 60 * 60);
 pub const NOP_HEAL: &str = "";
@@ -60,12 +81,13 @@ pub struct HealSequenceStatus {
     pub items: Vec<HealResultItem>,
 }
 
+#[derive(Debug, Default)]
 pub struct HealSource {
     pub bucket: String,
     pub object: String,
     pub version_id: String,
     pub no_wait: bool,
-    opts: Option<HealOpts>,
+    pub opts: Option<HealOpts>,
 }
 
 #[derive(Clone, Debug)]
@@ -73,8 +95,8 @@ pub struct HealSequence {
     pub bucket: String,
     pub object: String,
     pub report_progress: bool,
-    pub start_time: u64,
-    pub end_time: Arc<RwLock<u64>>,
+    pub start_time: SystemTime,
+    pub end_time: Arc<RwLock<SystemTime>>,
     pub client_token: String,
     pub client_address: String,
     pub force_started: bool,
@@ -93,6 +115,30 @@ pub struct HealSequence {
     rx: Arc<RwLock<Receiver<bool>>>,
 }
 
+pub fn new_bg_heal_sequence() -> HealSequence {
+    let hs = HealOpts {
+        remove: HEAL_DELETE_DANGLING,
+        ..Default::default()
+    };
+
+    HealSequence {
+        start_time: SystemTime::now(),
+        client_token: BG_HEALING_UUID.to_string(),
+        bucket: RUESTFS_RESERVED_BUCKET.to_string(),
+        setting: hs,
+        current_status: Arc::new(RwLock::new(HealSequenceStatus {
+            summary: HEAL_NOT_STARTED_STATUS.to_string(),
+            heal_setting: hs,
+            ..Default::default()
+        })),
+        report_progress: false,
+        scanned_items_map: HashMap::new(),
+        healed_items_map: HashMap::new(),
+        heal_failed_items_map: HashMap::new(),
+        ..Default::default()
+    }
+}
+
 impl Default for HealSequence {
     fn default() -> Self {
         let (h_tx, h_rx) = mpsc::channel(1);
@@ -101,8 +147,8 @@ impl Default for HealSequence {
             bucket: Default::default(),
             object: Default::default(),
             report_progress: Default::default(),
-            start_time: Default::default(),
-            end_time: Default::default(),
+            start_time: SystemTime::now(),
+            end_time: Arc::new(RwLock::new(SystemTime::now())),
             client_token: Default::default(),
             client_address: Default::default(),
             force_started: Default::default(),
@@ -144,23 +190,23 @@ impl HealSequence {
 }
 
 impl HealSequence {
-    fn get_scanned_items_count(&self) -> usize {
+    fn _get_scanned_items_count(&self) -> usize {
         self.scanned_items_map.values().sum()
     }
 
-    fn get_scanned_items_map(&self) -> ItemsMap {
+    fn _get_scanned_items_map(&self) -> ItemsMap {
         self.scanned_items_map.clone()
     }
 
-    fn get_healed_items_map(&self) -> ItemsMap {
+    fn _get_healed_items_map(&self) -> ItemsMap {
         self.healed_items_map.clone()
     }
 
-    fn get_heal_failed_items_map(&self) -> ItemsMap {
+    fn _get_heal_failed_items_map(&self) -> ItemsMap {
         self.heal_failed_items_map.clone()
     }
 
-    fn count_failed(&mut self, heal_type: HealItemType) {
+    pub fn count_failed(&mut self, heal_type: HealItemType) {
         *self.heal_failed_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -168,7 +214,7 @@ impl HealSequence {
             .as_secs();
     }
 
-    fn count_scanned(&mut self, heal_type: HealItemType) {
+    pub fn count_scanned(&mut self, heal_type: HealItemType) {
         *self.scanned_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -176,7 +222,7 @@ impl HealSequence {
             .as_secs();
     }
 
-    fn count_healed(&mut self, heal_type: HealItemType) {
+    pub fn count_healed(&mut self, heal_type: HealItemType) {
         *self.healed_items_map.entry(heal_type).or_insert(0) += 1;
         self.last_heal_activity = SystemTime::now()
             .duration_since(UNIX_EPOCH)
@@ -193,11 +239,11 @@ impl HealSequence {
     }
 
     async fn has_ended(&self) -> bool {
-        if self.client_token == BG_HEALING_UUID.to_string() {
+        if self.client_token == *BG_HEALING_UUID {
             return false;
         }
 
-        !(*(self.end_time.read().await) == self.start_time)
+        *(self.end_time.read().await) != self.start_time
     }
 
     async fn stop(&self) {
@@ -208,6 +254,7 @@ impl HealSequence {
     async fn push_heal_result_item(&self, r: &HealResultItem) -> Result<()> {
         let mut r = r.clone();
         let mut interval_timer = interval(HEAL_UNCONSUMED_TIMEOUT);
+        #[allow(unused_assignments)]
         let mut items_len = 0;
         loop {
             {
@@ -244,7 +291,7 @@ impl HealSequence {
         Ok(())
     }
 
-    async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> {
+    pub async fn queue_heal_task(&mut self, source: HealSource, heal_type: HealItemType) -> Result<()> {
         let mut task = HealTask::new(&source.bucket, &source.object, &source.version_id, &self.setting);
         if let Some(opts) = source.opts {
             task.opts = opts;
@@ -252,31 +299,104 @@ impl HealSequence {
             task.opts.scan_mode = HEAL_UNKNOWN_SCAN;
         }
 
-        self.count_scanned(heal_type);
+        self.count_scanned(heal_type.clone());
 
-        if source.no_wait {}
-
-        todo!()
-    }
-
-    fn heal_disk_meta() -> Result<()> {
-        todo!()
-    }
-
-    fn heal_items(&self, buckets_only: bool) -> Result<()> {
-        if self.client_token == BG_HEALING_UUID.to_string() {
+        if source.no_wait {
+            let task_str = format!("{:?}", task);
+            if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() {
+                info!("Task in the queue: {:?}", task_str);
+            }
             return Ok(());
         }
 
-        todo!()
+        let (resp_tx, mut resp_rx) = mpsc::channel(1);
+        task.resp_tx = Some(resp_tx);
+
+        let task_str = format!("{:?}", task);
+        if GLOBAL_BackgroundHealRoutine.read().await.tasks_tx.try_send(task).is_ok() {
+            info!("Task in the queue: {:?}", task_str);
+        }
+        let count_ok_drives = |drivers: &[HealDriveInfo]| {
+            let mut count = 0;
+            for drive in drivers.iter() {
+                if drive.state == DRIVE_STATE_OK {
+                    count += 1;
+                }
+            }
+            count
+        };
+
+        match resp_rx.recv().await {
+            Some(mut res) => {
+                if res.err.is_none() {
+                    self.count_healed(heal_type.clone());
+                } else {
+                    self.count_failed(heal_type.clone());
+                }
+                if !self.report_progress {
+                    if let Some(err) = res.err {
+                        if err.to_string() == ERR_SKIP_FILE {
+                            return Ok(());
+                        }
+                        return Err(err);
+                    } else {
+                        return Ok(());
+                    }
+                }
+                res.result.heal_item_type = heal_type.clone();
+                if let Some(err) = res.err.as_ref() {
+                    res.result.detail = err.to_string();
+                }
+                if res.result.parity_blocks > 0 && res.result.data_blocks > 0 && res.result.data_blocks > res.result.parity_blocks
+                {
+                    let got = count_ok_drives(&res.result.after);
+                    if got < res.result.parity_blocks {
+                        res.result.detail = format!(
+                            "quorum loss - expected {} minimum, got drive states in OK {}",
+                            res.result.parity_blocks, got
+                        );
+                    }
+                }
+                self.push_heal_result_item(&res.result).await
+            }
+            None => Ok(()),
+        }
     }
 
-    async fn traverse_and_heal(&self) {
+    async fn heal_disk_meta(h: Arc<RwLock<HealSequence>>) -> Result<()> {
+        HealSequence::heal_rustfs_sys_meta(h, CONFIG_PREFIX).await
+    }
+
+    async fn heal_items(h: Arc<RwLock<HealSequence>>, buckets_only: bool) -> Result<()> {
+        if h.read().await.client_token == *BG_HEALING_UUID {
+            return Ok(());
+        }
+
+        Self::heal_disk_meta(h.clone()).await?;
+        let bucket = h.read().await.bucket.clone();
+        Self::heal_bucket(h.clone(), &bucket, buckets_only).await
+    }
+
+    async fn traverse_and_heal(h: Arc<RwLock<HealSequence>>) {
         let buckets_only = false;
+        let result = match Self::heal_items(h.clone(), buckets_only).await {
+            Ok(_) => None,
+            Err(err) => Some(err),
+        };
+        let _ = h.read().await.traverse_and_heal_done_tx.read().await.send(result).await;
     }
 
-    fn heal_rustfs_sys_meta(&self, meta_prefix: String) -> Result<()> {
-        todo!()
+    async fn heal_rustfs_sys_meta(h: Arc<RwLock<HealSequence>>, meta_prefix: &str) -> Result<()> {
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
+        };
+        let setting = h.read().await.setting;
+        store
+            .heal_objects(RUSTFS_META_BUCKET, meta_prefix, &setting, h.clone(), true)
+            .await
     }
 
     async fn is_done(&self) -> bool {
@@ -286,13 +406,101 @@ impl HealSequence {
         }
         false
     }
+
+    pub async fn heal_bucket(hs: Arc<RwLock<HealSequence>>, bucket: &str, bucket_only: bool) -> Result<()> {
+        let (object, setting) = {
+            let mut hs_w = hs.write().await;
+            hs_w.queue_heal_task(
+                HealSource {
+                    bucket: bucket.to_string(),
+                    ..Default::default()
+                },
+                HEAL_ITEM_BUCKET.to_string(),
+            )
+            .await?;
+
+            if bucket_only {
+                return Ok(());
+            }
+
+            if !hs_w.setting.recursive {
+                if !hs_w.object.is_empty() {
+                    HealSequence::heal_object(hs.clone(), bucket, &hs_w.object, "", hs_w.setting.scan_mode).await?;
+                }
+                return Ok(());
+            }
+            (hs_w.object.clone(), hs_w.setting)
+        };
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))),
+        };
+        store.heal_objects(bucket, &object, &setting, hs.clone(), false).await
+    }
+
+    pub async fn heal_object(
+        hs: Arc<RwLock<HealSequence>>,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        _scan_mode: HealScanMode,
+    ) -> Result<()> {
+        let mut hs_w = hs.write().await;
+        if hs_w.is_quitting().await {
+            return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED));
+        }
+
+        let setting = hs_w.setting;
+        hs_w.queue_heal_task(
+            HealSource {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: version_id.to_string(),
+                opts: Some(setting),
+                ..Default::default()
+            },
+            HEAL_ITEM_OBJECT.to_string(),
+        )
+        .await?;
+
+        Ok(())
+    }
+
+    pub async fn heal_meta_object(
+        hs: Arc<RwLock<HealSequence>>,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        _scan_mode: HealScanMode,
+    ) -> Result<()> {
+        let mut hs_w = hs.write().await;
+        if hs_w.is_quitting().await {
+            return Err(Error::from_string(ERR_HEAL_STOP_SIGNALLED));
+        }
+
+        hs_w.queue_heal_task(
+            HealSource {
+                bucket: bucket.to_string(),
+                object: object.to_string(),
+                version_id: version_id.to_string(),
+                ..Default::default()
+            },
+            HEAL_ITEM_BUCKET_METADATA.to_string(),
+        )
+        .await?;
+
+        Ok(())
+    }
 }
 
-pub async fn heal_sequence_start(h: Arc<HealSequence>) {
+pub async fn heal_sequence_start(h: Arc<RwLock<HealSequence>>) {
+    let r = h.read().await;
     {
-        let mut current_status_w = h.current_status.write().await;
-        (*current_status_w).summary = HEAL_RUNNING_STATUS.to_string();
-        (*current_status_w).start_time = SystemTime::now()
+        let mut current_status_w = r.current_status.write().await;
+        current_status_w.summary = HEAL_RUNNING_STATUS.to_string();
+        current_status_w.start_time = SystemTime::now()
             .duration_since(UNIX_EPOCH)
             .expect("Time went backwards")
             .as_secs();
@@ -300,42 +508,35 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
 
     let h_clone = h.clone();
     spawn(async move {
-        h_clone.traverse_and_heal().await;
+        HealSequence::traverse_and_heal(h_clone).await;
     });
 
     let h_clone_1 = h.clone();
-    let mut x = h.traverse_and_heal_done_rx.write().await;
+    let mut x = r.traverse_and_heal_done_rx.write().await;
     select! {
-        _ = h.is_done() => {
-            *(h.end_time.write().await) = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .expect("Time went backwards")
-            .as_secs();
-        let mut current_status_w = h.current_status.write().await;
-        (*current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
+        _ = r.is_done() => {
+            *(r.end_time.write().await) = SystemTime::now();
+            let mut current_status_w = r.current_status.write().await;
+            current_status_w.summary = HEAL_FINISHED_STATUS.to_string();
 
-        spawn(async move {
-            let mut rx_w = h_clone_1.traverse_and_heal_done_rx.write().await;
-            rx_w.recv().await;
+            spawn(async move {
+                let binding = h_clone_1.read().await;
+                let mut rx_w = binding.traverse_and_heal_done_rx.write().await;
+                rx_w.recv().await;
             });
         }
         result = x.recv() => {
-            match result {
-                Some(err) => {
-                    match err {
-                        Some(err) => {
-                            let mut current_status_w = h.current_status.write().await;
-                            (current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
-                            (current_status_w).failure_detail = err.to_string();
-                        },
-                        None => {
-                            let mut current_status_w = h.current_status.write().await;
-                            (current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
-                        }
+            if let Some(err) = result {
+                match err {
+                    Some(err) => {
+                        let mut current_status_w = r.current_status.write().await;
+                        (current_status_w).summary = HEAL_STOPPED_STATUS.to_string();
+                        (current_status_w).failure_detail = err.to_string();
+                    },
+                    None => {
+                        let mut current_status_w = r.current_status.write().await;
+                        (current_status_w).summary = HEAL_FINISHED_STATUS.to_string();
                     }
-                },
-                None => {
-
                 }
             }
 
@@ -347,22 +548,37 @@ pub async fn heal_sequence_start(h: Arc<HealSequence>) {
 pub struct AllHealState {
     mu: RwLock<bool>,
 
-    heal_seq_map: HashMap<String, HealSequence>,
+    heal_seq_map: HashMap<String, Arc<RwLock<HealSequence>>>,
     heal_local_disks: HashMap<Endpoint, bool>,
     heal_status: HashMap<String, HealingTracker>,
 }
 
 impl AllHealState {
-    pub fn new(cleanup: bool) -> Self {
-        let hstate = AllHealState::default();
+    pub fn new(cleanup: bool) -> Arc<RwLock<Self>> {
+        let hstate = Arc::new(RwLock::new(AllHealState::default()));
+        let (_, mut rx) = broadcast::channel(1);
         if cleanup {
-            // spawn(f);
+            let hstate_clone = hstate.clone();
+            tokio::spawn(async move {
+                loop {
+                    select! {
+                        result = rx.recv() =>{
+                            if let Ok(true) = result {
+                                return;
+                            }
+                        }
+                        _ = sleep(Duration::from_secs(5 * 60)) => {
+                            hstate_clone.write().await.periodic_heal_seqs_clean().await;
+                        }
+                    }
+                }
+            });
         }
 
         hstate
     }
 
-    async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
+    pub async fn pop_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
         let _ = self.mu.write().await;
 
         self.heal_local_disks.retain(|k, _| {
@@ -382,14 +598,14 @@ impl AllHealState {
         });
     }
 
-    async fn update_heal_status(&mut self, tracker: &HealingTracker) {
+    pub async fn update_heal_status(&mut self, tracker: &HealingTracker) {
         let _ = self.mu.write().await;
         let _ = tracker.mu.read().await;
 
         self.heal_status.insert(tracker.id.clone(), tracker.clone());
     }
 
-    async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
+    pub async fn get_local_healing_disks(&self) -> HashMap<String, HealingDisk> {
         let _ = self.mu.read().await;
 
         let mut dst = HashMap::new();
@@ -400,7 +616,7 @@ impl AllHealState {
         dst
     }
 
-    async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
+    pub async fn get_heal_local_disk_endpoints(&self) -> Endpoints {
         let _ = self.mu.read().await;
 
         let mut endpoints = Vec::new();
@@ -413,13 +629,13 @@ impl AllHealState {
         Endpoints::from(endpoints)
     }
 
-    async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
+    pub async fn set_disk_healing_status(&mut self, ep: Endpoint, healing: bool) {
         let _ = self.mu.write().await;
 
         self.heal_local_disks.insert(ep, healing);
     }
 
-    async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
+    pub async fn push_heal_local_disks(&mut self, heal_local_disks: &[Endpoint]) {
         let _ = self.mu.write().await;
 
         heal_local_disks.iter().for_each(|heal_local_disk| {
@@ -427,37 +643,28 @@ impl AllHealState {
         });
     }
 
-    async fn periodic_heal_seqs_clean(&mut self, mut rx: Receiver<bool>) {
-        loop {
-            select! {
-                result = rx.recv() =>{
-                    if let Ok(true) = result {
-                        return;
-                    }
-                }
-                _ = sleep(Duration::from_secs(5 * 60)) => {
-                    let _ = self.mu.write().await;
-                    let now = SystemTime::now();
+    pub async fn periodic_heal_seqs_clean(&mut self) {
+        let _ = self.mu.write().await;
+        let now = SystemTime::now();
 
-                    let mut keys_to_reomve = Vec::new();
-                    for (k, v) in self.heal_seq_map.iter() {
-                        if v.has_ended().await && (UNIX_EPOCH + Duration::from_secs(*(v.end_time.read().await)) + KEEP_HEAL_SEQ_STATE_DURATION) < now {
-                            keys_to_reomve.push(k.clone())
-                        }
-                    }
-                    for key in keys_to_reomve.iter() {
-                        self.heal_seq_map.remove(key);
-                    }
-                }
+        let mut keys_to_reomve = Vec::new();
+        for (k, v) in self.heal_seq_map.iter() {
+            let r = v.read().await;
+            if r.has_ended().await && now.duration_since(*(r.end_time.read().await)).unwrap() > KEEP_HEAL_SEQ_STATE_DURATION {
+                keys_to_reomve.push(k.clone())
             }
         }
+        for key in keys_to_reomve.iter() {
+            self.heal_seq_map.remove(key);
+        }
     }
 
-    async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<HealSequence>, bool) {
+    pub async fn get_heal_sequence_by_token(&self, token: &str) -> (Option<Arc<RwLock<HealSequence>>>, bool) {
         let _ = self.mu.read().await;
 
         for v in self.heal_seq_map.values() {
-            if v.client_token == token {
+            let r = v.read().await;
+            if r.client_token == token {
                 return (Some(v.clone()), true);
             }
         }
@@ -465,15 +672,16 @@ impl AllHealState {
         (None, false)
     }
 
-    async fn get_heal_sequence(&self, path: &str) -> Option<HealSequence> {
+    pub async fn get_heal_sequence(&self, path: &str) -> Option<Arc<RwLock<HealSequence>>> {
         let _ = self.mu.read().await;
 
         self.heal_seq_map.get(path).cloned()
     }
 
-    async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
+    pub async fn stop_heal_sequence(&mut self, path: &str) -> Result<Vec<u8>> {
         let mut hsp = HealStopSuccess::default();
         if let Some(he) = self.get_heal_sequence(path).await {
+            let he = he.read().await;
             let client_token = he.client_token.clone();
             if *GLOBAL_IsDistErasure.read().await {
                 // TODO: proxy
@@ -513,23 +721,22 @@ impl AllHealState {
     // `keepHealSeqStateDuration`. This function also launches a
     // background routine to clean up heal results after the
     // aforementioned duration.
-    pub async fn launch_new_heal_sequence(&mut self, heal_sequence: &HealSequence) -> Result<Vec<u8>> {
-        let path = Path::new(&heal_sequence.bucket).join(heal_sequence.object.clone());
+    pub async fn launch_new_heal_sequence(&mut self, heal_sequence: Arc<RwLock<HealSequence>>) -> Result<Vec<u8>> {
+        let r = heal_sequence.read().await;
+        let path = Path::new(&r.bucket).join(r.object.clone());
         let path_s = path.to_str().unwrap();
-        if heal_sequence.force_started {
+        if r.force_started {
             self.stop_heal_sequence(path_s).await?;
-        } else {
-            if let Some(hs) = self.get_heal_sequence(path_s).await {
-                if !hs.has_ended().await {
-                    return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {}, token is {}", heal_sequence.client_address, heal_sequence.start_time, heal_sequence.client_token)));
-                }
+        } else if let Some(hs) = self.get_heal_sequence(path_s).await {
+            if !hs.read().await.has_ended().await {
+                return Err(Error::from_string(format!("Heal is already running on the given path (use force-start option to stop and start afresh). The heal was started by IP {} at {:?}, token is {}", r.client_address, r.start_time, r.client_token)));
             }
         }
 
         let _ = self.mu.write().await;
 
         for (k, v) in self.heal_seq_map.iter() {
-            if !v.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) {
+            if !v.read().await.has_ended().await && (has_profix(k, path_s) || has_profix(path_s, k)) {
                 return Err(Error::from_string(format!(
                     "The provided heal sequence path overlaps with an existing heal path: {}",
                     k
@@ -539,14 +746,25 @@ impl AllHealState {
 
         self.heal_seq_map.insert(path_s.to_string(), heal_sequence.clone());
 
-        let client_token = heal_sequence.client_token.clone();
+        let client_token = r.client_token.clone();
         if *GLOBAL_IsDistErasure.read().await {
             // TODO: proxy
         }
 
-        if heal_sequence.client_token == BG_HEALING_UUID {
+        if r.client_token == BG_HEALING_UUID {
             // For background heal do nothing, do not spawn an unnecessary goroutine.
+        } else {
+            let heal_sequence_clone = heal_sequence.clone();
+            tokio::spawn(async {
+                heal_sequence_start(heal_sequence_clone).await;
+            });
         }
-        todo!()
+
+        let b = serde_json::to_vec(&HealStartSuccess {
+            client_token,
+            client_address: r.client_address.clone(),
+            start_time: r.start_time,
+        })?;
+        Ok(b)
     }
 }

ecstore/src/heal/mod.rs

@@ -1,3 +1,8 @@
 pub mod background_heal_ops;
+pub mod data_scanner;
+pub mod data_scanner_metric;
+pub mod data_usage;
+pub mod data_usage_cache;
+pub mod error;
 pub mod heal_commands;
 pub mod heal_ops;

ecstore/src/lib.rs

@@ -9,7 +9,7 @@ pub mod erasure;
 pub mod error;
 mod file_meta;
 mod global;
-mod heal;
+pub mod heal;
 pub mod peer;
 mod quorum;
 pub mod set_disk;

ecstore/src/options.rs

@@ -77,7 +77,7 @@ fn get_default_opts(
 pub fn extract_metadata(headers: &HeaderMap<HeaderValue>) -> HashMap<String, String> {
     let mut metadata = HashMap::new();
 
-    extract_metadata_from_mime(&headers, &mut metadata);
+    extract_metadata_from_mime(headers, &mut metadata);
 
     metadata
 }

ecstore/src/peer.rs

@@ -1,14 +1,26 @@
 use async_trait::async_trait;
 use futures::future::join_all;
 use protos::node_service_time_out_client;
-use protos::proto_gen::node_service::{DeleteBucketRequest, GetBucketInfoRequest, ListBucketRequest, MakeBucketRequest};
+use protos::proto_gen::node_service::{
+    DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
+};
 use regex::Regex;
 use std::{collections::HashMap, fmt::Debug, sync::Arc};
+use tokio::sync::RwLock;
 use tonic::Request;
 use tracing::warn;
 
-use crate::disk::DiskAPI;
+use crate::disk::error::{is_all_buckets_not_found, is_all_not_found};
+use crate::disk::{DiskAPI, DiskStore};
+use crate::global::GLOBAL_LOCAL_DISK_MAP;
+use crate::heal::heal_commands::{
+    HealDriveInfo, HealOpts, HealResultItem, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK,
+    HEAL_ITEM_BUCKET,
+};
+use crate::heal::heal_ops::RUESTFS_RESERVED_BUCKET;
+use crate::quorum::{bucket_op_ignored_errs, reduce_write_quorum_errs};
 use crate::store::all_local_disk;
+use crate::utils::wildcard::is_rustfs_meta_bucket_name;
 use crate::{
     disk::{self, error::DiskError, VolumeInfo},
     endpoints::{EndpointServerPools, Node},
@@ -20,6 +32,7 @@ type Client = Arc<Box<dyn PeerS3Client>>;
 
 #[async_trait]
 pub trait PeerS3Client: Debug + Sync + Send + 'static {
+    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
     async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>;
     async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>>;
     async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>;
@@ -61,6 +74,79 @@ impl S3PeerSys {
 }
 
 impl S3PeerSys {
+    pub async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+        let mut opts = *opts;
+        let mut futures = Vec::with_capacity(self.clients.len());
+        for client in self.clients.iter() {
+            // client_clon
+            futures.push(async move {
+                match client.get_bucket_info(bucket, &BucketOptions::default()).await {
+                    Ok(_) => None,
+                    Err(err) => Some(err),
+                }
+            });
+        }
+        let errs = join_all(futures).await;
+
+        let mut pool_errs = Vec::new();
+        for pool_idx in 0..self.pools_count {
+            let mut per_pool_errs = Vec::new();
+            for (i, client) in self.clients.iter().enumerate() {
+                if let Some(v) = client.get_pools() {
+                    if v.contains(&pool_idx) {
+                        per_pool_errs.push(errs[i].clone());
+                    }
+                }
+            }
+            let qu = per_pool_errs.len() / 2;
+            pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, &bucket_op_ignored_errs(), qu));
+        }
+
+        if !opts.recreate {
+            opts.remove = is_all_not_found(&pool_errs);
+            opts.recursive = !opts.remove;
+        }
+
+        let mut futures = Vec::new();
+        let heal_bucket_results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.clients.len()]));
+        for (idx, client) in self.clients.iter().enumerate() {
+            let opts_clone = opts;
+            let heal_bucket_results_clone = heal_bucket_results.clone();
+            futures.push(async move {
+                match client.heal_bucket(bucket, &opts_clone).await {
+                    Ok(res) => {
+                        heal_bucket_results_clone.write().await[idx] = res;
+                        None
+                    }
+                    Err(err) => Some(err),
+                }
+            });
+        }
+        let errs = join_all(futures).await;
+
+        for pool_idx in 0..self.pools_count {
+            let mut per_pool_errs = Vec::new();
+            for (i, client) in self.clients.iter().enumerate() {
+                if let Some(v) = client.get_pools() {
+                    if v.contains(&pool_idx) {
+                        per_pool_errs.push(errs[i].clone());
+                    }
+                }
+            }
+            let qu = per_pool_errs.len() / 2;
+            if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, &bucket_op_ignored_errs(), qu) {
+                return Err(pool_err);
+            }
+        }
+
+        for (i, err) in errs.iter().enumerate() {
+            if err.is_none() {
+                return Ok(heal_bucket_results.read().await[i].clone());
+            }
+        }
+        Err(DiskError::VolumeNotFound.into())
+    }
+
     pub async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
         let mut futures = Vec::with_capacity(self.clients.len());
         for cli in self.clients.iter() {
@@ -236,6 +322,11 @@ impl PeerS3Client for LocalPeerS3Client {
     fn get_pools(&self) -> Option<Vec<usize>> {
         self.pools.clone()
     }
+
+    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+        heal_bucket_local(bucket, opts).await
+    }
+
     async fn list_bucket(&self, _opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         let local_disks = all_local_disk().await;
 
@@ -420,6 +511,29 @@ impl PeerS3Client for RemotePeerS3Client {
     fn get_pools(&self) -> Option<Vec<usize>> {
         self.pools.clone()
     }
+
+    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+        let options: String = serde_json::to_string(opts)?;
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?;
+        let request = Request::new(HealBucketRequest {
+            bucket: bucket.to_string(),
+            options,
+        });
+        let response = client.heal_bucket(request).await?.into_inner();
+        if !response.success {
+            return Err(Error::from_string(response.error_info.unwrap_or_default()));
+        }
+
+        Ok(HealResultItem {
+            heal_item_type: HEAL_ITEM_BUCKET.to_string(),
+            bucket: bucket.to_string(),
+            set_count: 0,
+            ..Default::default()
+        })
+    }
+
     async fn list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
         let options = serde_json::to_string(opts)?;
         let mut client = node_service_time_out_client(&self.addr)
@@ -528,7 +642,7 @@ fn is_reserved_bucket(bucket_name: &str) -> bool {
 }
 
 // 检查桶名是否为保留名或无效名
-fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool {
+pub fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool {
     if bucket_entry.is_empty() {
         return true;
     }
@@ -538,3 +652,134 @@ fn is_reserved_or_invalid_bucket(bucket_entry: &str, strict: bool) -> bool {
 
     result || is_meta_bucket(bucket_entry) || is_reserved_bucket(bucket_entry)
 }
+
+pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
+    let disks = clone_drives().await;
+    let before_state = Arc::new(RwLock::new(vec![String::new(); disks.len()]));
+    let after_state = Arc::new(RwLock::new(vec![String::new(); disks.len()]));
+
+    let mut futures = Vec::new();
+    for (index, disk) in disks.iter().enumerate() {
+        let disk = disk.clone();
+        let bucket = bucket.to_string();
+        let bs_clone = before_state.clone();
+        let as_clone = after_state.clone();
+        futures.push(async move {
+            let disk = match disk {
+                Some(disk) => disk,
+                None => {
+                    bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string();
+                    as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string();
+                    return Some(Error::new(DiskError::DiskNotFound));
+                }
+            };
+            bs_clone.write().await[index] = DRIVE_STATE_OK.to_string();
+            as_clone.write().await[index] = DRIVE_STATE_OK.to_string();
+
+            if bucket == RUESTFS_RESERVED_BUCKET {
+                return None;
+            }
+
+            match disk.stat_volume(&bucket).await {
+                Ok(_) => None,
+                Err(err) => match err.downcast_ref() {
+                    Some(DiskError::DiskNotFound) => {
+                        bs_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string();
+                        as_clone.write().await[index] = DRIVE_STATE_OFFLINE.to_string();
+                        Some(err)
+                    }
+                    Some(DiskError::VolumeNotFound) => {
+                        bs_clone.write().await[index] = DRIVE_STATE_MISSING.to_string();
+                        as_clone.write().await[index] = DRIVE_STATE_MISSING.to_string();
+                        Some(err)
+                    }
+                    _ => {
+                        bs_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string();
+                        as_clone.write().await[index] = DRIVE_STATE_CORRUPT.to_string();
+                        Some(err)
+                    }
+                },
+            }
+        });
+    }
+    let errs = join_all(futures).await;
+    let mut res = HealResultItem {
+        heal_item_type: HEAL_ITEM_BUCKET.to_string(),
+        bucket: bucket.to_string(),
+        disk_count: disks.len(),
+        set_count: 0,
+        ..Default::default()
+    };
+
+    if opts.dry_run {
+        return Ok(res);
+    }
+
+    for (disk, state) in disks.iter().zip(before_state.read().await.iter()) {
+        res.before.push(HealDriveInfo {
+            uuid: "".to_string(),
+            endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
+            state: state.to_string(),
+        });
+    }
+
+    if !is_rustfs_meta_bucket_name(bucket) && !is_all_buckets_not_found(&errs) && opts.remove {
+        let mut futures = Vec::new();
+        for disk in disks.iter() {
+            let disk = disk.clone();
+            let bucket = bucket.to_string();
+            futures.push(async move {
+                match disk {
+                    Some(disk) => {
+                        let _ = disk.delete_volume(&bucket).await;
+                        None
+                    }
+                    None => Some(Error::new(DiskError::DiskNotFound)),
+                }
+            });
+        }
+
+        let _ = join_all(futures).await;
+    }
+
+    if !opts.remove {
+        let mut futures = Vec::new();
+        for (idx, disk) in disks.iter().enumerate() {
+            let disk = disk.clone();
+            let bucket = bucket.to_string();
+            let bs_clone = before_state.clone();
+            let as_clone = after_state.clone();
+            let errs_clone = errs.clone();
+            futures.push(async move {
+                if bs_clone.read().await[idx] == DRIVE_STATE_MISSING {
+                    match disk.as_ref().unwrap().make_volume(&bucket).await {
+                        Ok(_) => {
+                            as_clone.write().await[idx] = DRIVE_STATE_OK.to_string();
+                            return None;
+                        }
+                        Err(err) => {
+                            return Some(err);
+                        }
+                    }
+                }
+                errs_clone[idx].clone()
+            });
+        }
+
+        let _ = join_all(futures).await;
+    }
+
+    for (disk, state) in disks.iter().zip(after_state.read().await.iter()) {
+        res.before.push(HealDriveInfo {
+            uuid: "".to_string(),
+            endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
+            state: state.to_string(),
+        });
+    }
+
+    Ok(res)
+}
+
+async fn clone_drives() -> Vec<Option<DiskStore>> {
+    GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::<Vec<_>>()
+}

ecstore/src/pools.rs

@@ -58,18 +58,19 @@ pub struct PoolDecommissionInfo {
     pub bytes_failed: usize,
 }
 
-struct PoolSpaceInfo {
+#[derive(Debug)]
+pub struct PoolSpaceInfo {
     pub free: usize,
     pub total: usize,
     pub used: usize,
 }
 
 impl ECStore {
-    pub fn status(&self, idx: usize) -> Result<PoolStatus> {
+    pub fn status(&self, _idx: usize) -> Result<PoolStatus> {
         unimplemented!()
     }
 
-    async fn get_decommission_pool_space_info(&self, idx: usize) -> Result<PoolSpaceInfo> {
+    async fn _get_decommission_pool_space_info(&self, idx: usize) -> Result<PoolSpaceInfo> {
         if let Some(sets) = self.pools.get(idx) {
             let mut info = sets.storage_info().await;
             info.backend = self.backend_info().await;
@@ -81,8 +82,8 @@ impl ECStore {
     }
 }
 
-fn get_total_usable_capacity(disks: &Vec<StorageDisk>, info: &StorageInfo) -> usize {
-    for disk in disks.iter() {
+fn _get_total_usable_capacity(disks: &Vec<StorageDisk>, _info: &StorageInfo) -> usize {
+    for _disk in disks.iter() {
         // if disk.pool_index < 0 || info.backend.standard_scdata.len() <= disk.pool_index {
         //     continue;
         // }

ecstore/src/quorum.rs

@@ -40,6 +40,16 @@ pub fn object_op_ignored_errs() -> Vec<Box<dyn CheckErrorFn>> {
     base
 }
 
+// bucket_op_ignored_errs
+pub fn bucket_op_ignored_errs() -> Vec<Box<dyn CheckErrorFn>> {
+    let mut base = base_ignored_errs();
+
+    let ext: Vec<Box<dyn CheckErrorFn>> = vec![Box::new(DiskError::DiskAccessDenied), Box::new(DiskError::UnformattedDisk)];
+
+    base.extend(ext);
+    base
+}
+
 // 用于检查错误是否被忽略的函数
 fn is_err_ignored(err: &Error, ignored_errs: &[Box<dyn CheckErrorFn>]) -> bool {
     ignored_errs.iter().any(|ignored_err| ignored_err.is(err))

ecstore/src/sets.rs

@@ -10,15 +10,16 @@ use uuid::Uuid;
 
 use crate::{
     disk::{
+        error::DiskError,
         format::{DistributionAlgoVersion, FormatV3},
-        DiskAPI, DiskStore,
+        new_disk, DiskAPI, DiskInfo, DiskOption, DiskStore,
     },
-    endpoints::PoolEndpoints,
+    endpoints::{Endpoints, PoolEndpoints},
     error::{Error, Result},
     global::{is_dist_erasure, GLOBAL_LOCAL_DISK_SET_DRIVES},
-    heal::{
-        heal_commands::{HealOpts, HealResultItem},
-        heal_ops::HealObjectFn,
+    heal::heal_commands::{
+        HealDriveInfo, HealOpts, HealResultItem, DRIVE_STATE_CORRUPT, DRIVE_STATE_MISSING, DRIVE_STATE_OFFLINE, DRIVE_STATE_OK,
+        HEAL_ITEM_METADATA,
     },
     set_disk::SetDisks,
     store_api::{
@@ -26,9 +27,11 @@ use crate::{
         ListMultipartsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOptions,
         ObjectToDelete, PartInfo, PutObjReader, StorageAPI, StorageInfo,
     },
+    store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
     utils::hash,
 };
 
+use crate::heal::heal_ops::HealSequence;
 use tokio::time::Duration;
 use tokio_util::sync::CancellationToken;
 use tracing::info;
@@ -177,7 +180,7 @@ impl Sets {
         self.connect_disks().await;
 
         // TODO: config interval
-        let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3));
+        let mut interval = tokio::time::interval(Duration::from_secs(15 * 3));
         let cloned_token = self.ctx.clone();
         loop {
             tokio::select! {
@@ -523,7 +526,7 @@ impl StorageAPI for Sets {
             .complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
             .await
     }
-    async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
+    async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
         unimplemented!()
     }
 
@@ -534,24 +537,226 @@ impl StorageAPI for Sets {
     async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> {
         unimplemented!()
     }
-    async fn heal_format(&self, dry_run: bool) -> Result<HealResultItem> {
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        let (disks, _) = init_storage_disks_with_errors(
+            &self.endpoints.endpoints,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await;
+        let (formats, errs) = load_format_erasure_all(&disks, true).await;
+        if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
+            return Ok((HealResultItem::default(), Some(err)));
+        }
+        let ref_format = match get_format_erasure_in_quorum(&formats) {
+            Ok(format) => format,
+            Err(err) => return Ok((HealResultItem::default(), Some(err))),
+        };
+        let mut res = HealResultItem {
+            heal_item_type: HEAL_ITEM_METADATA.to_string(),
+            detail: "disk-format".to_string(),
+            disk_count: self.set_count * self.set_drive_count,
+            set_count: self.set_count,
+            ..Default::default()
+        };
+        let before_derives = formats_to_drives_info(&self.endpoints.endpoints, &formats, &errs);
+        res.before = vec![HealDriveInfo::default(); before_derives.len()];
+        res.after = vec![HealDriveInfo::default(); before_derives.len()];
+
+        for v in before_derives.iter() {
+            res.before.push(v.clone());
+            res.after.push(v.clone());
+        }
+        if DiskError::UnformattedDisk.count_errs(&errs) == 0 {
+            return Ok((res, Some(Error::new(DiskError::NoHealRequired))));
+        }
+
+        if !self.format.eq(&ref_format) {
+            return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
+        }
+
+        let format_op_id = Uuid::new_v4().to_string();
+        let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);
+        if !dry_run {
+            let mut tmp_new_formats = vec![None; self.set_count * self.set_drive_count];
+            for (i, set) in new_format_sets.iter().enumerate() {
+                for (j, fm) in set.iter().enumerate() {
+                    if let Some(fm) = fm {
+                        res.after[i * self.set_drive_count + j].uuid = fm.erasure.this.to_string();
+                        res.after[i * self.set_drive_count + j].state = DRIVE_STATE_OK.to_string();
+                        tmp_new_formats[i * self.set_drive_count + j] = Some(fm.clone());
+                    }
+                }
+            }
+            // Save new formats `format.json` on unformatted disks.
+            for (fm, disk) in tmp_new_formats.iter_mut().zip(disks.iter()) {
+                if fm.is_some() && disk.is_some() && save_format_file(disk, fm, &format_op_id).await.is_err() {
+                    let _ = disk.as_ref().unwrap().close().await;
+                    *fm = None;
+                }
+            }
+
+            for (index, fm) in tmp_new_formats.iter().enumerate() {
+                if let Some(fm) = fm {
+                    let (m, n) = match ref_format.find_disk_index_by_disk_id(fm.erasure.this) {
+                        Ok((m, n)) => (m, n),
+                        Err(_) => continue,
+                    };
+                    if let Some(set) = self.disk_set.get(m) {
+                        if let Some(Some(disk)) = set.disks.read().await.get(n) {
+                            let _ = disk.close().await;
+                        }
+                    }
+
+                    if let Some(Some(disk)) = disks.get(index) {
+                        self.disk_set[m].renew_disk(&disk.endpoint()).await;
+                    }
+                }
+            }
+        }
+        Ok((res, None))
+    }
+    async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
         unimplemented!()
     }
-    async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
-        unimplemented!()
-    }
-    async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result<HealResultItem> {
+    async fn heal_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        opts: &HealOpts,
+    ) -> Result<(HealResultItem, Option<Error>)> {
         self.get_disks_by_key(object)
             .heal_object(bucket, object, version_id, opts)
             .await
     }
-    async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> {
+    async fn heal_objects(
+        &self,
+        _bucket: &str,
+        _prefix: &str,
+        _opts: &HealOpts,
+        _hs: Arc<RwLock<HealSequence>>,
+        _is_meta: bool,
+    ) -> Result<()> {
         unimplemented!()
     }
-    async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
+    async fn get_pool_and_set(&self, _id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
         unimplemented!()
     }
-    async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
+    async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> {
         unimplemented!()
     }
 }
+
+async fn _close_storage_disks(disks: &[Option<DiskStore>]) {
+    let mut futures = Vec::with_capacity(disks.len());
+    for disk in disks.iter().flatten() {
+        let disk = disk.clone();
+        futures.push(tokio::spawn(async move {
+            let _ = disk.close().await;
+        }));
+    }
+    let _ = join_all(futures).await;
+}
+
+async fn init_storage_disks_with_errors(
+    endpoints: &Endpoints,
+    opts: &DiskOption,
+) -> (Vec<Option<DiskStore>>, Vec<Option<Error>>) {
+    // Bootstrap disks.
+    let disks = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()]));
+    let errs = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()]));
+    let mut futures = Vec::with_capacity(endpoints.as_ref().len());
+    for (index, endpoint) in endpoints.as_ref().iter().enumerate() {
+        let ep = endpoint.clone();
+        let opt = opts.clone();
+        let disks_clone = disks.clone();
+        let errs_clone = errs.clone();
+        futures.push(tokio::spawn(async move {
+            match new_disk(&ep, &opt).await {
+                Ok(disk) => {
+                    disks_clone.write().await[index] = Some(disk);
+                    errs_clone.write().await[index] = None;
+                }
+                Err(err) => {
+                    disks_clone.write().await[index] = None;
+                    errs_clone.write().await[index] = Some(err);
+                }
+            }
+        }));
+    }
+    let _ = join_all(futures).await;
+    let disks = disks.read().await.clone();
+    let errs = errs.read().await.clone();
+    (disks, errs)
+}
+
+fn formats_to_drives_info(endpoints: &Endpoints, formats: &[Option<FormatV3>], errs: &[Option<Error>]) -> Vec<HealDriveInfo> {
+    let mut before_drives = Vec::with_capacity(endpoints.as_ref().len());
+    for (index, format) in formats.iter().enumerate() {
+        let drive = endpoints.get_string(index);
+        let state = if format.is_some() {
+            DRIVE_STATE_OK
+        } else {
+            if let Some(Some(err)) = errs.get(index) {
+                match err.downcast_ref::<DiskError>() {
+                    Some(DiskError::UnformattedDisk) => DRIVE_STATE_MISSING,
+                    Some(DiskError::DiskNotFound) => DRIVE_STATE_OFFLINE,
+                    _ => DRIVE_STATE_CORRUPT,
+                };
+            }
+            DRIVE_STATE_CORRUPT
+        };
+
+        let uuid = if let Some(format) = format {
+            format.erasure.this.to_string()
+        } else {
+            "".to_string()
+        };
+        before_drives.push(HealDriveInfo {
+            uuid,
+            endpoint: drive,
+            state: state.to_string(),
+        });
+    }
+    before_drives
+}
+
+fn new_heal_format_sets(
+    ref_format: &FormatV3,
+    set_count: usize,
+    set_drive_count: usize,
+    formats: &[Option<FormatV3>],
+    errs: &[Option<Error>],
+) -> (Vec<Vec<Option<FormatV3>>>, Vec<Vec<DiskInfo>>) {
+    let mut new_formats = vec![vec![None; set_drive_count]; set_count];
+    let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
+    for (i, set) in ref_format.erasure.sets.iter().enumerate() {
+        for j in 0..set.len() {
+            if let Some(Some(err)) = errs.get(i * set_drive_count + j) {
+                if let Some(DiskError::UnformattedDisk) = err.downcast_ref::<DiskError>() {
+                    let mut fm = FormatV3::new(set_count, set_drive_count);
+                    fm.id = ref_format.id;
+                    fm.format = ref_format.format.clone();
+                    fm.version = ref_format.version.clone();
+                    fm.erasure.this = ref_format.erasure.sets[i][j];
+                    fm.erasure.sets = ref_format.erasure.sets.clone();
+                    fm.erasure.version = ref_format.erasure.version.clone();
+                    fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
+                    new_formats[i][j] = Some(fm);
+                }
+            }
+            if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j]) {
+                if let Some(info) = &format.disk_info {
+                    if !info.endpoint.is_empty() {
+                        current_disks_info[i][j] = info.clone();
+                    }
+                }
+            }
+        }
+    }
+
+    (new_formats, current_disks_info)
+}

ecstore/src/store.rs

@@ -1,18 +1,19 @@
 #![allow(clippy::map_entry)]
 
-use crate::bucket::metadata;
 use crate::bucket::metadata_sys::{self, init_bucket_metadata_sys, set_bucket_metadata};
 use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
 use crate::config::GLOBAL_StorageClass;
 use crate::config::{self, storageclass, GLOBAL_ConfigSys};
-use crate::disk::endpoint::EndpointType;
+use crate::disk::endpoint::{Endpoint, EndpointType};
 use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions, MetaCacheEntry};
 use crate::global::{
     is_dist_erasure, is_erasure_sd, set_global_deployment_id, set_object_layer, DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION,
     DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES,
 };
-use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode};
-use crate::heal::heal_ops::HealObjectFn;
+use crate::heal::data_usage::{DataUsageInfo, DATA_USAGE_ROOT};
+use crate::heal::data_usage_cache::{DataUsageCache, DataUsageCacheInfo};
+use crate::heal::heal_commands::{HealOpts, HealResultItem, HealScanMode, HEAL_ITEM_METADATA};
+use crate::heal::heal_ops::{HealEntryFn, HealSequence};
 use crate::new_object_layer_fn;
 use crate::pools::PoolMeta;
 use crate::store_api::{BackendByte, BackendDisks, BackendInfo, ListMultipartsInfo, ObjectIO, StorageInfo};
@@ -45,18 +46,21 @@ use http::HeaderMap;
 use lazy_static::lazy_static;
 use rand::Rng;
 use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
+use s3s::{S3Error, S3ErrorCode};
 use std::cmp::Ordering;
 use std::slice::Iter;
+use std::time::SystemTime;
 use std::{
     collections::{HashMap, HashSet},
     sync::Arc,
     time::Duration,
 };
 use time::OffsetDateTime;
-use tokio::fs;
-use tokio::sync::{RwLock, Semaphore};
-
-use tracing::{debug, info, warn};
+use tokio::sync::mpsc::Sender;
+use tokio::sync::{broadcast, mpsc, RwLock, Semaphore};
+use tokio::time::interval;
+use tokio::{fs, select};
+use tracing::{debug, info};
 use uuid::Uuid;
 
 const MAX_UPLOADS_LIST: usize = 10000;
@@ -494,6 +498,103 @@ impl ECStore {
         internal_get_pool_info_existing_with_opts(&self.pools, bucket, object, opts).await
     }
 
+    pub async fn ns_scanner(
+        &self,
+        updates: Sender<DataUsageInfo>,
+        want_cycle: usize,
+        heal_scan_mode: HealScanMode,
+    ) -> Result<()> {
+        let all_buckets = self.list_bucket(&BucketOptions::default()).await?;
+        if all_buckets.is_empty() {
+            let _ = updates.send(DataUsageInfo::default()).await;
+            return Ok(());
+        }
+
+        let mut total_results = 0;
+        let mut result_index = 0;
+        self.pools.iter().for_each(|pool| {
+            total_results += pool.disk_set.len();
+        });
+        let results = Arc::new(RwLock::new(vec![DataUsageCache::default(); total_results]));
+        let (cancel, _) = broadcast::channel(100);
+        let first_err = Arc::new(RwLock::new(None));
+        let mut futures = Vec::new();
+        for pool in self.pools.iter() {
+            for set in pool.disk_set.iter() {
+                let index = result_index;
+                let results_clone = results.clone();
+                let first_err_clone = first_err.clone();
+                let cancel_clone = cancel.clone();
+                let all_buckets_clone = all_buckets.clone();
+                futures.push(async move {
+                    let (tx, mut rx) = mpsc::channel(100);
+                    let task = tokio::spawn(async move {
+                        loop {
+                            match rx.recv().await {
+                                Some(info) => {
+                                    results_clone.write().await[index] = info;
+                                }
+                                None => {
+                                    return;
+                                }
+                            }
+                        }
+                    });
+                    if let Err(err) = set
+                        .ns_scanner(&all_buckets_clone, want_cycle as u32, tx, heal_scan_mode)
+                        .await
+                    {
+                        let mut f_w = first_err_clone.write().await;
+                        if f_w.is_none() {
+                            *f_w = Some(err);
+                        }
+                        let _ = cancel_clone.send(true);
+                        return;
+                    }
+                    let _ = task.await;
+                });
+                result_index += 1;
+            }
+        }
+        let (update_closer_tx, mut update_close_rx) = mpsc::channel(10);
+        let mut ctx_clone = cancel.subscribe();
+        let all_buckets_clone = all_buckets.clone();
+        let task = tokio::spawn(async move {
+            let mut last_update: Option<SystemTime> = None;
+            let mut interval = interval(Duration::from_secs(30));
+            let all_merged = Arc::new(RwLock::new(DataUsageCache::default()));
+            loop {
+                select! {
+                    _ = ctx_clone.recv() => {
+                        return;
+                    }
+                    _ = update_close_rx.recv() => {
+                        update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await;
+                        return;
+                    }
+                    _ = interval.tick() => {
+                        update_scan(all_merged.clone(), results.clone(), &mut last_update, all_buckets_clone.clone(), updates.clone()).await;
+                    }
+                }
+            }
+        });
+        let _ = join_all(futures).await;
+        let mut ctx_closer = cancel.subscribe();
+        select! {
+            _ = update_closer_tx.send(true) => {
+
+            }
+            _ = ctx_closer.recv() => {
+
+            }
+        }
+        let _ = task.await;
+        if let Some(err) = first_err.read().await.as_ref() {
+            return Err(err.clone());
+        }
+        Ok(())
+    }
+
     async fn get_latest_object_info_with_idx(
         &self,
         bucket: &str,
@@ -515,8 +616,7 @@ impl ECStore {
 
         let mut idx_res = Vec::with_capacity(self.pools.len());
 
-        let mut idx = 0;
-        for result in results {
+        for (idx, result) in results.into_iter().enumerate() {
             match result {
                 Ok(res) => {
                     idx_res.push(IndexRes {
@@ -533,8 +633,6 @@ impl ECStore {
                     });
                 }
             }
-
-            idx += 1;
         }
 
         // TODO: test order
@@ -589,6 +687,33 @@ impl ECStore {
     }
 }
 
+async fn update_scan(
+    all_merged: Arc<RwLock<DataUsageCache>>,
+    results: Arc<RwLock<Vec<DataUsageCache>>>,
+    last_update: &mut Option<SystemTime>,
+    all_buckets: Vec<BucketInfo>,
+    updates: Sender<DataUsageInfo>,
+) {
+    let mut w = all_merged.write().await;
+    *w = DataUsageCache {
+        info: DataUsageCacheInfo {
+            name: DATA_USAGE_ROOT.to_string(),
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    for info in results.read().await.iter() {
+        if info.info.last_update.is_none() {
+            return;
+        }
+        w.merge(info);
+    }
+    if w.info.last_update > *last_update && w.root().is_none() {
+        let _ = updates.send(w.dui(&w.info.name, &all_buckets)).await;
+        *last_update = w.info.last_update;
+    }
+}
+
 pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     let disk_path = match fs::canonicalize(disk_path).await {
         Ok(disk_path) => disk_path,
@@ -606,6 +731,14 @@ pub async fn find_local_disk(disk_path: &String) -> Option<DiskStore> {
     None
 }
 
+pub async fn get_disk_via_endpoint(endpoint: &Endpoint) -> Option<DiskStore> {
+    let global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await;
+    if global_set_drives.is_empty() {
+        return GLOBAL_LOCAL_DISK_MAP.read().await[&endpoint.to_string()].clone();
+    }
+    global_set_drives[endpoint.pool_idx as usize][endpoint.set_idx as usize][endpoint.disk_idx as usize].clone()
+}
+
 pub async fn all_local_disk_path() -> Vec<String> {
     let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await;
     disk_map.keys().cloned().collect()
@@ -1306,7 +1439,7 @@ impl StorageAPI for ECStore {
         Ok(ListMultipartsInfo {
             key_marker: key_marker.to_owned(),
             upload_id_marker: upload_id_marker.to_owned(),
-            max_uploads: max_uploads,
+            max_uploads,
             uploads,
             prefix: prefix.to_owned(),
             delimiter: delimiter.to_owned(),
@@ -1439,53 +1572,187 @@ impl StorageAPI for ECStore {
         }
         counts
     }
-    async fn heal_format(&self, dry_run: bool) -> Result<HealResultItem> {
-        unimplemented!()
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        let mut r = HealResultItem {
+            heal_item_type: HEAL_ITEM_METADATA.to_string(),
+            detail: "disk-format".to_string(),
+            ..Default::default()
+        };
+
+        let mut count_no_heal = 0;
+        for pool in self.pools.iter() {
+            let (mut result, err) = pool.heal_format(dry_run).await?;
+            if let Some(err) = err {
+                match err.downcast_ref::<DiskError>() {
+                    Some(DiskError::NoHealRequired) => {
+                        count_no_heal += 1;
+                    }
+                    _ => {
+                        continue;
+                    }
+                }
+            }
+            r.disk_count += result.disk_count;
+            r.set_count += result.set_count;
+            r.before.append(&mut result.before);
+            r.after.append(&mut result.after);
+        }
+        if count_no_heal == self.pools.len() {
+            return Ok((r, Some(Error::new(DiskError::NoHealRequired))));
+        }
+        Ok((r, None))
     }
 
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
-        unimplemented!()
+        self.peer_sys.heal_bucket(bucket, opts).await
     }
-    async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result<HealResultItem> {
+    async fn heal_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        opts: &HealOpts,
+    ) -> Result<(HealResultItem, Option<Error>)> {
         let object = utils::path::encode_dir_object(object);
-        let mut errs = HashMap::new();
-        let mut results = HashMap::new();
+        let errs = Arc::new(RwLock::new(vec![None; self.pools.len()]));
+        let results = Arc::new(RwLock::new(vec![HealResultItem::default(); self.pools.len()]));
+        let mut futures = Vec::with_capacity(self.pools.len());
         for (idx, pool) in self.pools.iter().enumerate() {
             //TODO: IsSuspended
-            match pool.heal_object(bucket, &object, version_id, opts).await {
-                Ok(mut result) => {
-                    result.object = utils::path::decode_dir_object(&result.object);
-                    results.insert(idx, result);
+            let object = object.clone();
+            let results = results.clone();
+            let errs = errs.clone();
+            futures.push(async move {
+                match pool.heal_object(bucket, &object, version_id, opts).await {
+                    Ok((mut result, err)) => {
+                        result.object = utils::path::decode_dir_object(&result.object);
+                        results.write().await.insert(idx, result);
+                        errs.write().await.insert(idx, err);
+                    }
+                    Err(err) => {
+                        errs.write().await.insert(idx, Some(err));
+                    }
                 }
-                Err(err) => {
-                    errs.insert(idx, err);
-                }
-            }
+            });
         }
+        let _ = join_all(futures).await;
 
         // Return the first nil error
-        for i in 0..self.pools.len() {
-            if !errs.contains_key(&i) {
-                return Ok(results.remove(&i).unwrap());
+        for (index, err) in errs.read().await.iter().enumerate() {
+            if err.is_none() {
+                return Ok((results.write().await.remove(index), None));
             }
         }
 
         // No pool returned a nil error, return the first non 'not found' error
-        for (k, err) in errs.iter() {
-            match err.downcast_ref::<DiskError>() {
-                Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
-                _ => return Ok(results.remove(k).unwrap()),
+        for (index, err) in errs.read().await.iter().enumerate() {
+            match err {
+                Some(err) => match err.downcast_ref::<DiskError>() {
+                    Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
+                    _ => return Ok((results.write().await.remove(index), Some(err.clone()))),
+                },
+                None => {
+                    return Ok((results.write().await.remove(index), None));
+                }
             }
         }
 
         // At this stage, all errors are 'not found'
         if !version_id.is_empty() {
-            return Err(Error::new(DiskError::FileVersionNotFound));
+            return Ok((HealResultItem::default(), Some(Error::new(DiskError::FileVersionNotFound))));
         }
 
-        Err(Error::new(DiskError::FileNotFound))
+        Ok((HealResultItem::default(), Some(Error::new(DiskError::FileNotFound))))
     }
-    async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()> {
+    async fn heal_objects(
+        &self,
+        bucket: &str,
+        prefix: &str,
+        opts: &HealOpts,
+        hs: Arc<RwLock<HealSequence>>,
+        is_meta: bool,
+    ) -> Result<()> {
+        let opts_clone = *opts;
+        let heal_entry: HealEntryFn = Arc::new(move |bucket: String, entry: MetaCacheEntry, scan_mode: HealScanMode| {
+            let opts_clone = opts_clone;
+            let hs_clone = hs.clone();
+            Box::pin(async move {
+                if entry.is_dir() {
+                    return Ok(());
+                }
+
+                if bucket == RUSTFS_META_BUCKET
+                    && Pattern::new("buckets/*/.metacache/*")
+                        .map(|p| p.matches(&entry.name))
+                        .unwrap_or(false)
+                    || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
+                    || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
+                    || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
+                {
+                    return Ok(());
+                }
+                let fivs = match entry.file_info_versions(&bucket) {
+                    Ok(fivs) => fivs,
+                    Err(_) => {
+                        if is_meta {
+                            return HealSequence::heal_meta_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await;
+                        } else {
+                            return HealSequence::heal_object(hs_clone.clone(), &bucket, &entry.name, "", scan_mode).await;
+                        }
+                    }
+                };
+
+                if opts_clone.remove && !opts_clone.dry_run {
+                    let layer = new_object_layer_fn();
+                    let lock = layer.read().await;
+                    let store = match lock.as_ref() {
+                        Some(s) => s,
+                        None => {
+                            return Err(Error::from(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())))
+                        }
+                    };
+                    if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts_clone).await {
+                        info!("unable to check object {}/{} for abandoned data: {}", bucket, entry.name, err.to_string());
+                    }
+                }
+                for version in fivs.versions.iter() {
+                    if is_meta {
+                        if let Err(err) = HealSequence::heal_meta_object(
+                            hs_clone.clone(),
+                            &bucket,
+                            &version.name,
+                            &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()),
+                            scan_mode,
+                        )
+                        .await
+                        {
+                            match err.downcast_ref() {
+                                Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
+                                _ => {
+                                    return Err(err);
+                                }
+                            }
+                        }
+                    } else if let Err(err) = HealSequence::heal_object(
+                        hs_clone.clone(),
+                        &bucket,
+                        &version.name,
+                        &version.version_id.map(|v| v.to_string()).unwrap_or("".to_string()),
+                        scan_mode,
+                    )
+                    .await
+                    {
+                        match err.downcast_ref() {
+                            Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
+                            _ => {
+                                return Err(err);
+                            }
+                        }
+                    }
+                }
+                Ok(())
+            })
+        });
         let mut first_err = None;
         for (idx, pool) in self.pools.iter().enumerate() {
             if opts.pool.is_some() && opts.pool.unwrap() != idx {
@@ -1498,7 +1765,7 @@ impl StorageAPI for ECStore {
                     continue;
                 }
 
-                if let Err(err) = set.list_and_heal(bucket, prefix, opts, func.clone()).await {
+                if let Err(err) = set.list_and_heal(bucket, prefix, opts, heal_entry.clone()).await {
                     if first_err.is_none() {
                         first_err = Some(err)
                     }
@@ -1736,67 +2003,6 @@ fn check_put_object_args(bucket: &str, object: &str) -> Result<()> {
     Ok(())
 }
 
-pub async fn heal_entry(
-    bucket: String,
-    entry: MetaCacheEntry,
-    scan_mode: HealScanMode,
-    opts: HealOpts,
-    func: HealObjectFn,
-) -> Result<()> {
-    if entry.is_dir() {
-        return Ok(());
-    }
-
-    // We might land at .metacache, .trash, .multipart
-    // no need to heal them skip, only when bucket
-    // is '.rustfs.sys'
-    if bucket == RUSTFS_META_BUCKET {
-        if Pattern::new("buckets/*/.metacache/*")
-            .map(|p| p.matches(&entry.name))
-            .unwrap_or(false)
-            || Pattern::new("tmp/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
-            || Pattern::new("multipart/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
-            || Pattern::new("tmp-old/*").map(|p| p.matches(&entry.name)).unwrap_or(false)
-        {
-            return Ok(());
-        }
-    }
-
-    let layer = new_object_layer_fn();
-    let lock = layer.read().await;
-    let store = match lock.as_ref() {
-        Some(s) => s,
-        None => return Err(Error::msg("errServerNotInitialized")),
-    };
-
-    match entry.file_info_versions(&bucket) {
-        Ok(fivs) => {
-            if opts.remove && !opts.dry_run {
-                if let Err(err) = store.check_abandoned_parts(&bucket, &entry.name, &opts).await {
-                    return Err(Error::from_string(format!(
-                        "unable to check object {}/{} for abandoned data: {}",
-                        bucket, entry.name, err
-                    )));
-                }
-            }
-
-            for version in fivs.versions.iter() {
-                let version_id = version.version_id.map_or("".to_string(), |version_id| version_id.to_string());
-                if let Err(err) = func(&bucket, &entry.name, &version_id, scan_mode) {
-                    match err.downcast_ref::<DiskError>() {
-                        Some(DiskError::FileNotFound) | Some(DiskError::FileVersionNotFound) => {}
-                        _ => return Err(err),
-                    }
-                }
-            }
-        }
-        Err(_) => {
-            return func(&bucket, &entry.name, "", scan_mode);
-        }
-    }
-    Ok(())
-}
-
 async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>> {
     let opts = &DiskInfoOptions::default();
     let mut res = vec![None; disks.len()];

ecstore/src/store_api.rs

@@ -1,12 +1,8 @@
-use std::collections::HashMap;
-
+use crate::heal::heal_ops::HealSequence;
 use crate::{
     disk::DiskStore,
     error::{Error, Result},
-    heal::{
-        heal_commands::{HealOpts, HealResultItem},
-        heal_ops::HealObjectFn,
-    },
+    heal::heal_commands::{HealOpts, HealResultItem},
     utils::path::decode_dir_object,
     xhttp,
 };
@@ -15,13 +11,18 @@ use http::HeaderMap;
 use rmp_serde::Serializer;
 use s3s::dto::StreamingBlob;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
 use time::OffsetDateTime;
+use tokio::sync::RwLock;
 use uuid::Uuid;
 
 pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
 pub const BLOCK_SIZE_V2: usize = 1048576; // 1M
 pub const RESERVED_METADATA_PREFIX: &str = "X-Rustfs-Internal-";
 pub const RESERVED_METADATA_PREFIX_LOWER: &str = "X-Rustfs-Internal-";
+pub const RUSTFS_HEALING: &str = "X-Rustfs-Internal-healing";
+pub const RUSTFS_DATA_MOVE: &str = "X-Rustfs-Internal-data-mov";
 
 // #[derive(Debug, Clone)]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
@@ -237,6 +238,16 @@ impl FileInfo {
         Err(Error::msg("part not found"))
     }
 
+    pub fn set_healing(&mut self) {
+        if self.metadata.is_none() {
+            self.metadata = Some(HashMap::new());
+        }
+
+        if let Some(metadata) = self.metadata.as_mut() {
+            metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
+        }
+    }
+
     pub fn set_inline_data(&mut self) {
         if let Some(meta) = self.metadata.as_mut() {
             meta.insert("x-rustfs-inline-data".to_owned(), "true".to_owned());
@@ -978,10 +989,23 @@ pub trait StorageAPI: ObjectIO {
     async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
     async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
 
-    async fn heal_format(&self, dry_run: bool) -> Result<HealResultItem>;
+    async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)>;
     async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
-    async fn heal_object(&self, bucket: &str, object: &str, version_id: &str, opts: &HealOpts) -> Result<HealResultItem>;
-    async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, func: HealObjectFn) -> Result<()>;
+    async fn heal_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        version_id: &str,
+        opts: &HealOpts,
+    ) -> Result<(HealResultItem, Option<Error>)>;
+    async fn heal_objects(
+        &self,
+        bucket: &str,
+        prefix: &str,
+        opts: &HealOpts,
+        hs: Arc<RwLock<HealSequence>>,
+        is_meta: bool,
+    ) -> Result<()>;
     async fn get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)>;
     async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>;
 }

ecstore/src/store_init.rs

@@ -4,10 +4,11 @@ use crate::{
     disk::{
         error::DiskError,
         format::{FormatErasureVersion, FormatMetaVersion, FormatV3},
-        new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
+        new_disk, DiskInfoOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET,
     },
     endpoints::Endpoints,
     error::{Error, Result},
+    heal::heal_commands::init_healing_tracker,
 };
 use futures::future::join_all;
 use std::{
@@ -114,7 +115,7 @@ fn init_format_erasure(
     fms
 }
 
-fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
+pub fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3> {
     let mut countmap = HashMap::new();
 
     for f in formats.iter() {
@@ -148,7 +149,7 @@ fn get_format_erasure_in_quorum(formats: &[Option<FormatV3>]) -> Result<FormatV3
     Ok(format)
 }
 
-fn check_format_erasure_values(
+pub fn check_format_erasure_values(
     formats: &[Option<FormatV3>],
     // disks: &Vec<Option<DiskStore>>,
     set_drive_count: usize,
@@ -184,7 +185,7 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> {
 }
 
 // load_format_erasure_all 读取所有foramt.json
-async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
+pub async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Vec<Option<FormatV3>>, Vec<Option<Error>>) {
     let mut futures = Vec::with_capacity(disks.len());
     let mut datas = Vec::with_capacity(disks.len());
     let mut errors = Vec::with_capacity(disks.len());
@@ -220,7 +221,7 @@ async fn load_format_erasure_all(disks: &[Option<DiskStore>], heal: bool) -> (Ve
     (datas, errors)
 }
 
-pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result<FormatV3, Error> {
+pub async fn load_format_erasure(disk: &DiskStore, heal: bool) -> Result<FormatV3, Error> {
     let data = disk
         .read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
         .await
@@ -231,9 +232,17 @@ pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result<Format
             None => e,
         })?;
 
-    let fm = FormatV3::try_from(data.as_slice())?;
+    let mut fm = FormatV3::try_from(data.as_slice())?;
 
-    // TODO: heal
+    if heal {
+        let info = disk
+            .disk_info(&DiskInfoOptions {
+                noop: heal,
+                ..Default::default()
+            })
+            .await?;
+        fm.disk_info = Some(info);
+    }
 
     Ok(fm)
 }
@@ -242,7 +251,7 @@ async fn save_format_file_all(disks: &[Option<DiskStore>], formats: &[Option<For
     let mut futures = Vec::with_capacity(disks.len());
 
     for (i, disk) in disks.iter().enumerate() {
-        futures.push(save_format_file(disk, &formats[i]));
+        futures.push(save_format_file(disk, &formats[i], ""));
     }
 
     let mut errors = Vec::with_capacity(disks.len());
@@ -262,7 +271,7 @@ async fn save_format_file_all(disks: &[Option<DiskStore>], formats: &[Option<For
     errors
 }
 
-async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3>) -> Result<()> {
+pub async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3>, heal_id: &str) -> Result<()> {
     if disk.is_none() {
         return Err(Error::new(DiskError::DiskNotFound));
     }
@@ -281,6 +290,10 @@ async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3>) -
         .await?;
 
     disk.set_disk_id(Some(format.erasure.this)).await?;
+    if !heal_id.is_empty() {
+        let mut ht = init_healing_tracker(disk.clone(), heal_id).await?;
+        return ht.save().await;
+    }
 
     Ok(())
 }

ecstore/src/utils/bool_flag.rs

@@ -0,0 +1,9 @@
+use crate::error::{Error, Result};
+
+pub fn parse_bool(str: &str) -> Result<bool> {
+    match str {
+        "1" | "t" | "T" | "true" | "TRUE" | "True" | "on" | "ON" | "On" | "enabled" => Ok(true),
+        "0" | "f" | "F" | "false" | "FALSE" | "False" | "off" | "OFF" | "Off" | "disabled" => Ok(false),
+        _ => Err(Error::from_string(format!("ParseBool: parsing {}", str))),
+    }
+}

ecstore/src/utils/ellipses.rs

@@ -40,6 +40,10 @@ impl Pattern {
     pub fn len(&self) -> usize {
         self.seq.len()
     }
+
+    pub fn is_empty(&self) -> bool {
+        self.seq.is_empty()
+    }
 }
 
 /// contains a list of patterns provided in the input.

ecstore/src/utils/mod.rs

@@ -1,3 +1,4 @@
+pub mod bool_flag;
 pub mod crypto;
 pub mod ellipses;
 pub mod fs;

ecstore/src/utils/path.rs

@@ -69,6 +69,19 @@ pub fn path_join(elem: &[PathBuf]) -> PathBuf {
     joined_path
 }
 
+pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) {
+    let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR);
+    if let Some(m) = path.find(SLASH_SEPARATOR) {
+        return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string());
+    }
+
+    (path.to_string(), "".to_string())
+}
+
+pub fn path_to_bucket_object(s: &str) -> (String, String) {
+    path_to_bucket_object_with_base_path("", s)
+}
+
 pub fn base_dir_from_prefix(prefix: &str) -> String {
     let mut base_dir = dir(prefix).to_owned();
     if base_dir == "." || base_dir == "./" || base_dir == "/" {

ecstore/src/utils/wildcard.rs

@@ -1,3 +1,5 @@
+use crate::disk::RUSTFS_META_BUCKET;
+
 pub fn match_simple(pattern: &str, name: &str) -> bool {
     if pattern.is_empty() {
         return name == pattern;
@@ -65,3 +67,7 @@ pub fn match_as_pattern_prefix(pattern: &str, text: &str) -> bool {
     }
     text.len() <= pattern.len()
 }
+
+pub fn is_rustfs_meta_bucket_name(bucket: &str) -> bool {
+    bucket.starts_with(RUSTFS_META_BUCKET)
+}

rustfs/src/grpc.rs

@@ -6,6 +6,7 @@ use ecstore::{
         UpdateMetadataOpts, WalkDirOptions,
     },
     erasure::Writer,
+    heal::{data_usage_cache::DataUsageCache, heal_commands::HealOpts},
     peer::{LocalPeerS3Client, PeerS3Client},
     store::{all_local_disk_path, find_local_disk},
     store_api::{BucketOptions, DeleteBucketOptions, FileInfo, MakeBucketOptions},
@@ -20,14 +21,14 @@ use protos::{
         DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, DeleteVersionRequest,
         DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse,
         DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest,
-        GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest,
-        ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest,
-        MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse,
-        ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse,
-        RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse,
-        StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest,
-        VerifyFileResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest,
-        WriteMetadataResponse, WriteRequest, WriteResponse,
+        GetBucketInfoResponse, HealBucketRequest, HealBucketResponse, ListBucketRequest, ListBucketResponse, ListDirRequest,
+        ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest,
+        MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, NsScannerRequest, NsScannerResponse, PingRequest,
+        PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse,
+        ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse,
+        RenameFileRequst, RenameFileResponse, RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse,
+        UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, WalkDirRequest, WalkDirResponse,
+        WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse,
     },
 };
 use tokio::sync::mpsc;
@@ -109,6 +110,32 @@ impl Node for NodeService {
         }))
     }
 
+    async fn heal_bucket(&self, request: Request<HealBucketRequest>) -> Result<Response<HealBucketResponse>, Status> {
+        debug!("heal bucket");
+        let request = request.into_inner();
+        let options = match serde_json::from_str::<HealOpts>(&request.options) {
+            Ok(options) => options,
+            Err(err) => {
+                return Ok(tonic::Response::new(HealBucketResponse {
+                    success: false,
+                    error_info: Some(format!("decode HealOpts failed: {}", err)),
+                }))
+            }
+        };
+
+        match self.local_peer.heal_bucket(&request.bucket, &options).await {
+            Ok(_) => Ok(tonic::Response::new(HealBucketResponse {
+                success: true,
+                error_info: None,
+            })),
+
+            Err(err) => Ok(tonic::Response::new(HealBucketResponse {
+                success: false,
+                error_info: Some(format!("heal bucket failed: {}", err)),
+            })),
+        }
+    }
+
     async fn list_bucket(&self, request: Request<ListBucketRequest>) -> Result<Response<ListBucketResponse>, Status> {
         debug!("list bucket");
 
@@ -1270,6 +1297,96 @@ impl Node for NodeService {
         }
     }
 
+    type NsScannerStream = ResponseStream<NsScannerResponse>;
+    async fn ns_scanner(&self, request: Request<Streaming<NsScannerRequest>>) -> Result<Response<Self::NsScannerStream>, Status> {
+        info!("ns_scanner");
+
+        let mut in_stream = request.into_inner();
+        let (tx, rx) = mpsc::channel(10);
+
+        tokio::spawn(async move {
+            match in_stream.next().await {
+                Some(Ok(request)) => {
+                    if let Some(disk) = find_local_disk(&request.disk).await {
+                        let cache = match serde_json::from_str::<DataUsageCache>(&request.cache) {
+                            Ok(cache) => cache,
+                            Err(_) => {
+                                tx.send(Ok(NsScannerResponse {
+                                    success: false,
+                                    update: "".to_string(),
+                                    data_usage_cache: "".to_string(),
+                                    error_info: Some("can not decode DataUsageCache".to_string()),
+                                }))
+                                .await
+                                .expect("working rx");
+                                return;
+                            }
+                        };
+                        let (updates_tx, mut updates_rx) = mpsc::channel(100);
+                        let tx_clone = tx.clone();
+                        let task = tokio::spawn(async move {
+                            loop {
+                                match updates_rx.recv().await {
+                                    Some(update) => {
+                                        let update = serde_json::to_string(&update).expect("encode failed");
+                                        tx_clone
+                                            .send(Ok(NsScannerResponse {
+                                                success: true,
+                                                update,
+                                                data_usage_cache: "".to_string(),
+                                                error_info: Some("can not decode DataUsageCache".to_string()),
+                                            }))
+                                            .await
+                                            .expect("working rx");
+                                    }
+                                    None => return,
+                                }
+                            }
+                        });
+                        let data_usage_cache = disk.ns_scanner(&cache, updates_tx, request.scan_mode as usize).await;
+                        let _ = task.await;
+                        match data_usage_cache {
+                            Ok(data_usage_cache) => {
+                                let data_usage_cache = serde_json::to_string(&data_usage_cache).expect("encode failed");
+                                tx.send(Ok(NsScannerResponse {
+                                    success: true,
+                                    update: "".to_string(),
+                                    data_usage_cache,
+                                    error_info: Some("can not decode DataUsageCache".to_string()),
+                                }))
+                                .await
+                                .expect("working rx");
+                            }
+                            Err(_) => {
+                                tx.send(Ok(NsScannerResponse {
+                                    success: false,
+                                    update: "".to_string(),
+                                    data_usage_cache: "".to_string(),
+                                    error_info: Some("scanner failed".to_string()),
+                                }))
+                                .await
+                                .expect("working rx");
+                            }
+                        }
+                    } else {
+                        tx.send(Ok(NsScannerResponse {
+                            success: false,
+                            update: "".to_string(),
+                            data_usage_cache: "".to_string(),
+                            error_info: Some("can not find disk".to_string()),
+                        }))
+                        .await
+                        .expect("working rx");
+                    }
+                }
+                _ => todo!(),
+            }
+        });
+
+        let out_stream = ReceiverStream::new(rx);
+        Ok(tonic::Response::new(Box::pin(out_stream)))
+    }
+
     async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
         let request = request.into_inner();
         match &serde_json::from_str::<LockArgs>(&request.args) {

rustfs/src/main.rs

@@ -7,6 +7,7 @@ use clap::Parser;
 use common::error::{Error, Result};
 use ecstore::{
     endpoints::EndpointServerPools,
+    heal::data_scanner::init_data_scanner,
     set_global_endpoints,
     store::{init_local_disks, ECStore},
     update_erasure_type,
@@ -184,6 +185,8 @@ async fn run(opt: config::Opt) -> Result<()> {
         .map_err(|err| Error::from_string(err.to_string()))?;
 
     store.init().await.map_err(|err| Error::from_string(err.to_string()))?;
+    // init scanner
+    init_data_scanner().await;
 
     tokio::select! {
         _ = tokio::signal::ctrl_c() => {

rustfs/src/storage/acess.rs

@@ -10,7 +10,7 @@ use s3s::auth::Credentials;
 use s3s::{dto::*, s3_error, S3Request, S3Result};
 use uuid::Uuid;
 
-#[derive(Debug, Default, Clone)]
+#[derive(Default, Clone)]
 struct ReqInfo {
     pub card: Option<Credentials>,
     pub action: Option<Action>,
@@ -51,7 +51,7 @@ impl S3Access for FS {
 
         let req_info = ReqInfo {
             card: cx.credentials().cloned(),
-            action: action,
+            action,
             ..Default::default()
         };
 

rustfs/src/storage/ecfs.rs

@@ -341,15 +341,14 @@ impl S3 for FS {
 
         let content_type = {
             if let Some(content_type) = info.content_type {
-                let ct = match ContentType::from_str(&content_type) {
+                match ContentType::from_str(&content_type) {
                     Ok(res) => Some(res),
                     Err(err) => {
                         error!("parse content-type err {} {:?}", &content_type, err);
                         //
                         None
                     }
-                };
-                ct
+                }
             } else {
                 None
             }
@@ -411,15 +410,14 @@ impl S3 for FS {
 
         let content_type = {
             if let Some(content_type) = info.content_type {
-                let ct = match ContentType::from_str(&content_type) {
+                match ContentType::from_str(&content_type) {
                     Ok(res) => Some(res),
                     Err(err) => {
                         error!("parse content-type err {} {:?}", &content_type, err);
                         //
                         None
                     }
-                };
-                ct
+                }
             } else {
                 None
             }
@@ -1536,9 +1534,7 @@ impl S3 for FS {
             }
         }
 
-        let mut grants = Vec::new();
-
-        grants.push(Grant {
+        let grants = vec![Grant {
             grantee: Some(Grantee {
                 type_: Type::from_static(Type::CANONICAL_USER),
                 display_name: None,
@@ -1547,12 +1543,11 @@ impl S3 for FS {
                 uri: None,
             }),
             permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
-        });
+        }];
 
         Ok(S3Response::new(GetBucketAclOutput {
             grants: Some(grants),
             owner: Some(RUSTFS_OWNER.to_owned()),
-            ..Default::default()
         }))
     }
 
@@ -1589,7 +1584,7 @@ impl S3 for FS {
                 v.grants.is_some_and(|gs| {
                     //
                     !gs.is_empty()
-                        && gs.get(0).is_some_and(|g| {
+                        && gs.first().is_some_and(|g| {
                             g.to_owned()
                                 .permission
                                 .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)
@@ -1617,9 +1612,7 @@ impl S3 for FS {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("{}", e)));
         }
 
-        let mut grants = Vec::new();
-
-        grants.push(Grant {
+        let grants = vec![Grant {
             grantee: Some(Grantee {
                 type_: Type::from_static(Type::CANONICAL_USER),
                 display_name: None,
@@ -1628,7 +1621,7 @@ impl S3 for FS {
                 uri: None,
             }),
             permission: Some(Permission::from_static(Permission::FULL_CONTROL)),
-        });
+        }];
 
         Ok(S3Response::new(GetObjectAclOutput {
             grants: Some(grants),
@@ -1665,7 +1658,7 @@ impl S3 for FS {
                 v.grants.is_some_and(|gs| {
                     //
                     !gs.is_empty()
-                        && gs.get(0).is_some_and(|g| {
+                        && gs.first().is_some_and(|g| {
                             g.to_owned()
                                 .permission
                                 .is_some_and(|p| p.as_str() == Permission::FULL_CONTROL)