merge main

Cargo.lock

@@ -681,26 +681,6 @@ dependencies = [
  "cipher",
 ]
 
-[[package]]
-name = "darwin-libproc"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb90051930c9a0f09e585762152048e23ac74d20c10590ef7cf01c0343c3046"
-dependencies = [
- "darwin-libproc-sys",
- "libc",
- "memchr",
-]
-
-[[package]]
-name = "darwin-libproc-sys"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57cebb5bde66eecdd30ddc4b9cd208238b15db4982ccc72db59d699ea10867c1"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "deranged"
 version = "0.3.11"
@@ -711,17 +691,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "derive_more"
-version = "0.99.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -801,10 +770,9 @@ dependencies = [
  "madmin",
  "md-5",
  "netif",
- "nix 0.29.0",
+ "nix",
  "num",
  "num_cpus",
- "openssl",
  "path-absolutize",
  "path-clean",
  "pin-project-lite",
@@ -899,21 +867,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -1670,15 +1623,6 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
-[[package]]
-name = "mach2"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "madmin"
 version = "0.0.1"
@@ -1687,7 +1631,6 @@ dependencies = [
  "common",
  "humantime",
  "hyper",
- "psutil",
  "serde",
  "time",
  "tracing",
@@ -1779,17 +1722,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "nix"
-version = "0.24.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
-dependencies = [
- "bitflags 1.3.2",
- "cfg-if",
- "libc",
-]
-
 [[package]]
 name = "nix"
 version = "0.29.0"
@@ -1956,44 +1888,6 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
-[[package]]
-name = "openssl"
-version = "0.10.68"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5"
-dependencies = [
- "bitflags 2.6.0",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-macros",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "outref"
 version = "0.5.1"
@@ -2146,12 +2040,6 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
-[[package]]
-name = "platforms"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94"
-
 [[package]]
 name = "poly1305"
 version = "0.8.0"
@@ -2297,25 +2185,6 @@ dependencies = [
  "tower 0.5.1",
 ]
 
-[[package]]
-name = "psutil"
-version = "3.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e617cc9058daa5e1fe5a0d23ed745773a5ee354111dad1ec0235b0cc16b6730"
-dependencies = [
- "cfg-if",
- "darwin-libproc",
- "derive_more",
- "glob",
- "mach2",
- "nix 0.24.3",
- "num_cpus",
- "once_cell",
- "platforms",
- "thiserror 1.0.69",
- "unescape",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.37.1"
@@ -3393,12 +3262,6 @@ dependencies = [
  "tz-rs",
 ]
 
-[[package]]
-name = "unescape"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccb97dac3243214f8d8507998906ca3e2e0b900bf9bf4870477f125b82e68f6e"
-
 [[package]]
 name = "unicode-ident"
 version = "1.0.14"

common/lock/src/drwmutex.rs

@@ -210,7 +210,7 @@ impl DRWMutex {
 
     pub async fn un_lock(&mut self) {
         if self.write_locks.is_empty() || !self.is_locked() {
-            panic!("Trying to un_lock() while no lock() is active, write_locks: {:?}", self.write_locks)
+            warn!("Trying to un_lock() while no lock() is active, write_locks: {:?}", self.write_locks)
         }
 
         let tolerance = self.lockers.len() / 2;
@@ -231,7 +231,7 @@ impl DRWMutex {
 
     pub async fn un_r_lock(&mut self) {
         if self.read_locks.is_empty() || !self.is_r_locked() {
-            panic!("Trying to un_r_lock() while no r_lock() is active, read_locks: {:?}", self.read_locks)
+            warn!("Trying to un_r_lock() while no r_lock() is active, read_locks: {:?}", self.read_locks)
         }
 
         let tolerance = self.lockers.len() / 2;

common/protos/src/node.proto

@@ -2,6 +2,11 @@ syntax = "proto3";
 package node_service;
 
 /* -------------------------------------------------------------------- */
+message Error {
+  uint32 code = 1;
+  string error_info = 2;
+}
+
 message PingRequest {
   uint64 version = 1;
   bytes body = 2;
@@ -19,7 +24,7 @@ message HealBucketRequest {
 
 message HealBucketResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message ListBucketRequest {
@@ -29,7 +34,7 @@ message ListBucketRequest {
 message ListBucketResponse {
   bool success = 1;
   repeated string bucket_infos = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message MakeBucketRequest {
@@ -39,7 +44,7 @@ message MakeBucketRequest {
 
 message MakeBucketResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message GetBucketInfoRequest {
@@ -50,7 +55,7 @@ message GetBucketInfoRequest {
 message GetBucketInfoResponse {
   bool success = 1;
   string bucket_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message DeleteBucketRequest {
@@ -59,7 +64,7 @@ message DeleteBucketRequest {
 
 message DeleteBucketResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message ReadAllRequest {
@@ -71,7 +76,7 @@ message ReadAllRequest {
 message ReadAllResponse {
   bool success = 1;
   bytes data = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message WriteAllRequest {
@@ -83,7 +88,7 @@ message WriteAllRequest {
 
 message WriteAllResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message DeleteRequest {
@@ -95,7 +100,7 @@ message DeleteRequest {
 
 message DeleteResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message VerifyFileRequest {
@@ -108,7 +113,7 @@ message VerifyFileRequest {
 message VerifyFileResponse {
   bool success = 1;
   string check_parts_resp = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message CheckPartsRequest {
@@ -121,7 +126,7 @@ message CheckPartsRequest {
 message CheckPartsResponse {
   bool success = 1;
   string check_parts_resp = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message RenamePartRequst {
@@ -135,7 +140,7 @@ message RenamePartRequst {
 
 message RenamePartResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message RenameFileRequst {
@@ -148,7 +153,7 @@ message RenameFileRequst {
 
 message RenameFileResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message WriteRequest {
@@ -161,7 +166,7 @@ message WriteRequest {
 
 message WriteResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 // message AppendRequest {
@@ -173,7 +178,7 @@ message WriteResponse {
 // 
 // message AppendResponse {
 //   bool success = 1;
-//   optional string error_info = 2;
+//   optional Error error = 2;
 // }
 
 message ReadAtRequest {
@@ -188,7 +193,7 @@ message ReadAtResponse {
   bool success = 1;
   bytes data = 2;
   int64 read_size = 3;
-  optional string error_info = 4;
+  optional Error error = 4;
 }
 
 message ListDirRequest {
@@ -199,7 +204,7 @@ message ListDirRequest {
 message ListDirResponse {
   bool success = 1;
   repeated string volumes = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message WalkDirRequest {
@@ -225,7 +230,7 @@ message RenameDataRequest {
 message RenameDataResponse {
   bool success = 1;
   string rename_data_resp = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message MakeVolumesRequest {
@@ -235,7 +240,7 @@ message MakeVolumesRequest {
 
 message MakeVolumesResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message MakeVolumeRequest {
@@ -245,7 +250,7 @@ message MakeVolumeRequest {
 
 message MakeVolumeResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message ListVolumesRequest {
@@ -255,7 +260,7 @@ message ListVolumesRequest {
 message ListVolumesResponse {
   bool success = 1;
   repeated string volume_infos = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message StatVolumeRequest {
@@ -266,7 +271,7 @@ message StatVolumeRequest {
 message StatVolumeResponse {
   bool success = 1;
   string volume_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message DeletePathsRequest {
@@ -277,7 +282,7 @@ message DeletePathsRequest {
 
 message DeletePathsResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message UpdateMetadataRequest {
@@ -290,7 +295,7 @@ message UpdateMetadataRequest {
 
 message UpdateMetadataResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message WriteMetadataRequest {
@@ -302,7 +307,7 @@ message WriteMetadataRequest {
 
 message WriteMetadataResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message ReadVersionRequest {
@@ -316,7 +321,7 @@ message ReadVersionRequest {
 message ReadVersionResponse {
   bool success = 1;
   string file_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message ReadXLRequest {
@@ -329,7 +334,7 @@ message ReadXLRequest {
 message ReadXLResponse {
   bool success = 1;
   string raw_file_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message DeleteVersionRequest {
@@ -344,7 +349,7 @@ message DeleteVersionRequest {
 message DeleteVersionResponse {
   bool success = 1;
   string raw_file_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message DeleteVersionsRequest {
@@ -357,7 +362,7 @@ message DeleteVersionsRequest {
 message DeleteVersionsResponse {
   bool success = 1;
   repeated string errors = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message ReadMultipleRequest {
@@ -368,7 +373,7 @@ message ReadMultipleRequest {
 message ReadMultipleResponse {
   bool success = 1;
   repeated string read_multiple_resps = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message DeleteVolumeRequest {
@@ -378,7 +383,7 @@ message DeleteVolumeRequest {
 
 message DeleteVolumeResponse {
   bool success = 1;
-  optional string error_info = 2;
+  optional Error error = 2;
 }
 
 message DiskInfoRequest {
@@ -389,7 +394,7 @@ message DiskInfoRequest {
 message DiskInfoResponse {
   bool success = 1;
   string disk_info = 2;
-  optional string error_info = 3;
+  optional Error error = 3;
 }
 
 message NsScannerRequest {
@@ -402,7 +407,7 @@ message NsScannerResponse {
   bool success = 1;
   string update = 2;
   string data_usage_cache = 3;
-  optional string error_info = 4;
+  optional Error error = 4;
 }
 
 // lock api have same argument type

e2e_test/src/reliant/node_interact_test.rs

@@ -73,7 +73,7 @@ async fn make_volume() -> Result<(), Box<dyn Error>> {
     if response.success {
         println!("success");
     } else {
-        println!("failed: {:?}", response.error_info);
+        println!("failed: {:?}", response.error);
     }
     Ok(())
 }

ecstore/Cargo.toml

@@ -69,14 +69,11 @@ workers.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
 
-openssl = "0.10.68"
 nix = { version = "0.29.0", features = ["fs"] }
 
 [target.'cfg(windows)'.dependencies]
 
-
 winapi = "0.3.9"
-openssl = "0.10.66"
 
 
 [dev-dependencies]

ecstore/src/bucket/error.rs

@@ -29,3 +29,32 @@ impl BucketMetadataError {
         }
     }
 }
+
+impl BucketMetadataError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            BucketMetadataError::TaggingNotFound => 0x01,
+            BucketMetadataError::BucketPolicyNotFound => 0x02,
+            BucketMetadataError::BucketObjectLockConfigNotFound => 0x03,
+            BucketMetadataError::BucketLifecycleNotFound => 0x04,
+            BucketMetadataError::BucketSSEConfigNotFound => 0x05,
+            BucketMetadataError::BucketQuotaConfigNotFound => 0x06,
+            BucketMetadataError::BucketReplicationConfigNotFound => 0x07,
+            BucketMetadataError::BucketRemoteTargetNotFound => 0x08,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(BucketMetadataError::TaggingNotFound),
+            0x02 => Some(BucketMetadataError::BucketPolicyNotFound),
+            0x03 => Some(BucketMetadataError::BucketObjectLockConfigNotFound),
+            0x04 => Some(BucketMetadataError::BucketLifecycleNotFound),
+            0x05 => Some(BucketMetadataError::BucketSSEConfigNotFound),
+            0x06 => Some(BucketMetadataError::BucketQuotaConfigNotFound),
+            0x07 => Some(BucketMetadataError::BucketReplicationConfigNotFound),
+            0x08 => Some(BucketMetadataError::BucketRemoteTargetNotFound),
+            _ => None,
+        }
+    }
+}

ecstore/src/cache_value/metacache_set.rs

@@ -1,20 +1,18 @@
-use std::{future::Future, pin::Pin, sync::Arc};
+use crate::{
-
+    disk::error::{is_err_eof, is_err_file_not_found, is_err_volume_not_found, DiskError},
+    metacache::writer::MetacacheReader,
+};
+use crate::{
+    disk::{DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions},
+    error::{Error, Result},
+};
 use futures::future::join_all;
+use std::{future::Future, pin::Pin, sync::Arc};
 use tokio::{
     spawn,
     sync::{broadcast::Receiver as B_Receiver, RwLock},
 };
-use tracing::error;
+use tracing::{error, info};
-
-use crate::{
-    disk::{
-        error::{is_err_eof, is_err_file_not_found, is_err_volume_not_found, DiskError},
-        DiskAPI, DiskStore, MetaCacheEntries, MetaCacheEntry, WalkDirOptions,
-    },
-    error::{Error, Result},
-    metacache::writer::MetacacheReader,
-};
 
 type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type PartialFn = Box<dyn Fn(MetaCacheEntries, &[Option<Error>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
@@ -60,12 +58,14 @@ impl Clone for ListPathRawOptions {
 
 pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -> Result<()> {
     if opts.disks.is_empty() {
+        info!("list_path_raw 0 drives provided");
         return Err(Error::from_string("list_path_raw: 0 drives provided"));
     }
 
     let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), Error>>> = Vec::new();
     let mut readers = Vec::with_capacity(opts.disks.len());
     let fds = Arc::new(RwLock::new(opts.fallback_disks.clone()));
+
     for disk in opts.disks.iter() {
         let opdisk = disk.clone();
         let opts_clone = opts.clone();

ecstore/src/config/error.rs

@@ -1,6 +1,6 @@
 use crate::{disk, error::Error, store_err::is_err_object_not_found};
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub enum ConfigError {
     #[error("config not found")]
     NotFound,
@@ -15,6 +15,22 @@ impl ConfigError {
         matches!(self, Self::NotFound)
     }
 }
+
+impl ConfigError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            ConfigError::NotFound => 0x01,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(Self::NotFound),
+            _ => None,
+        }
+    }
+}
+
 pub fn is_not_found(err: &Error) -> bool {
     if let Some(e) = err.downcast_ref::<ConfigError>() {
         ConfigError::is_not_found(e)

ecstore/src/disk/error.rs

@@ -2,6 +2,7 @@ use std::io::{self, ErrorKind};
 
 use tracing::error;
 
+use crate::utils::ERROR_TYPE_MASK;
 use crate::{
     error::{Error, Result},
     quorum::CheckErrorFn,
@@ -167,6 +168,87 @@ impl DiskError {
     }
 }
 
+impl DiskError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            DiskError::MaxVersionsExceeded => 0x01,
+            DiskError::Unexpected => 0x02,
+            DiskError::CorruptedFormat => 0x03,
+            DiskError::CorruptedBackend => 0x04,
+            DiskError::UnformattedDisk => 0x05,
+            DiskError::InconsistentDisk => 0x06,
+            DiskError::UnsupportedDisk => 0x07,
+            DiskError::DiskFull => 0x08,
+            DiskError::DiskNotDir => 0x09,
+            DiskError::DiskNotFound => 0x0A,
+            DiskError::DiskOngoingReq => 0x0B,
+            DiskError::DriveIsRoot => 0x0C,
+            DiskError::FaultyRemoteDisk => 0x0D,
+            DiskError::FaultyDisk => 0x0E,
+            DiskError::DiskAccessDenied => 0x0F,
+            DiskError::FileNotFound => 0x10,
+            DiskError::FileVersionNotFound => 0x11,
+            DiskError::TooManyOpenFiles => 0x12,
+            DiskError::FileNameTooLong => 0x13,
+            DiskError::VolumeExists => 0x14,
+            DiskError::IsNotRegular => 0x15,
+            DiskError::PathNotFound => 0x16,
+            DiskError::VolumeNotFound => 0x17,
+            DiskError::VolumeNotEmpty => 0x18,
+            DiskError::VolumeAccessDenied => 0x19,
+            DiskError::FileAccessDenied => 0x1A,
+            DiskError::FileCorrupt => 0x1B,
+            DiskError::BitrotHashAlgoInvalid => 0x1C,
+            DiskError::CrossDeviceLink => 0x1D,
+            DiskError::LessData => 0x1E,
+            DiskError::MoreData => 0x1F,
+            DiskError::OutdatedXLMeta => 0x20,
+            DiskError::PartMissingOrCorrupt => 0x21,
+            DiskError::NoHealRequired => 0x22,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error & ERROR_TYPE_MASK {
+            0x01 => Some(DiskError::MaxVersionsExceeded),
+            0x02 => Some(DiskError::Unexpected),
+            0x03 => Some(DiskError::CorruptedFormat),
+            0x04 => Some(DiskError::CorruptedBackend),
+            0x05 => Some(DiskError::UnformattedDisk),
+            0x06 => Some(DiskError::InconsistentDisk),
+            0x07 => Some(DiskError::UnsupportedDisk),
+            0x08 => Some(DiskError::DiskFull),
+            0x09 => Some(DiskError::DiskNotDir),
+            0x0A => Some(DiskError::DiskNotFound),
+            0x0B => Some(DiskError::DiskOngoingReq),
+            0x0C => Some(DiskError::DriveIsRoot),
+            0x0D => Some(DiskError::FaultyRemoteDisk),
+            0x0E => Some(DiskError::FaultyDisk),
+            0x0F => Some(DiskError::DiskAccessDenied),
+            0x10 => Some(DiskError::FileNotFound),
+            0x11 => Some(DiskError::FileVersionNotFound),
+            0x12 => Some(DiskError::TooManyOpenFiles),
+            0x13 => Some(DiskError::FileNameTooLong),
+            0x14 => Some(DiskError::VolumeExists),
+            0x15 => Some(DiskError::IsNotRegular),
+            0x16 => Some(DiskError::PathNotFound),
+            0x17 => Some(DiskError::VolumeNotFound),
+            0x18 => Some(DiskError::VolumeNotEmpty),
+            0x19 => Some(DiskError::VolumeAccessDenied),
+            0x1A => Some(DiskError::FileAccessDenied),
+            0x1B => Some(DiskError::FileCorrupt),
+            0x1C => Some(DiskError::BitrotHashAlgoInvalid),
+            0x1D => Some(DiskError::CrossDeviceLink),
+            0x1E => Some(DiskError::LessData),
+            0x1F => Some(DiskError::MoreData),
+            0x20 => Some(DiskError::OutdatedXLMeta),
+            0x21 => Some(DiskError::PartMissingOrCorrupt),
+            0x22 => Some(DiskError::NoHealRequired),
+            _ => None,
+        }
+    }
+}
+
 impl PartialEq for DiskError {
     fn eq(&self, other: &Self) -> bool {
         core::mem::discriminant(self) == core::mem::discriminant(other)

ecstore/src/disk/local.rs

@@ -50,6 +50,7 @@ use path_absolutize::Absolutize;
 use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::io::Cursor;
+#[cfg(target_family = "unix")]
 use std::os::unix::fs::MetadataExt;
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
@@ -1030,7 +1031,7 @@ impl DiskAPI for LocalDisk {
         self.endpoint.host_port()
     }
     async fn is_online(&self) -> bool {
-        true
+        self.check_format_json().await.is_ok()
     }
 
     fn endpoint(&self) -> Endpoint {
@@ -1231,7 +1232,7 @@ impl DiskAPI for LocalDisk {
                         resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
                         continue;
                     }
-                    if (st.size() as usize) < fi.erasure.shard_file_size(part.size) {
+                    if (st.len() as usize) < fi.erasure.shard_file_size(part.size) {
                         resp.results[i] = CHECK_PART_FILE_CORRUPT;
                         continue;
                     }

ecstore/src/disk/mod.rs

@@ -14,6 +14,7 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json";
 pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
+use crate::utils::proto_err_to_err;
 use crate::{
     bucket::{metadata_sys::get_versioning_config, versioning::VersioningApi},
     erasure::Writer,
@@ -1393,9 +1394,11 @@ impl Writer for RemoteFileWriter {
             if resp.success {
                 info!("write stream success");
             } else {
-                let error_info = resp.error_info.unwrap_or("".to_string());
+                return if let Some(err) = &resp.error {
-                info!("write stream failed: {}", error_info);
+                    Err(proto_err_to_err(err))
-                return Err(Error::from_string(error_info));
+                } else {
+                    Err(Error::from_string(""))
+                };
             }
         } else {
             let error_info = "can not get response";
@@ -1571,9 +1574,11 @@ impl Reader for RemoteFileReader {
 
                 Ok(resp.read_size.try_into().unwrap())
             } else {
-                let error_info = resp.error_info.unwrap_or("".to_string());
+                return if let Some(err) = &resp.error {
-                info!("read at stream failed: {}", error_info);
+                    Err(proto_err_to_err(err))
-                Err(Error::from_string(error_info))
+                } else {
+                    Err(Error::from_string(""))
+                };
             }
         } else {
             let error_info = "can not get response";

ecstore/src/disk/remote.rs

@@ -26,6 +26,7 @@ use super::{
     FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter,
     RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
 };
+use crate::utils::proto_err_to_err;
 use crate::{
     disk::error::DiskError,
     error::{Error, Result},
@@ -167,7 +168,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.write_all(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -189,7 +194,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.delete(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -211,7 +220,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.verify_file(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
@@ -235,7 +248,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.check_parts(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let check_parts_resp = serde_json::from_str::<CheckPartsResp>(&response.check_parts_resp)?;
@@ -260,7 +277,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.rename_part(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -281,7 +302,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.rename_file(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -347,7 +372,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.list_dir(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(response.volumes)
@@ -412,7 +441,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.rename_data(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let rename_data_resp = serde_json::from_str::<RenameDataResp>(&response.rename_data_resp)?;
@@ -433,7 +466,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.make_volumes(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -452,7 +489,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.make_volume(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -470,7 +511,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.list_volumes(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let infos = response
@@ -495,7 +540,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.stat_volume(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let volume_info = serde_json::from_str::<VolumeInfo>(&response.volume_info)?;
@@ -518,7 +567,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.delete_paths(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -542,7 +595,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.update_metadata(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -564,7 +621,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.write_metadata(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -594,7 +655,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.read_version(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let file_info = serde_json::from_str::<FileInfo>(&response.file_info)?;
@@ -617,7 +682,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.read_xl(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
@@ -651,7 +720,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.delete_version(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         // let raw_file_info = serde_json::from_str::<RawFileInfo>(&response.raw_file_info)?;
@@ -682,10 +755,11 @@ impl DiskAPI for RemoteDisk {
 
         let response = client.delete_versions(request).await?.into_inner();
         if !response.success {
-            return Err(Error::from_string(format!(
+            return if let Some(err) = &response.error {
-                "delete versions remote err: {}",
+                Err(proto_err_to_err(err))
-                response.error_info.unwrap_or("None".to_string())
+            } else {
-            )));
+                Err(Error::from_string(""))
+            };
         }
         let errors = response
             .errors
@@ -716,7 +790,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.read_multiple(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let read_multiple_resps = response
@@ -741,7 +819,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.delete_volume(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -761,7 +843,11 @@ impl DiskAPI for RemoteDisk {
         let response = client.disk_info(request).await?.into_inner();
 
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or("".to_string())));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;

ecstore/src/erasure.rs

@@ -212,9 +212,9 @@ impl Erasure {
         offset: usize,
         length: usize,
         total_length: usize,
-    ) -> Result<usize> {
+    ) -> (usize, Option<Error>) {
         if length == 0 {
-            return Ok(0);
+            return (0, None);
         }
 
         let mut reader = ShardReader::new(readers, self, offset, total_length);
@@ -247,15 +247,24 @@ impl Erasure {
 
             // debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length);
 
-            let mut bufs = reader.read().await?;
+            let mut bufs = match reader.read().await {
+                Ok(bufs) => bufs,
+                Err(err) => return (bytes_writed, Some(err)),
+            };
 
             if self.parity_shards > 0 {
-                self.decode_data(&mut bufs)?;
+                if let Err(err) = self.decode_data(&mut bufs) {
+                    return (bytes_writed, Some(err));
+                }
             }
 
-            let writed_n = self
+            let writed_n = match self
                 .write_data_blocks(writer, bufs, self.data_shards, block_offset, block_length)
-                .await?;
+                .await
+            {
+                Ok(n) => n,
+                Err(err) => return (bytes_writed, Some(err)),
+            };
 
             bytes_writed += writed_n;
 
@@ -264,10 +273,10 @@ impl Erasure {
 
         if bytes_writed != length {
             // debug!("bytes_writed != length: {} != {} ", bytes_writed, length);
-            return Err(Error::msg("erasure decode less data"));
+            return (bytes_writed, Some(Error::msg("erasure decode less data")));
         }
 
-        Ok(bytes_writed)
+        (bytes_writed, None)
     }
 
     async fn write_data_blocks(

ecstore/src/global.rs

@@ -6,6 +6,7 @@ use std::{
 use tokio::sync::RwLock;
 use uuid::Uuid;
 
+use crate::heal::mrf::MRFState;
 use crate::{
     disk::DiskStore,
     endpoints::{EndpointServerPools, PoolEndpoints, SetupType},
@@ -34,6 +35,7 @@ lazy_static! {
     pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
     pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
     pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
+    pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
     static ref globalDeploymentIDPtr: RwLock<Uuid> = RwLock::new(Uuid::nil());
 }
 

ecstore/src/heal/background_heal_ops.rs

@@ -1,6 +1,7 @@
 use madmin::heal_commands::HealResultItem;
 use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration};
 use tokio::{
+    spawn,
     sync::{
         mpsc::{self, Receiver, Sender},
         RwLock,
@@ -14,7 +15,10 @@ use super::{
     heal_commands::HealOpts,
     heal_ops::{new_bg_heal_sequence, HealSequence},
 };
+use crate::global::GLOBAL_MRFState;
 use crate::heal::error::ERR_RETRY_HEALING;
+use crate::heal::heal_commands::{HealScanMode, HEAL_ITEM_BUCKET};
+use crate::heal::heal_ops::{HealSource, BG_HEALING_UUID};
 use crate::{
     config::RUSTFS_CONFIG_PREFIX,
     disk::{endpoint::Endpoint, error::DiskError, DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
@@ -36,24 +40,26 @@ pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10)
 
 pub async fn init_auto_heal() {
     init_background_healing().await;
-    if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
+    let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string());
-        if v == "on" {
+    if v == "on" {
-            info!("start monitor local disks and heal");
+        info!("start monitor local disks and heal");
-            GLOBAL_BackgroundHealState
+        GLOBAL_BackgroundHealState
-                .push_heal_local_disks(&get_local_disks_to_heal().await)
+            .push_heal_local_disks(&get_local_disks_to_heal().await)
-                .await;
+            .await;
-            tokio::spawn(async {
+        spawn(async {
-                monitor_local_disks_and_heal().await;
+            monitor_local_disks_and_heal().await;
-            });
+        });
-        }
     }
+    spawn(async {
+        GLOBAL_MRFState.heal_routine().await;
+    });
 }
 
 async fn init_background_healing() {
     let bg_seq = Arc::new(new_bg_heal_sequence());
     for _ in 0..GLOBAL_BackgroundHealRoutine.workers {
         let bg_seq_clone = bg_seq.clone();
-        tokio::spawn(async {
+        spawn(async {
             GLOBAL_BackgroundHealRoutine.add_worker(bg_seq_clone).await;
         });
     }
@@ -66,12 +72,14 @@ pub async fn get_local_disks_to_heal() -> Vec<Endpoint> {
         if let Some(disk) = disk {
             if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
                 if let Some(DiskError::UnformattedDisk) = err.downcast_ref() {
+                    info!("get_local_disks_to_heal, disk is unformatted: {}", err);
                     disks_to_heal.push(disk.endpoint());
                 }
             }
             let h = disk.healing().await;
             if let Some(h) = h {
                 if !h.finished {
+                    info!("get_local_disks_to_heal, disk healing not finished");
                     disks_to_heal.push(disk.endpoint());
                 }
             }
@@ -92,13 +100,17 @@ async fn monitor_local_disks_and_heal() {
         interval.tick().await;
         let heal_disks = GLOBAL_BackgroundHealState.get_heal_local_disk_endpoints().await;
         if heal_disks.is_empty() {
+            info!("heal local disks is empty");
             interval.reset();
             continue;
         }
 
+        info!("heal local disks: {:?}", heal_disks);
+
         let store = new_object_layer_fn().expect("errServerNotInitialized");
-        if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
+        if let (result, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
-            if let Some(DiskError::NoHealRequired) = err.downcast_ref() {
+            error!("heal local disk format error: {}", err);
+            if let Some(DiskError::NoHealRequired) = err.downcast_ref::<DiskError>() {
             } else {
                 info!("heal format err: {}", err.to_string());
                 interval.reset();
@@ -108,7 +120,7 @@ async fn monitor_local_disks_and_heal() {
 
         for disk in heal_disks.into_ref().iter() {
             let disk_clone = disk.clone();
-            tokio::spawn(async move {
+            spawn(async move {
                 GLOBAL_BackgroundHealState
                     .set_disk_healing_status(disk_clone.clone(), true)
                     .await;
@@ -420,3 +432,30 @@ async fn heal_disk_format(opts: HealOpts) -> Result<(HealResultItem, Option<Erro
     }
     Ok((res, err))
 }
+
+pub(crate) async fn heal_bucket(bucket: &str) -> Result<()> {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if ok {
+        // bg_seq must be Some when ok is true
+        return bg_seq
+            .unwrap()
+            .queue_heal_task(
+                HealSource {
+                    bucket: bucket.to_string(),
+                    ..Default::default()
+                },
+                HEAL_ITEM_BUCKET.to_string(),
+            )
+            .await;
+    }
+    Ok(())
+}
+
+pub(crate) async fn heal_object(bucket: &str, object: &str, version_id: &str, scan_mode: HealScanMode) -> Result<()> {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if ok {
+        // bg_seq must be Some when ok is true
+        return HealSequence::heal_object(bg_seq.unwrap(), bucket, object, version_id, scan_mode).await;
+    }
+    Ok(())
+}

ecstore/src/heal/heal_commands.rs

@@ -388,6 +388,7 @@ pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTra
                 )));
             }
             healing_tracker.id = disk_id;
+            healing_tracker.disk = Some(disk.clone());
             Ok(healing_tracker)
         } else {
             Err(Error::from_string("loadHealingTracker: disk not have id"))
@@ -400,7 +401,10 @@ pub async fn load_healing_tracker(disk: &Option<DiskStore>) -> Result<HealingTra
 pub async fn init_healing_tracker(disk: DiskStore, heal_id: &str) -> Result<HealingTracker> {
     let disk_location = disk.get_disk_location();
     Ok(HealingTracker {
-        id: disk.get_disk_id().await?.map_or("".to_string(), |id| id.to_string()),
+        id: disk
+            .get_disk_id()
+            .await
+            .map_or("".to_string(), |id| id.map_or("".to_string(), |id| id.to_string())),
         heal_id: heal_id.to_string(),
         path: disk.to_string(),
         endpoint: disk.endpoint().to_string(),

ecstore/src/heal/mod.rs

@@ -6,3 +6,4 @@ pub mod data_usage_cache;
 pub mod error;
 pub mod heal_commands;
 pub mod heal_ops;
+pub mod mrf;

ecstore/src/heal/mrf.rs

@@ -0,0 +1,124 @@
+use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
+use crate::heal::background_heal_ops::{heal_bucket, heal_object};
+use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_NORMAL_SCAN};
+use crate::utils::path::SLASH_SEPARATOR;
+use chrono::{DateTime, TimeDelta, Utc};
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::ops::Sub;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use tokio::sync::mpsc::{Receiver, Sender};
+use tokio::sync::RwLock;
+use tokio::time::sleep;
+use tracing::error;
+use uuid::Uuid;
+
+pub const MRF_OPS_QUEUE_SIZE: u64 = 100000;
+pub const HEAL_DIR: &'static str = ".heal";
+pub const HEAL_MRFMETA_FORMAT: u64 = 1;
+pub const HEAL_MRFMETA_VERSION_V1: u64 = 1;
+
+lazy_static! {
+    pub static ref HEAL_MRF_DIR: String =
+        format!("{}{}{}{}{}", BUCKET_META_PREFIX, SLASH_SEPARATOR, HEAL_DIR, SLASH_SEPARATOR, "mrf");
+    static ref PATTERNS: Vec<Regex> = vec![
+        Regex::new(r"^buckets/.*/.metacache/.*").unwrap(),
+        Regex::new(r"^tmp/.*").unwrap(),
+        Regex::new(r"^multipart/.*").unwrap(),
+        Regex::new(r"^tmp-old/.*").unwrap(),
+    ];
+}
+
+#[derive(Default)]
+pub struct PartialOperation {
+    pub bucket: String,
+    pub object: String,
+    pub version_id: Option<String>,
+    pub versions: Vec<u8>,
+    pub set_index: usize,
+    pub pool_index: usize,
+    pub queued: DateTime<Utc>,
+    pub bitrot_scan: bool,
+}
+
+pub struct MRFState {
+    tx: Sender<PartialOperation>,
+    rx: RwLock<Receiver<PartialOperation>>,
+    closed: AtomicBool,
+    closing: AtomicBool,
+}
+
+impl MRFState {
+    pub fn new() -> MRFState {
+        let (tx, rx) = tokio::sync::mpsc::channel(MRF_OPS_QUEUE_SIZE as usize);
+        MRFState {
+            tx,
+            rx: RwLock::new(rx),
+            closed: Default::default(),
+            closing: Default::default(),
+        }
+    }
+
+    pub async fn add_partial(&self, op: PartialOperation) {
+        if self.closed.load(Ordering::SeqCst) || self.closing.load(Ordering::SeqCst) {
+            return;
+        }
+        let _ = self.tx.send(op).await;
+    }
+
+    pub async fn heal_routine(&self) {
+        loop {
+            // rx used only there,
+            if let Some(op) = self.rx.write().await.recv().await {
+                if op.bucket == RUSTFS_META_BUCKET {
+                    for pattern in &*PATTERNS {
+                        if pattern.is_match(&op.object) {
+                            return;
+                        }
+                    }
+                }
+
+                let now = Utc::now();
+                if now.sub(op.queued).num_seconds() < 1 {
+                    sleep(Duration::from_secs(1)).await;
+                }
+
+                let scan_mode = if op.bitrot_scan { HEAL_DEEP_SCAN } else { HEAL_NORMAL_SCAN };
+                if op.object.is_empty() {
+                    if let Err(err) = heal_bucket(&op.bucket).await {
+                        error!("heal bucket failed, bucket: {}, err: {:?}", op.bucket, err);
+                    }
+                } else {
+                    if op.versions.is_empty() {
+                        if let Err(err) =
+                            heal_object(&op.bucket, &op.object, &op.version_id.clone().unwrap_or_default(), scan_mode).await
+                        {
+                            error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err);
+                        }
+                    } else {
+                        let vers = op.versions.len() / 16;
+                        if vers > 0 {
+                            for i in 0..vers {
+                                let start = i * 16;
+                                let end = start + 16;
+                                if let Err(err) = heal_object(
+                                    &op.bucket,
+                                    &op.object,
+                                    &Uuid::from_slice(&op.versions[start..end]).expect("").to_string(),
+                                    scan_mode,
+                                )
+                                .await
+                                {
+                                    error!("heal object failed, bucket: {}, object: {}, err: {:?}", op.bucket, op.object, err);
+                                }
+                            }
+                        }
+                    }
+                }
+            } else {
+                return;
+            }
+        }
+    }
+}

ecstore/src/peer.rs

@@ -9,7 +9,7 @@ use regex::Regex;
 use std::{collections::HashMap, fmt::Debug, sync::Arc};
 use tokio::sync::RwLock;
 use tonic::Request;
-use tracing::{info, warn};
+use tracing::info;
 
 use crate::disk::error::is_all_buckets_not_found;
 use crate::disk::{DiskAPI, DiskStore};
@@ -20,6 +20,7 @@ use crate::heal::heal_commands::{
 use crate::heal::heal_ops::RUESTFS_RESERVED_BUCKET;
 use crate::quorum::{bucket_op_ignored_errs, reduce_write_quorum_errs};
 use crate::store::all_local_disk;
+use crate::utils::proto_err_to_err;
 use crate::utils::wildcard::is_rustfs_meta_bucket_name;
 use crate::{
     disk::{self, error::DiskError, VolumeInfo},
@@ -523,7 +524,11 @@ impl PeerS3Client for RemotePeerS3Client {
         });
         let response = client.heal_bucket(request).await?.into_inner();
         if !response.success {
-            return Err(Error::from_string(response.error_info.unwrap_or_default()));
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(HealResultItem {
@@ -541,6 +546,13 @@ impl PeerS3Client for RemotePeerS3Client {
             .map_err(|err| Error::from_string(format!("can not get client, err: {}", err)))?;
         let request = Request::new(ListBucketRequest { options });
         let response = client.list_bucket(request).await?.into_inner();
+        if !response.success {
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
+        }
         let bucket_infos = response
             .bucket_infos
             .into_iter()
@@ -562,7 +574,11 @@ impl PeerS3Client for RemotePeerS3Client {
 
         // TODO: deal with error
         if !response.success {
-            warn!("make bucket error: {:?}", response.error_info);
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
         }
 
         Ok(())
@@ -577,6 +593,13 @@ impl PeerS3Client for RemotePeerS3Client {
             options,
         });
         let response = client.get_bucket_info(request).await?.into_inner();
+        if !response.success {
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
+        }
         let bucket_info = serde_json::from_str::<BucketInfo>(&response.bucket_info)?;
 
         Ok(bucket_info)
@@ -590,7 +613,14 @@ impl PeerS3Client for RemotePeerS3Client {
         let request = Request::new(DeleteBucketRequest {
             bucket: bucket.to_string(),
         });
-        let _response = client.delete_bucket(request).await?.into_inner();
+        let response = client.delete_bucket(request).await?.into_inner();
+        if !response.success {
+            return if let Some(err) = &response.error {
+                Err(proto_err_to_err(err))
+            } else {
+                Err(Error::from_string(""))
+            };
+        }
 
         Ok(())
     }

ecstore/src/peer_rest_client.rs

@@ -53,7 +53,7 @@ impl PeerRestClient {
         let eps = eps.clone();
         let hosts = eps.hosts_sorted();
         let mut remote = vec![None; hosts.len()];
-        let mut all = Vec::with_capacity(hosts.len());
+        let mut all = vec![None; hosts.len()];
         for (i, hs_host) in hosts.iter().enumerate() {
             if let Some(host) = hs_host {
                 if let Some(grid_host) = eps.find_grid_hosts_from_peer(host) {

ecstore/src/quorum.rs

@@ -1,13 +1,13 @@
+use crate::config::error::ConfigError;
 use crate::{disk::error::DiskError, error::Error};
 use std::{collections::HashMap, fmt::Debug};
-
 // pub type CheckErrorFn = fn(e: &Error) -> bool;
 
 pub trait CheckErrorFn: Debug + Send + Sync + 'static {
     fn is(&self, e: &Error) -> bool;
 }
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub enum QuorumError {
     #[error("Read quorum not met")]
     Read,
@@ -15,6 +15,23 @@ pub enum QuorumError {
     Write,
 }
 
+impl QuorumError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            QuorumError::Read => 0x01,
+            QuorumError::Write => 0x02,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(QuorumError::Read),
+            0x02 => Some(QuorumError::Write),
+            _ => None,
+        }
+    }
+}
+
 pub fn base_ignored_errs() -> Vec<Box<dyn CheckErrorFn>> {
     vec![
         Box::new(DiskError::DiskNotFound),

ecstore/src/set_disk.rs

@@ -6,7 +6,10 @@ use std::{
     time::Duration,
 };
 
+use crate::config::error::is_not_found;
+use crate::global::GLOBAL_MRFState;
 use crate::heal::heal_ops::{HealEntryFn, HealSequence};
+use crate::heal::mrf::PartialOperation;
 use crate::{
     bitrot::{bitrot_verify, close_bitrot_writers, new_bitrot_filereader, new_bitrot_filewriter, BitrotFileWriter},
     cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
@@ -46,7 +49,7 @@ use crate::{
     store_init::{load_format_erasure, ErasureError},
     utils::{
         self,
-        crypto::{base64_decode, base64_encode, hex, sha256},
+        crypto::{base64_decode, base64_encode, hex},
         path::{encode_dir_object, has_suffix, SLASH_SEPARATOR},
     },
     xhttp,
@@ -56,6 +59,7 @@ use crate::{
     heal::data_scanner::{globalHealConfig, HEAL_DELETE_DANGLING},
     store_api::ListObjectVersionsInfo,
 };
+use chrono::Utc;
 use futures::future::join_all;
 use glob::Pattern;
 use http::HeaderMap;
@@ -619,7 +623,9 @@ impl SetDisks {
 
     fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
         let path = format!("{}/{}", bucket, object);
-        hex(sha256(path.as_bytes()).as_ref())
+        let mut hasher = Sha256::new();
+        hasher.update(path);
+        hex(hasher.finalize())
     }
 
     fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
@@ -1318,6 +1324,7 @@ impl SetDisks {
         for (i, opdisk) in disks.iter().enumerate() {
             if let Some(disk) = opdisk {
                 if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
+                    info!("Disk {:?} is online", disk);
                     continue;
                 }
 
@@ -1325,6 +1332,7 @@ impl SetDisks {
             }
 
             if let Some(endpoint) = self.set_endpoints.get(i) {
+                info!("will renew disk, opdisk: {:?}", opdisk);
                 self.renew_disk(endpoint).await;
             }
         }
@@ -1338,12 +1346,20 @@ impl SetDisks {
             Err(e) => {
                 warn!("connect_endpoint err {:?}", &e);
                 if ep.is_local && DiskError::UnformattedDisk.is(&e) {
-                    // TODO: pushHealLocalDisks
+                    GLOBAL_BackgroundHealState.push_heal_local_disks(&[ep.clone()]).await;
                 }
                 return;
             }
         };
 
+        if new_disk.is_local() {
+            if let Some(h) = new_disk.healing().await {
+                if !h.finished {
+                    GLOBAL_BackgroundHealState.push_heal_local_disks(&[new_disk.endpoint()]).await;
+                }
+            }
+        }
+
         let (set_idx, disk_idx) = match self.find_disk_index(&fm) {
             Ok(res) => res,
             Err(e) => {
@@ -1706,6 +1722,19 @@ impl SetDisks {
         let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize);
 
         let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?;
+        if errs.iter().any(|err| err.is_some()) {
+            GLOBAL_MRFState
+                .add_partial(PartialOperation {
+                    bucket: fi.volume.to_string(),
+                    object: fi.name.to_string(),
+                    queued: Utc::now(),
+                    version_id: fi.version_id.map(|v| v.to_string()),
+                    set_index: self.set_index,
+                    pool_index: self.pool_index,
+                    ..Default::default()
+                })
+                .await;
+        }
         // debug!("get_object_fileinfo pick fi {:?}", &fi);
 
         // let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
@@ -1724,6 +1753,8 @@ impl SetDisks {
         fi: FileInfo,
         files: Vec<FileInfo>,
         disks: &[Option<DiskStore>],
+        set_index: usize,
+        pool_index: usize,
     ) -> Result<()> {
         let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi);
 
@@ -1809,10 +1840,34 @@ impl SetDisks {
             //     "read part {} part_offset {},part_length {},part_size {}  ",
             //     part_number, part_offset, part_length, part_size
             // );
-            let _n = erasure
+            let (written, mut err) = erasure
                 .decode(writer, readers, part_offset as usize, part_length, part_size)
-                .await?;
+                .await;
-
+            if let Some(e) = err.as_ref() {
+                if written == part_length {
+                    match e.downcast_ref::<DiskError>() {
+                        Some(DiskError::FileNotFound) | Some(DiskError::FileCorrupt) => {
+                            GLOBAL_MRFState
+                                .add_partial(PartialOperation {
+                                    bucket: bucket.to_string(),
+                                    object: object.to_string(),
+                                    queued: Utc::now(),
+                                    version_id: fi.version_id.map(|v| v.to_string()),
+                                    set_index,
+                                    pool_index,
+                                    bitrot_scan: !is_not_found(e),
+                                    ..Default::default()
+                                })
+                                .await;
+                            err = None;
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            if let Some(err) = err {
+                return Err(err);
+            }
             // debug!("ec decode {} writed size {}", part_number, n);
 
             total_readed += part_length as i64;
@@ -2941,7 +2996,6 @@ impl SetDisks {
         info!("ns_scanner start");
         let _ = join_all(futures).await;
         drop(buckets_results_tx);
-        info!("1");
         let _ = task.await;
         info!("ns_scanner completed");
         Ok(())
@@ -3070,6 +3124,7 @@ impl SetDisks {
         let mut ret_err = None;
         for bucket in buckets.iter() {
             if tracker.read().await.is_healed(bucket).await {
+                info!("bucket{} was healed", bucket);
                 continue;
             }
 
@@ -3190,6 +3245,7 @@ impl SetDisks {
             let bg_seq_clone = bg_seq.clone();
             let send_clone = send.clone();
             let heal_entry = Arc::new(move |bucket: String, entry: MetaCacheEntry| {
+                info!("heal entry, bucket: {}, entry: {:?}", bucket, entry);
                 let jt_clone = jt_clone.clone();
                 let self_clone = self_clone.clone();
                 let started = started_clone;
@@ -3469,8 +3525,14 @@ impl ObjectIO for SetDisks {
         // let disks = disks.clone();
         let bucket = String::from(bucket);
         let object = String::from(object);
+        let set_index = self.set_index.clone();
+        let pool_index = self.pool_index.clone();
         tokio::spawn(async move {
-            if let Err(e) = Self::get_object_with_fileinfo(&bucket, &object, offset, length, &mut wd, fi, files, &disks).await {
+            if let Err(e) = Self::get_object_with_fileinfo(
+                &bucket, &object, offset, length, &mut wd, fi, files, &disks, set_index, pool_index,
+            )
+            .await
+            {
                 error!("get_object_with_fileinfo err {:?}", e);
             };
         });
@@ -4529,7 +4591,7 @@ impl StorageAPI for SetDisks {
             }
         }
 
-        let (online_disks, _, op_old_dir) = Self::rename_data(
+        let (online_disks, versions, op_old_dir) = Self::rename_data(
             &shuffle_disks,
             RUSTFS_META_MULTIPART_BUCKET,
             &upload_id_path,
@@ -4558,6 +4620,19 @@ impl StorageAPI for SetDisks {
             self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
                 .await?;
         }
+        if let Some(versions) = versions {
+            GLOBAL_MRFState
+                .add_partial(PartialOperation {
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    queued: Utc::now(),
+                    versions,
+                    set_index: self.set_index,
+                    pool_index: self.pool_index,
+                    ..Default::default()
+                })
+                .await;
+        }
 
         let _ = self.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await;
 

ecstore/src/sets.rs

@@ -589,6 +589,7 @@ impl StorageAPI for Sets {
         .await;
         let (formats, errs) = load_format_erasure_all(&disks, true).await;
         if let Err(err) = check_format_erasure_values(&formats, self.set_drive_count) {
+            info!("failed to check formats erasure values: {}", err);
             return Ok((HealResultItem::default(), Some(err)));
         }
         let ref_format = match get_format_erasure_in_quorum(&formats) {
@@ -611,12 +612,14 @@ impl StorageAPI for Sets {
             res.after.drives.push(v.clone());
         }
         if DiskError::UnformattedDisk.count_errs(&errs) == 0 {
+            info!("disk formats success, NoHealRequired, errs: {:?}", errs);
             return Ok((res, Some(Error::new(DiskError::NoHealRequired))));
         }
 
-        if !self.format.eq(&ref_format) {
+        // if !self.format.eq(&ref_format) {
-            return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
+        //     info!("format ({:?}) not eq ref_format ({:?})", self.format, ref_format);
-        }
+        //     return Ok((res, Some(Error::new(DiskError::CorruptedFormat))));
+        // }
 
         let format_op_id = Uuid::new_v4().to_string();
         let (new_format_sets, _) = new_heal_format_sets(&ref_format, self.set_count, self.set_drive_count, &formats, &errs);

ecstore/src/store.rs

@@ -669,8 +669,10 @@ impl ECStore {
         want_cycle: usize,
         heal_scan_mode: HealScanMode,
     ) -> Result<()> {
+        info!("ns_scanner updates - {}", want_cycle);
         let all_buckets = self.list_bucket(&BucketOptions::default()).await?;
         if all_buckets.is_empty() {
+            info!("No buckets found");
             let _ = updates.send(DataUsageInfo::default()).await;
             return Ok(());
         }
@@ -1876,6 +1878,7 @@ impl StorageAPI for ECStore {
         counts
     }
     async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
+        info!("heal_format");
         let mut r = HealResultItem {
             heal_item_type: HEAL_ITEM_METADATA.to_string(),
             detail: "disk-format".to_string(),
@@ -1901,8 +1904,10 @@ impl StorageAPI for ECStore {
             r.after.drives.append(&mut result.after.drives);
         }
         if count_no_heal == self.pools.len() {
+            info!("heal format success, NoHealRequired");
             return Ok((r, Some(Error::new(DiskError::NoHealRequired))));
         }
+        info!("heal format success result: {:?}", r);
         Ok((r, None))
     }
 

ecstore/src/store_err.rs

@@ -79,6 +79,73 @@ pub enum StorageError {
     DecommissionNotStarted,
 }
 
+impl StorageError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            StorageError::NotImplemented => 0x01,
+            StorageError::InvalidArgument(_, _, _) => 0x02,
+            StorageError::MethodNotAllowed => 0x03,
+            StorageError::BucketNotFound(_) => 0x04,
+            StorageError::BucketNotEmpty(_) => 0x05,
+            StorageError::BucketNameInvalid(_) => 0x06,
+            StorageError::ObjectNameInvalid(_, _) => 0x07,
+            StorageError::BucketExists(_) => 0x08,
+            StorageError::StorageFull => 0x09,
+            StorageError::SlowDown => 0x0A,
+            StorageError::PrefixAccessDenied(_, _) => 0x0B,
+            StorageError::InvalidUploadIDKeyCombination(_, _) => 0x0C,
+            StorageError::MalformedUploadID(_) => 0x0D,
+            StorageError::ObjectNameTooLong(_, _) => 0x0E,
+            StorageError::ObjectNamePrefixAsSlash(_, _) => 0x0F,
+            StorageError::ObjectNotFound(_, _) => 0x10,
+            StorageError::VersionNotFound(_, _, _) => 0x11,
+            StorageError::InvalidUploadID(_, _, _) => 0x12,
+            StorageError::InvalidVersionID(_, _, _) => 0x13,
+            StorageError::DataMovementOverwriteErr(_, _, _) => 0x14,
+            StorageError::ObjectExistsAsDirectory(_, _) => 0x15,
+            StorageError::InsufficientReadQuorum => 0x16,
+            StorageError::InsufficientWriteQuorum => 0x17,
+            StorageError::DecommissionNotStarted => 0x18,
+            StorageError::VolumeNotFound(_) => 0x19,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(StorageError::NotImplemented),
+            0x02 => Some(StorageError::InvalidArgument(Default::default(), Default::default(), Default::default())),
+            0x03 => Some(StorageError::MethodNotAllowed),
+            0x04 => Some(StorageError::BucketNotFound(Default::default())),
+            0x05 => Some(StorageError::BucketNotEmpty(Default::default())),
+            0x06 => Some(StorageError::BucketNameInvalid(Default::default())),
+            0x07 => Some(StorageError::ObjectNameInvalid(Default::default(), Default::default())),
+            0x08 => Some(StorageError::BucketExists(Default::default())),
+            0x09 => Some(StorageError::StorageFull),
+            0x0A => Some(StorageError::SlowDown),
+            0x0B => Some(StorageError::PrefixAccessDenied(Default::default(), Default::default())),
+            0x0C => Some(StorageError::InvalidUploadIDKeyCombination(Default::default(), Default::default())),
+            0x0D => Some(StorageError::MalformedUploadID(Default::default())),
+            0x0E => Some(StorageError::ObjectNameTooLong(Default::default(), Default::default())),
+            0x0F => Some(StorageError::ObjectNamePrefixAsSlash(Default::default(), Default::default())),
+            0x10 => Some(StorageError::ObjectNotFound(Default::default(), Default::default())),
+            0x11 => Some(StorageError::VersionNotFound(Default::default(), Default::default(), Default::default())),
+            0x12 => Some(StorageError::InvalidUploadID(Default::default(), Default::default(), Default::default())),
+            0x13 => Some(StorageError::InvalidVersionID(Default::default(), Default::default(), Default::default())),
+            0x14 => Some(StorageError::DataMovementOverwriteErr(
+                Default::default(),
+                Default::default(),
+                Default::default(),
+            )),
+            0x15 => Some(StorageError::ObjectExistsAsDirectory(Default::default(), Default::default())),
+            0x16 => Some(StorageError::InsufficientReadQuorum),
+            0x17 => Some(StorageError::InsufficientWriteQuorum),
+            0x18 => Some(StorageError::DecommissionNotStarted),
+            0x19 => Some(StorageError::VolumeNotFound(Default::default())),
+            _ => None,
+        }
+    }
+}
+
 pub fn to_object_err(err: Error, params: Vec<&str>) -> Error {
     if let Some(e) = err.downcast_ref::<DiskError>() {
         match e {

ecstore/src/store_init.rs

@@ -16,7 +16,7 @@ use std::{
     fmt::Debug,
 };
 
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 use uuid::Uuid;
 
 pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec<Option<DiskStore>>, Vec<Option<Error>>) {
@@ -330,3 +330,26 @@ impl ErasureError {
         false
     }
 }
+
+impl ErasureError {
+    pub fn to_u32(&self) -> u32 {
+        match self {
+            ErasureError::ErasureReadQuorum => 0x01,
+            ErasureError::_ErasureWriteQuorum => 0x02,
+            ErasureError::NotFirstDisk => 0x03,
+            ErasureError::FirstDiskWait => 0x04,
+            ErasureError::InvalidPart(_) => 0x05,
+        }
+    }
+
+    pub fn from_u32(error: u32) -> Option<Self> {
+        match error {
+            0x01 => Some(ErasureError::ErasureReadQuorum),
+            0x02 => Some(ErasureError::_ErasureWriteQuorum),
+            0x03 => Some(ErasureError::NotFirstDisk),
+            0x04 => Some(ErasureError::FirstDiskWait),
+            0x05 => Some(ErasureError::InvalidPart(Default::default())),
+            _ => None,
+        }
+    }
+}

ecstore/src/utils/crypto.rs

@@ -10,16 +10,16 @@ pub fn hex(data: impl AsRef<[u8]>) -> String {
     hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower)
 }
 
-#[cfg(windows)]
+// #[cfg(windows)]
-pub fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> {
+// pub fn sha256(data: &[u8]) -> impl AsRef<[u8; 32]> {
-    use sha2::{Digest, Sha256};
+//     use sha2::{Digest, Sha256};
-    <Sha256 as Digest>::digest(data)
+//     <Sha256 as Digest>::digest(data)
-}
+// }
 
-#[cfg(not(windows))]
+// #[cfg(not(windows))]
-pub fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
+// pub fn sha256(data: &[u8]) -> impl AsRef<[u8]> {
-    use openssl::hash::{Hasher, MessageDigest};
+//     use openssl::hash::{Hasher, MessageDigest};
-    let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
+//     let mut h = Hasher::new(MessageDigest::sha256()).unwrap();
-    h.update(data).unwrap();
+//     h.update(data).unwrap();
-    h.finish().unwrap()
+//     h.finish().unwrap()
-}
+// }

ecstore/src/utils/mod.rs

@@ -1,3 +1,12 @@
+use crate::bucket::error::BucketMetadataError;
+use crate::config::error::ConfigError;
+use crate::disk::error::DiskError;
+use crate::error::Error;
+use crate::quorum::QuorumError;
+use crate::store_err::StorageError;
+use crate::store_init::ErasureError;
+use protos::proto_gen::node_service::Error as Proto_Error;
+
 pub mod bool_flag;
 pub mod crypto;
 pub mod ellipses;
@@ -8,3 +17,104 @@ pub mod os;
 pub mod path;
 pub mod wildcard;
 pub mod xml;
+
+const ERROR_MODULE_MASK: u32 = 0xFF00;
+pub const ERROR_TYPE_MASK: u32 = 0x00FF;
+const DISK_ERROR_MASK: u32 = 0x0100;
+const STORAGE_ERROR_MASK: u32 = 0x0200;
+const BUCKET_METADATA_ERROR_MASK: u32 = 0x0300;
+const CONFIG_ERROR_MASK: u32 = 0x04000;
+const QUORUM_ERROR_MASK: u32 = 0x0500;
+const ERASURE_ERROR_MASK: u32 = 0x0600;
+
+// error to u8
+pub fn error_to_u32(err: &Error) -> u32 {
+    if let Some(e) = err.downcast_ref::<DiskError>() {
+        DISK_ERROR_MASK | e.to_u32()
+    } else if let Some(e) = err.downcast_ref::<StorageError>() {
+        STORAGE_ERROR_MASK | e.to_u32()
+    } else if let Some(e) = err.downcast_ref::<BucketMetadataError>() {
+        BUCKET_METADATA_ERROR_MASK | e.to_u32()
+    } else if let Some(e) = err.downcast_ref::<ConfigError>() {
+        CONFIG_ERROR_MASK | e.to_u32()
+    } else if let Some(e) = err.downcast_ref::<QuorumError>() {
+        QUORUM_ERROR_MASK | e.to_u32()
+    } else if let Some(e) = err.downcast_ref::<ErasureError>() {
+        ERASURE_ERROR_MASK | e.to_u32()
+    } else {
+        0
+    }
+}
+
+pub fn u32_to_error(e: u32) -> Option<Error> {
+    match e & ERROR_MODULE_MASK {
+        DISK_ERROR_MASK => DiskError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        STORAGE_ERROR_MASK => StorageError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        BUCKET_METADATA_ERROR_MASK => BucketMetadataError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        CONFIG_ERROR_MASK => ConfigError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        QUORUM_ERROR_MASK => QuorumError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        ERASURE_ERROR_MASK => ErasureError::from_u32(e & ERROR_TYPE_MASK).map(|e| Error::new(e)),
+        _ => None,
+    }
+}
+
+pub fn err_to_proto_err(err: &Error, msg: &str) -> Proto_Error {
+    let num = error_to_u32(err);
+    Proto_Error {
+        code: num,
+        error_info: msg.to_string(),
+    }
+}
+
+pub fn proto_err_to_err(err: &Proto_Error) -> Error {
+    if let Some(e) = u32_to_error(err.code) {
+        e
+    } else {
+        Error::from_string(err.error_info.clone())
+    }
+}
+
+#[test]
+fn test_u32_to_error() {
+    let error = Error::new(DiskError::FileCorrupt);
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(new_error.unwrap().downcast_ref::<DiskError>(), Some(&DiskError::FileCorrupt));
+
+    let error = Error::new(StorageError::BucketNotEmpty(Default::default()));
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(
+        new_error.unwrap().downcast_ref::<StorageError>(),
+        Some(&StorageError::BucketNotEmpty(Default::default()))
+    );
+
+    let error = Error::new(BucketMetadataError::BucketObjectLockConfigNotFound);
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(
+        new_error.unwrap().downcast_ref::<BucketMetadataError>(),
+        Some(&BucketMetadataError::BucketObjectLockConfigNotFound)
+    );
+
+    let error = Error::new(ConfigError::NotFound);
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(new_error.unwrap().downcast_ref::<ConfigError>(), Some(&ConfigError::NotFound));
+
+    let error = Error::new(QuorumError::Read);
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(new_error.unwrap().downcast_ref::<QuorumError>(), Some(&QuorumError::Read));
+
+    let error = Error::new(ErasureError::ErasureReadQuorum);
+    let num = error_to_u32(&error);
+    let new_error = u32_to_error(num);
+    assert!(new_error.is_some());
+    assert_eq!(new_error.unwrap().downcast_ref::<ErasureError>(), Some(&ErasureError::ErasureReadQuorum));
+}

ecstore/src/utils/os/windows.rs

@@ -1,6 +1,8 @@
+#![allow(unsafe_code)] // TODO: audit unsafe code
+
 use super::IOStats;
 use crate::{disk::Info, error::Result};
-use std::io::{Error, ErrorKind, Result};
+use std::io::{Error, ErrorKind, Result as IoResult};
 use std::mem;
 use std::os::windows::ffi::OsStrExt;
 use std::path::Path;
@@ -32,7 +34,7 @@ pub fn get_info(p: impl AsRef<Path>) -> Result<Info> {
         )
     };
     if success == 0 {
-        return Err(Error::last_os_error());
+        return Err(Error::last_os_error().into());
     }
 
     let total = unsafe { *lp_total_number_of_bytes.QuadPart() };
@@ -47,7 +49,8 @@ pub fn get_info(p: impl AsRef<Path>) -> Result<Info> {
                 total,
                 p.as_ref().display()
             ),
-        ));
+        )
+        .into());
     }
 
     let mut lp_sectors_per_cluster: DWORD = 0;
@@ -65,7 +68,7 @@ pub fn get_info(p: impl AsRef<Path>) -> Result<Info> {
         )
     };
     if success == 0 {
-        return Err(Error::last_os_error());
+        return Err(Error::last_os_error().into());
     }
 
     Ok(Info {
@@ -87,7 +90,7 @@ fn get_volume_name(v: &[WCHAR]) -> Result<LPCWSTR> {
     let success = unsafe { GetVolumePathNameW(v.as_ptr(), lp_volume_name_buffer.as_mut_ptr(), volume_name_size) };
 
     if success == 0 {
-        return Err(Error::last_os_error());
+        return Err(Error::last_os_error().into());
     }
 
     Ok(lp_volume_name_buffer.as_ptr())
@@ -126,7 +129,7 @@ fn get_fs_type(p: &[WCHAR]) -> Result<String> {
     };
 
     if success == 0 {
-        return Err(Error::last_os_error());
+        return Err(Error::last_os_error().into());
     }
 
     Ok(utf16_to_string(&lp_file_system_name_buffer))
@@ -137,5 +140,5 @@ pub fn same_disk(disk1: &str, disk2: &str) -> Result<bool> {
 }
 
 pub fn get_drive_stats(major: u32, minor: u32) -> Result<IOStats> {
-    IOStats::default()
+    Ok(IOStats::default())
 }

iam/src/policy/action.rs

@@ -56,7 +56,7 @@ impl Action {
 }
 
 impl Action {
-    const S3_PREFIX: &str = "s3:";
+    const S3_PREFIX: &'static str = "s3:";
     const ADMIN_PREFIX: &str = "admin:";
     const STS_PREFIX: &str = "sts:";
     const KMS_PREFIX: &str = "kms:";

iam/src/store/object.rs

@@ -4,14 +4,12 @@ use ecstore::{
     config::error::is_not_found,
     store::ECStore,
     store_api::{HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader},
-    store_list_objects::ListPathOptions,
     utils::path::dir,
     StorageAPI,
 };
 use futures::future::try_join_all;
 use log::{debug, warn};
 use serde::{de::DeserializeOwned, Serialize};
-use tracing::error;
 
 use super::Store;
 use crate::{

madmin/Cargo.toml

@@ -14,7 +14,6 @@ chrono.workspace = true
 common.workspace = true
 humantime.workspace = true
 hyper.workspace = true
-psutil = "3.3.0"
 serde.workspace = true
-time.workspace =true
+time.workspace = true
 tracing.workspace = true