add down object heal

Signed-off-by: junxiang Mu <1948535941@qq.com>

ecstore/src/set_disk.rs

@@ -6,7 +6,10 @@ use std::{
     time::Duration,
 };
 
+use crate::config::error::is_not_found;
+use crate::global::GLOBAL_MRFState;
 use crate::heal::heal_ops::{HealEntryFn, HealSequence};
+use crate::heal::mrf::PartialOperation;
 use crate::{
     bitrot::{bitrot_verify, close_bitrot_writers, new_bitrot_filereader, new_bitrot_filewriter, BitrotFileWriter},
     cache_value::metacache_set::{list_path_raw, ListPathRawOptions},
@@ -56,6 +59,7 @@ use crate::{
     heal::data_scanner::{globalHealConfig, HEAL_DELETE_DANGLING},
     store_api::ListObjectVersionsInfo,
 };
+use chrono::Utc;
 use futures::future::join_all;
 use glob::Pattern;
 use http::HeaderMap;
@@ -1621,6 +1625,19 @@ impl SetDisks {
         let (op_online_disks, mot_time, etag) = Self::list_online_disks(&disks, &parts_metadata, &errs, read_quorum as usize);
 
         let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?;
+        if errs.iter().any(|err| err.is_some()) {
+            GLOBAL_MRFState
+                .add_partial(PartialOperation {
+                    bucket: fi.volume.to_string(),
+                    object: fi.name.to_string(),
+                    queued: Utc::now(),
+                    version_id: fi.version_id.map(|v| v.to_string()),
+                    set_index: self.set_index,
+                    pool_index: self.pool_index,
+                    ..Default::default()
+                })
+                .await;
+        }
         // debug!("get_object_fileinfo pick fi {:?}", &fi);
 
         // let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
@@ -1639,6 +1656,8 @@ impl SetDisks {
         fi: FileInfo,
         files: Vec<FileInfo>,
         disks: &[Option<DiskStore>],
+        set_index: usize,
+        pool_index: usize,
     ) -> Result<()> {
         let (disks, files) = Self::shuffle_disks_and_parts_metadata_by_index(disks, &files, &fi);
 
@@ -1724,10 +1743,34 @@ impl SetDisks {
             //     "read part {} part_offset {},part_length {},part_size {}  ",
             //     part_number, part_offset, part_length, part_size
             // );
-            let _n = erasure
+            let (written, mut err) = erasure
                 .decode(writer, readers, part_offset as usize, part_length, part_size)
-                .await?;
-
+                .await;
+            if let Some(e) = err.as_ref() {
+                if written == part_length {
+                    match e.downcast_ref::<DiskError>() {
+                        Some(DiskError::FileNotFound) | Some(DiskError::FileCorrupt) => {
+                            GLOBAL_MRFState
+                                .add_partial(PartialOperation {
+                                    bucket: bucket.to_string(),
+                                    object: object.to_string(),
+                                    queued: Utc::now(),
+                                    version_id: fi.version_id.map(|v| v.to_string()),
+                                    set_index,
+                                    pool_index,
+                                    bitrot_scan: !is_not_found(e),
+                                    ..Default::default()
+                                })
+                                .await;
+                            err = None;
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            if let Some(err) = err {
+                return Err(err);
+            }
             // debug!("ec decode {} writed size {}", part_number, n);
 
             total_readed += part_length as i64;
@@ -3378,8 +3421,14 @@ impl ObjectIO for SetDisks {
         // let disks = disks.clone();
         let bucket = String::from(bucket);
         let object = String::from(object);
+        let set_index = self.set_index.clone();
+        let pool_index = self.pool_index.clone();
         tokio::spawn(async move {
-            if let Err(e) = Self::get_object_with_fileinfo(&bucket, &object, offset, length, &mut wd, fi, files, &disks).await {
+            if let Err(e) = Self::get_object_with_fileinfo(
+                &bucket, &object, offset, length, &mut wd, fi, files, &disks, set_index, pool_index,
+            )
+            .await
+            {
                 error!("get_object_with_fileinfo err {:?}", e);
             };
         });
@@ -4438,7 +4487,7 @@ impl StorageAPI for SetDisks {
             }
         }
 
-        let (online_disks, _, op_old_dir) = Self::rename_data(
+        let (online_disks, versions, op_old_dir) = Self::rename_data(
             &shuffle_disks,
             RUSTFS_META_MULTIPART_BUCKET,
             &upload_id_path,
@@ -4467,6 +4516,19 @@ impl StorageAPI for SetDisks {
             self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
                 .await?;
         }
+        if let Some(versions) = versions {
+            GLOBAL_MRFState
+                .add_partial(PartialOperation {
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    queued: Utc::now(),
+                    versions,
+                    set_index: self.set_index,
+                    pool_index: self.pool_index,
+                    ..Default::default()
+                })
+                .await;
+        }
 
         let _ = self.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_id_path).await;