heal admin api(1)

Signed-off-by: mujunxiang <1948535941@qq.com>

common/protos/src/generated/flatbuffers_generated/models.rs

@@ -1,9 +1,10 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
+
 // @generated
 
-use core::cmp::Ordering;
 use core::mem;
+use core::cmp::Ordering;
 
 extern crate flatbuffers;
 use self::flatbuffers::{EndianScalar, Follow};
@@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow};
 #[allow(unused_imports, dead_code)]
 pub mod models {
 
-    use core::cmp::Ordering;
+  use core::mem;
-    use core::mem;
+  use core::cmp::Ordering;
 
-    extern crate flatbuffers;
+  extern crate flatbuffers;
-    use self::flatbuffers::{EndianScalar, Follow};
+  use self::flatbuffers::{EndianScalar, Follow};
 
-    pub enum PingBodyOffset {}
+pub enum PingBodyOffset {}
-    #[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq)]
 
-    pub struct PingBody<'a> {
+pub struct PingBody<'a> {
-        pub _tab: flatbuffers::Table<'a>,
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+  type Inner = PingBody<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> PingBody<'a> {
+  pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+
+  pub const fn get_fully_qualified_name() -> &'static str {
+    "models.PingBody"
+  }
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    PingBody { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args PingBodyArgs<'args>
+  ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
+    let mut builder = PingBodyBuilder::new(_fbb);
+    if let Some(x) = args.payload { builder.add_payload(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for PingBody<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct PingBodyArgs<'a> {
+    pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for PingBodyArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    PingBodyArgs {
+      payload: None,
     }
+  }
+}
 
-    impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        type Inner = PingBody<'a>;
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        #[inline]
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
-        unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+}
-            Self {
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
-                _tab: flatbuffers::Table::new(buf, loc),
+  #[inline]
-            }
+  pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
-        }
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    PingBodyBuilder {
+      fbb_: _fbb,
+      start_: start,
     }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
 
-    impl<'a> PingBody<'a> {
+impl core::fmt::Debug for PingBody<'_> {
-        pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("PingBody");
+      ds.field("payload", &self.payload());
+      ds.finish()
+  }
+}
+}  // pub mod models
 
-        pub const fn get_fully_qualified_name() -> &'static str {
-            "models.PingBody"
-        }
-
-        #[inline]
-        pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
-            PingBody { _tab: table }
-        }
-        #[allow(unused_mut)]
-        pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
-            _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
-            args: &'args PingBodyArgs<'args>,
-        ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
-            let mut builder = PingBodyBuilder::new(_fbb);
-            if let Some(x) = args.payload {
-                builder.add_payload(x);
-            }
-            builder.finish()
-        }
-
-        #[inline]
-        pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
-            // Safety:
-            // Created from valid Table for this object
-            // which contains a valid value in this slot
-            unsafe {
-                self._tab
-                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
-            }
-        }
-    }
-
-    impl flatbuffers::Verifiable for PingBody<'_> {
-        #[inline]
-        fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> {
-            use self::flatbuffers::Verifiable;
-            v.visit_table(pos)?
-                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
-                .finish();
-            Ok(())
-        }
-    }
-    pub struct PingBodyArgs<'a> {
-        pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
-    }
-    impl<'a> Default for PingBodyArgs<'a> {
-        #[inline]
-        fn default() -> Self {
-            PingBodyArgs { payload: None }
-        }
-    }
-
-    pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
-    }
-    impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
-        #[inline]
-        pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b, u8>>) {
-            self.fbb_
-                .push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
-        }
-        #[inline]
-        pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
-            let start = _fbb.start_table();
-            PingBodyBuilder {
-                fbb_: _fbb,
-                start_: start,
-            }
-        }
-        #[inline]
-        pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
-            let o = self.fbb_.end_table(self.start_);
-            flatbuffers::WIPOffset::new(o.value())
-        }
-    }
-
-    impl core::fmt::Debug for PingBody<'_> {
-        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-            let mut ds = f.debug_struct("PingBody");
-            ds.field("payload", &self.payload());
-            ds.finish()
-        }
-    }
-} // pub mod models

ecstore/src/cache_value/metacache_set.rs

@@ -139,7 +139,7 @@ pub async fn list_path_raw(mut rx: B_Receiver<bool>, opts: ListPathRawOptions) -
         });
     }
 
-    let errs: Vec<Option<Error>> = Vec::with_capacity(readers.len());
+    let errs: Vec<Option<Error>> = vec![None; readers.len()];
     loop {
         let mut current = MetaCacheEntry::default();
         let (mut at_eof, mut has_err, mut agree) = (0, 0, 0);

ecstore/src/heal/background_heal_ops.rs

@@ -37,6 +37,7 @@ pub async fn init_auto_heal() {
     init_background_healing().await;
     if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
         if v == "on" {
+            info!("start monitor local disks and heal");
             GLOBAL_BackgroundHealState
                 .write()
                 .await

ecstore/src/heal/data_scanner.rs

@@ -511,9 +511,7 @@ impl FolderScanner {
         let this_hash = hash_path(&folder.name);
         let was_compacted = into.compacted;
 
-        // do not really loop
+        'outer: {
-        let mut times = 1;
-        while times > 0 {
             let mut abandoned_children: DataUsageHashMap = if !into.compacted {
                 self.old_cache.find_children_copy(this_hash.clone())
             } else {
@@ -622,7 +620,7 @@ impl FolderScanner {
             }
             if found_objects && *GLOBAL_IsErasure.read().await {
                 // If we found an object in erasure mode, we skip subdirs (only datadirs)...
-                break;
+                break 'outer;
             }
 
             let should_compact = self.new_cache.info.name != folder.name
@@ -714,11 +712,11 @@ impl FolderScanner {
 
             // Scan for healing
             if abandoned_children.is_empty() || !self.should_heal().await {
-                break;
+                break 'outer;
             }
 
             if self.disks.is_empty() || self.disks_quorum == 0 {
-                break;
+                break 'outer;
             }
 
             let (bg_seq, found) = GLOBAL_BackgroundHealState
@@ -727,7 +725,7 @@ impl FolderScanner {
                 .get_heal_sequence_by_token(BG_HEALING_UUID)
                 .await;
             if !found {
-                break;
+                break 'outer;
             }
             let bg_seq = bg_seq.unwrap();
 
@@ -901,7 +899,6 @@ impl FolderScanner {
                     scan(&this, into, self).await;
                 }
             }
-            times -= 1;
         }
         if !was_compacted {
             self.new_cache.replace_hashed(&this_hash, &Some(folder.parent.clone()), into);

ecstore/src/heal/heal_ops.rs

@@ -7,6 +7,8 @@ use super::{
         HEAL_ITEM_BUCKET_METADATA,
     },
 };
+use crate::heal::heal_commands::{HEAL_DEEP_SCAN, HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT};
+use crate::store_api::StorageAPI;
 use crate::{
     config::common::CONFIG_PREFIX,
     disk::RUSTFS_META_BUCKET,
@@ -25,10 +27,6 @@ use crate::{
     new_object_layer_fn,
     utils::path::has_profix,
 };
-use crate::{
-    heal::heal_commands::{HEAL_ITEM_BUCKET, HEAL_ITEM_OBJECT},
-    store_api::StorageAPI,
-};
 use lazy_static::lazy_static;
 use std::{
     collections::HashMap,
@@ -140,6 +138,32 @@ pub fn new_bg_heal_sequence() -> HealSequence {
     }
 }
 
+pub fn new_heal_sequence(bucket: &str, obj_prefix: &str, client_addr: &str, hs: HealOpts, force_start: bool) -> HealSequence {
+    let client_token = Uuid::new_v4().to_string();
+    let (tx, rx) = mpsc::channel(10);
+    HealSequence {
+        bucket: bucket.to_string(),
+        object: obj_prefix.to_string(),
+        report_progress: true,
+        start_time: SystemTime::now(),
+        client_token,
+        client_address: client_addr.to_string(),
+        force_started: force_start,
+        setting: hs.clone(),
+        current_status: Arc::new(RwLock::new(HealSequenceStatus {
+            summary: HEAL_NOT_STARTED_STATUS.to_string(),
+            heal_setting: hs,
+            ..Default::default()
+        })),
+        traverse_and_heal_done_tx: Arc::new(RwLock::new(tx)),
+        traverse_and_heal_done_rx: Arc::new(RwLock::new(rx)),
+        scanned_items_map: HashMap::new(),
+        healed_items_map: HashMap::new(),
+        heal_failed_items_map: HashMap::new(),
+        ..Default::default()
+    }
+}
+
 impl Default for HealSequence {
     fn default() -> Self {
         let (h_tx, h_rx) = mpsc::channel(1);

ecstore/src/store.rs

@@ -2173,7 +2173,7 @@ async fn init_local_peer(endpoint_pools: &EndpointServerPools, host: &String, po
     *GLOBAL_Local_Node_Name.write().await = peer_set[0].clone();
 }
 
-fn is_valid_object_prefix(object: &str) -> bool {
+pub fn is_valid_object_prefix(object: &str) -> bool {
     // Implement object prefix validation
     !object.is_empty() // Placeholder
 }

rustfs/src/admin/mod.rs

@@ -2,13 +2,14 @@ pub mod handlers;
 pub mod router;
 
 use common::error::Result;
+use ecstore::global::{is_dist_erasure, is_erasure};
 use hyper::Method;
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
 
 const ADMIN_PREFIX: &str = "/rustfs/admin";
 
-pub fn make_admin_route() -> Result<impl S3Route> {
+pub async fn make_admin_route() -> Result<impl S3Route> {
     let mut r = S3Router::new();
 
     r.insert(Method::POST, "/", AdminOperation(&handlers::AssumeRoleHandle {}))?;
@@ -53,17 +54,6 @@ pub fn make_admin_route() -> Result<impl S3Route> {
         AdminOperation(&handlers::MetricsHandler {}),
     )?;
 
-    r.insert(
-        Method::POST,
-        format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
-        AdminOperation(&handlers::HealHandler {}),
-    )?;
-    r.insert(
-        Method::POST,
-        format!("{}{}", ADMIN_PREFIX, "/v3/background-heal/status").as_str(),
-        AdminOperation(&handlers::BackgroundHealStatusHandler {}),
-    )?;
-
     r.insert(
         Method::GET,
         format!("{}{}", ADMIN_PREFIX, "/v3/pools/list").as_str(),
@@ -101,5 +91,19 @@ pub fn make_admin_route() -> Result<impl S3Route> {
         AdminOperation(&handlers::RebalanceStop {}),
     )?;
 
+    // Some APIs are only available in EC mode
+    // if is_dist_erasure().await || is_erasure().await {
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/heal/{bucket}/{prefix}").as_str(),
+        AdminOperation(&handlers::HealHandler {}),
+    )?;
+    r.insert(
+        Method::POST,
+        format!("{}{}", ADMIN_PREFIX, "/v3/background-heal/status").as_str(),
+        AdminOperation(&handlers::BackgroundHealStatusHandler {}),
+    )?;
+    // }
+
     Ok(r)
 }

rustfs/src/main.rs

@@ -10,7 +10,7 @@ use clap::Parser;
 use common::error::{Error, Result};
 use ecstore::{
     endpoints::EndpointServerPools,
-    heal::data_scanner::init_data_scanner,
+    heal::{background_heal_ops::init_auto_heal, data_scanner::init_data_scanner},
     set_global_endpoints,
     store::{init_local_disks, ECStore},
     update_erasure_type,
@@ -192,7 +192,11 @@ async fn run(opt: config::Opt) -> Result<()> {
     })?;
     warn!(" init store success!");
     // init scanner
-    init_data_scanner().await;
+    // init_data_scanner().await;
+    // // init auto heal
+    // init_auto_heal().await;
+
+    info!("server was started");
 
     tokio::select! {
         _ = tokio::signal::ctrl_c() => {