gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-02 02:08:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement heal channel mechanism for admin-ahm communication

Browse Source 
- Add global unbounded channel in common crate for heal requests
- Implement channel processor in ahm to handle heal commands
- Add Start/Query/Cancel commands support via channel
- Integrate heal manager initialization in main.rs
- Replace direct MRF calls with channel-based heal requests in ecstore
- Support advanced heal options including pool_index and set_index
- Enable admin handlers to send heal requests via channel
This commit is contained in:
junxiang Mu
2025-07-15 17:40:51 +08:00
parent 3409cd8dff
commit 8e766b90cd
17 changed files with 679 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/global.rs
+1 -3
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::heal::mrf::MRFState;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::LifecycleSys,
disk::DiskStore,
@@ -53,12 +52,11 @@ pub static ref GLOBAL_Endpoints: OnceLock<EndpointServerPools> = OnceLock::new()
pub static ref GLOBAL_RootDiskThreshold: RwLock<u64> = RwLock::new(0);
pub static ref GLOBAL_BackgroundHealRoutine: Arc<HealRoutine> = HealRoutine::new();
pub static ref GLOBAL_BackgroundHealState: Arc<AllHealState> = AllHealState::new(false);
// pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
//pub static ref GLOBAL_RemoteTargetTransport
pub static ref GLOBAL_ALlHealState: Arc<AllHealState> = AllHealState::new(false);
pub static ref GLOBAL_MRFState: Arc<MRFState> = Arc::new(MRFState::new());
static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
crates/ecstore/src/heal/background_heal_ops.rs
+5 -5
View File
@@ -33,7 +33,7 @@ use super::{
heal_ops::{HealSequence, new_bg_heal_sequence},
};
use crate::error::{Error, Result};
use crate::global::{GLOBAL_MRFState, get_background_services_cancel_token};
use crate::global::get_background_services_cancel_token;
use crate::heal::error::ERR_RETRY_HEALING;
use crate::heal::heal_commands::{HEAL_ITEM_BUCKET, HealScanMode};
use crate::heal::heal_ops::{BG_HEALING_UUID, HealSource};
@@ -76,10 +76,10 @@ pub async fn init_auto_heal() {
});
}
let cancel_clone = cancel_token.clone();
spawn(async move {
GLOBAL_MRFState.heal_routine_with_cancel(cancel_clone).await;
});
// let cancel_clone = cancel_token.clone();
// spawn(async move {
// GLOBAL_MRFState.heal_routine_with_cancel(cancel_clone).await;
// });
}
async fn init_background_healing() {
crates/ecstore/src/set_disk.rs
+68 -45
View File
@@ -2033,17 +2033,23 @@ impl SetDisks {
let fi = Self::pick_valid_fileinfo(&parts_metadata, mot_time, etag, read_quorum as usize)?;
if errs.iter().any(|err| err.is_some()) {
GLOBAL_MRFState
.add_partial(PartialOperation {
bucket: fi.volume.to_string(),
object: fi.name.to_string(),
queued: Utc::now(),
version_id: fi.version_id.map(|v| v.to_string()),
set_index: self.set_index,
pool_index: self.pool_index,
..Default::default()
})
.await;
let _ = rustfs_common::heal_channel::send_heal_request(
rustfs_common::heal_channel::create_heal_request_with_options(
fi.volume.to_string(), // bucket
Some(fi.name.to_string()), // object_prefix
false, // force_start
Some(rustfs_common::heal_channel::HealChannelPriority::Normal), // priority
Some(self.pool_index), // pool_index
Some(self.set_index), // set_index
None, // scan_mode
None, // remove_corrupted
None, // recreate_missing
None, // update_parity
None, // recursive
None, // dry_run
None, // timeout_seconds
)
).await;
}
// debug!("get_object_fileinfo pick fi {:?}", &fi);
@@ -2174,18 +2180,23 @@ impl SetDisks {
match de_err {
DiskError::FileNotFound | DiskError::FileCorrupt => {
error!("erasure.decode err 111 {:?}", &de_err);
GLOBAL_MRFState
.add_partial(PartialOperation {
bucket: bucket.to_string(),
object: object.to_string(),
queued: Utc::now(),
version_id: fi.version_id.map(|v| v.to_string()),
set_index,
pool_index,
bitrot_scan: de_err == DiskError::FileCorrupt,
..Default::default()
})
.await;
let _ = rustfs_common::heal_channel::send_heal_request(
rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(rustfs_common::heal_channel::HealChannelPriority::Normal),
Some(pool_index),
Some(set_index),
None,
None,
None,
None,
None,
None,
None,
)
).await;
has_err = false;
}
_ => {}
@@ -4693,17 +4704,23 @@ impl StorageAPI for SetDisks {
#[tracing::instrument(skip(self))]
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
GLOBAL_MRFState
.add_partial(PartialOperation {
bucket: bucket.to_string(),
object: object.to_string(),
version_id: Some(version_id.to_string()),
queued: Utc::now(),
set_index: self.set_index,
pool_index: self.pool_index,
..Default::default()
})
.await;
let _ = rustfs_common::heal_channel::send_heal_request(
rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(rustfs_common::heal_channel::HealChannelPriority::Normal),
Some(self.pool_index),
Some(self.set_index),
None,
None,
None,
None,
None,
None,
None,
)
).await;
Ok(())
}
@@ -5845,17 +5862,23 @@ impl StorageAPI for SetDisks {
.await?;
}
if let Some(versions) = versions {
GLOBAL_MRFState
.add_partial(PartialOperation {
bucket: bucket.to_string(),
object: object.to_string(),
queued: Utc::now(),
versions,
set_index: self.set_index,
pool_index: self.pool_index,
..Default::default()
})
.await;
let _ = rustfs_common::heal_channel::send_heal_request(
rustfs_common::heal_channel::create_heal_request_with_options(
bucket.to_string(),
Some(object.to_string()),
false,
Some(rustfs_common::heal_channel::HealChannelPriority::Normal),
Some(self.pool_index),
Some(self.set_index),
None,
None,
None,
None,
None,
None,
None,
)
).await;
}
let upload_id_path = upload_id_path.clone();