add auto fix disk

Signed-off-by: junxiang Mu <1948535941@qq.com>
This commit is contained in:
junxiang Mu
2024-12-17 21:04:56 +08:00
parent fac0feeeaa
commit f472c16e8a
9 changed files with 952 additions and 3188 deletions
+15 -11
View File
@@ -40,16 +40,15 @@ pub static DEFAULT_MONITOR_NEW_DISK_INTERVAL: Duration = Duration::from_secs(10)
pub async fn init_auto_heal() {
init_background_healing().await;
if let Ok(v) = env::var("_RUSTFS_AUTO_DRIVE_HEALING") {
if v == "on" {
info!("start monitor local disks and heal");
GLOBAL_BackgroundHealState
.push_heal_local_disks(&get_local_disks_to_heal().await)
.await;
spawn(async {
monitor_local_disks_and_heal().await;
});
}
let v = env::var("_RUSTFS_AUTO_DRIVE_HEALING").unwrap_or("on".to_string());
if v == "on" {
info!("start monitor local disks and heal");
GLOBAL_BackgroundHealState
.push_heal_local_disks(&get_local_disks_to_heal().await)
.await;
spawn(async {
monitor_local_disks_and_heal().await;
});
}
spawn(async {
GLOBAL_MRFState.heal_routine().await;
@@ -73,12 +72,14 @@ pub async fn get_local_disks_to_heal() -> Vec<Endpoint> {
if let Some(disk) = disk {
if let Err(err) = disk.disk_info(&DiskInfoOptions::default()).await {
if let Some(DiskError::UnformattedDisk) = err.downcast_ref() {
info!("get_local_disks_to_heal, disk is unformatted: {}", err);
disks_to_heal.push(disk.endpoint());
}
}
let h = disk.healing().await;
if let Some(h) = h {
if !h.finished {
info!("get_local_disks_to_heal, disk healing not finished");
disks_to_heal.push(disk.endpoint());
}
}
@@ -104,8 +105,11 @@ async fn monitor_local_disks_and_heal() {
continue;
}
info!("heal local disks: {:?}", heal_disks);
let store = new_object_layer_fn().expect("errServerNotInitialized");
if let (_, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
if let (result, Some(err)) = store.heal_format(false).await.expect("heal format failed") {
error!("heal local disk format error: {}", err);
if let Some(DiskError::NoHealRequired) = err.downcast_ref::<DiskError>() {
} else {
info!("heal format err: {}", err.to_string());