mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
feat(ahm): add HealingTracker support & complete fresh-disk healing
• Introduce ecstore HealingTracker into ahm crate; load/init/save tracker • Re-implement heal_fresh_disk to use heal_erasure_set with tracker • Enhance auto-disk scanner: detect unformatted disks via get_disk_id() • Remove DataUsageCache handling for now • Refactor imports & types, clean up duplicate constants
This commit is contained in:
junxiang Mu
+193
-164
@@ -14,24 +14,25 @@
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::heal::{progress::HealProgress, storage::HealStorageAPI};
|
||||
use rustfs_ecstore::config::RUSTFS_CONFIG_PREFIX;
|
||||
use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
use crate::heal::storage::DiskStatus;
|
||||
use rustfs_ecstore::disk::error::DiskError;
|
||||
use rustfs_ecstore::disk::{DiskAPI, DiskInfoOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN;
|
||||
use rustfs_ecstore::heal::heal_commands::{init_healing_tracker, load_healing_tracker, HealScanMode};
|
||||
use rustfs_ecstore::new_object_layer_fn;
|
||||
use rustfs_ecstore::store::get_disk_via_endpoint;
|
||||
use rustfs_ecstore::store_api::BucketInfo;
|
||||
use rustfs_utils::path::path_join;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
use std::cmp::Ordering;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Heal scan mode
|
||||
pub type HealScanMode = usize;
|
||||
|
||||
pub const HEAL_UNKNOWN_SCAN: HealScanMode = 0;
|
||||
pub const HEAL_NORMAL_SCAN: HealScanMode = 1;
|
||||
pub const HEAL_DEEP_SCAN: HealScanMode = 2;
|
||||
|
||||
/// Heal type
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealType {
|
||||
@@ -42,22 +43,13 @@ pub enum HealType {
|
||||
version_id: Option<String>,
|
||||
},
|
||||
/// Bucket heal
|
||||
Bucket {
|
||||
bucket: String,
|
||||
},
|
||||
Bucket { bucket: String },
|
||||
/// Disk heal
|
||||
Disk {
|
||||
endpoint: Endpoint,
|
||||
},
|
||||
Disk { endpoint: Endpoint },
|
||||
/// Metadata heal
|
||||
Metadata {
|
||||
bucket: String,
|
||||
object: String,
|
||||
},
|
||||
Metadata { bucket: String, object: String },
|
||||
/// MRF heal
|
||||
MRF {
|
||||
meta_path: String,
|
||||
},
|
||||
MRF { meta_path: String },
|
||||
/// EC decode heal
|
||||
ECDecode {
|
||||
bucket: String,
|
||||
@@ -119,7 +111,7 @@ impl Default for HealOptions {
|
||||
}
|
||||
|
||||
/// Heal task status
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub enum HealTaskStatus {
|
||||
/// Pending
|
||||
Pending,
|
||||
@@ -174,27 +166,15 @@ impl HealRequest {
|
||||
}
|
||||
|
||||
pub fn bucket(bucket: String) -> Self {
|
||||
Self::new(
|
||||
HealType::Bucket { bucket },
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
)
|
||||
Self::new(HealType::Bucket { bucket }, HealOptions::default(), HealPriority::Normal)
|
||||
}
|
||||
|
||||
pub fn disk(endpoint: Endpoint) -> Self {
|
||||
Self::new(
|
||||
HealType::Disk { endpoint },
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)
|
||||
Self::new(HealType::Disk { endpoint }, HealOptions::default(), HealPriority::High)
|
||||
}
|
||||
|
||||
pub fn metadata(bucket: String, object: String) -> Self {
|
||||
Self::new(
|
||||
HealType::Metadata { bucket, object },
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)
|
||||
Self::new(HealType::Metadata { bucket, object }, HealOptions::default(), HealPriority::High)
|
||||
}
|
||||
|
||||
pub fn ec_decode(bucket: String, object: String, version_id: Option<String>) -> Self {
|
||||
@@ -264,24 +244,20 @@ impl HealTask {
|
||||
info!("Starting heal task: {} with type: {:?}", self.id, self.heal_type);
|
||||
|
||||
let result = match &self.heal_type {
|
||||
HealType::Object { bucket, object, version_id } => {
|
||||
self.heal_object(bucket, object, version_id.as_deref()).await
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
self.heal_bucket(bucket).await
|
||||
}
|
||||
HealType::Disk { endpoint } => {
|
||||
self.heal_disk(endpoint).await
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
self.heal_metadata(bucket, object).await
|
||||
}
|
||||
HealType::MRF { meta_path } => {
|
||||
self.heal_mrf(meta_path).await
|
||||
}
|
||||
HealType::ECDecode { bucket, object, version_id } => {
|
||||
self.heal_ec_decode(bucket, object, version_id.as_deref()).await
|
||||
}
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_object(bucket, object, version_id.as_deref()).await,
|
||||
HealType::Bucket { bucket } => self.heal_bucket(bucket).await,
|
||||
HealType::Disk { endpoint } => self.heal_disk(endpoint).await,
|
||||
HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await,
|
||||
HealType::MRF { meta_path } => self.heal_mrf(meta_path).await,
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => self.heal_ec_decode(bucket, object, version_id.as_deref()).await,
|
||||
};
|
||||
|
||||
// update completed time and status
|
||||
@@ -298,9 +274,7 @@ impl HealTask {
|
||||
}
|
||||
Err(e) => {
|
||||
let mut status = self.status.write().await;
|
||||
*status = HealTaskStatus::Failed {
|
||||
error: e.to_string(),
|
||||
};
|
||||
*status = HealTaskStatus::Failed { error: e.to_string() };
|
||||
error!("Heal task failed: {} with error: {}", self.id, e);
|
||||
}
|
||||
}
|
||||
@@ -327,11 +301,11 @@ impl HealTask {
|
||||
// specific heal implementation method
|
||||
async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!("Healing object: {}/{}", bucket, object);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{}/{}", bucket, object)));
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(0, 4, 0, 0); // 开始heal，总共4个步骤
|
||||
}
|
||||
|
||||
@@ -345,47 +319,24 @@ impl HealTask {
|
||||
return self.recreate_missing_object(bucket, object, version_id).await;
|
||||
} else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {}/{}", bucket, object),
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 4, 0, 0);
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Verify object integrity
|
||||
info!("Step 2: Verifying object integrity");
|
||||
let integrity_ok = self.storage.verify_object_integrity(bucket, object).await?;
|
||||
if integrity_ok {
|
||||
info!("Object integrity check passed: {}/{}", bucket, object);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
warn!("Object integrity check failed: {}/{}", bucket, object);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(2, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 3: Perform actual heal using ecstore
|
||||
info!("Step 3: Performing heal using ecstore");
|
||||
// Step 2: directly call ecstore to perform heal
|
||||
info!("Step 2: Performing heal using ecstore");
|
||||
let heal_opts = rustfs_ecstore::heal::heal_commands::HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: match self.options.scan_mode {
|
||||
crate::heal::task::HEAL_UNKNOWN_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_UNKNOWN_SCAN,
|
||||
crate::heal::task::HEAL_NORMAL_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN,
|
||||
crate::heal::task::HEAL_DEEP_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_DEEP_SCAN,
|
||||
_ => rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN,
|
||||
},
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: false,
|
||||
pool: None,
|
||||
@@ -396,7 +347,7 @@ impl HealTask {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
error!("Heal operation failed: {}/{} - {}", bucket, object, e);
|
||||
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
warn!("Removing corrupted object: {}/{}", bucket, object);
|
||||
@@ -410,29 +361,34 @@ impl HealTask {
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal object {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal object {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Verify heal result
|
||||
info!("Step 4: Verifying heal result");
|
||||
// Step 3: Verify heal result
|
||||
info!("Step 3: Verifying heal result");
|
||||
let object_size = result.object_size as u64;
|
||||
info!("Heal completed successfully: {}/{} ({} bytes, {} drives healed)",
|
||||
bucket, object, object_size, result.after.drives.len());
|
||||
info!(
|
||||
"Heal completed successfully: {}/{} ({} bytes, {} drives healed)",
|
||||
bucket,
|
||||
object,
|
||||
object_size,
|
||||
result.after.drives.len()
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, object_size, object_size);
|
||||
progress.update_progress(3, 3, object_size, object_size);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Heal operation failed: {}/{} - {}", bucket, object, e);
|
||||
|
||||
|
||||
// If heal failed and remove_corrupted is enabled, delete the corrupted object
|
||||
if self.options.remove_corrupted {
|
||||
warn!("Removing corrupted object: {}/{}", bucket, object);
|
||||
@@ -446,11 +402,11 @@ impl HealTask {
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal object {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal object {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -459,7 +415,7 @@ impl HealTask {
|
||||
/// Recreate missing object (for EC decode scenarios)
|
||||
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!("Attempting to recreate missing object: {}/{}", bucket, object);
|
||||
|
||||
|
||||
// Use ecstore's heal_object with recreate option
|
||||
let heal_opts = rustfs_ecstore::heal::heal_commands::HealOpts {
|
||||
recursive: false,
|
||||
@@ -478,7 +434,7 @@ impl HealTask {
|
||||
if let Some(e) = error {
|
||||
error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -494,7 +450,7 @@ impl HealTask {
|
||||
Err(e) => {
|
||||
error!("Failed to recreate missing object: {}/{} - {}", bucket, object, e);
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -502,11 +458,11 @@ impl HealTask {
|
||||
|
||||
async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
info!("Healing bucket: {}", bucket);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("bucket: {}", bucket)));
|
||||
progress.set_current_object(Some(format!("bucket: {bucket}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
@@ -516,7 +472,7 @@ impl HealTask {
|
||||
if !bucket_exists {
|
||||
warn!("Bucket does not exist: {}", bucket);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket not found: {}", bucket),
|
||||
message: format!("Bucket not found: {bucket}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -532,12 +488,7 @@ impl HealTask {
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: match self.options.scan_mode {
|
||||
crate::heal::task::HEAL_UNKNOWN_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_UNKNOWN_SCAN,
|
||||
crate::heal::task::HEAL_NORMAL_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN,
|
||||
crate::heal::task::HEAL_DEEP_SCAN => rustfs_ecstore::heal::heal_commands::HEAL_DEEP_SCAN,
|
||||
_ => rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN,
|
||||
},
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: false,
|
||||
pool: None,
|
||||
@@ -547,7 +498,7 @@ impl HealTask {
|
||||
match self.storage.heal_bucket(bucket, &heal_opts).await {
|
||||
Ok(result) => {
|
||||
info!("Bucket heal completed successfully: {} ({} drives)", bucket, result.after.drives.len());
|
||||
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
@@ -561,7 +512,7 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal bucket {}: {}", bucket, e),
|
||||
message: format!("Failed to heal bucket {bucket}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -569,33 +520,17 @@ impl HealTask {
|
||||
|
||||
async fn heal_disk(&self, endpoint: &Endpoint) -> Result<()> {
|
||||
info!("Healing disk: {:?}", endpoint);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("disk: {:?}", endpoint)));
|
||||
progress.set_current_object(Some(format!("disk: {endpoint:?}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check disk status
|
||||
info!("Step 1: Checking disk status");
|
||||
let disk_status = self.storage.get_disk_status(endpoint).await?;
|
||||
if disk_status == DiskStatus::Ok {
|
||||
info!("Disk is already healthy: {:?}", endpoint);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
// Step 1: Perform disk format heal using ecstore
|
||||
info!("Step 1: Performing disk format heal using ecstore");
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Perform disk heal using ecstore
|
||||
info!("Step 2: Performing disk heal using ecstore");
|
||||
match self.storage.heal_format(self.options.dry_run).await {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
@@ -605,12 +540,21 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk {:?}: {}", endpoint, e),
|
||||
message: format!("Failed to heal disk {endpoint:?}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
info!("Disk heal completed successfully: {:?} ({} drives)", endpoint, result.after.drives.len());
|
||||
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(2, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Synchronize data/buckets on the fresh disk
|
||||
info!("Step 2: Healing buckets on fresh disk");
|
||||
self.heal_fresh_disk(endpoint).await?;
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
@@ -624,7 +568,7 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk {:?}: {}", endpoint, e),
|
||||
message: format!("Failed to heal disk {endpoint:?}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -632,11 +576,11 @@ impl HealTask {
|
||||
|
||||
async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
info!("Healing metadata: {}/{}", bucket, object);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("metadata: {}/{}", bucket, object)));
|
||||
progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
@@ -646,7 +590,7 @@ impl HealTask {
|
||||
if !object_exists {
|
||||
warn!("Object does not exist: {}/{}", bucket, object);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {}/{}", bucket, object),
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -678,12 +622,17 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal metadata {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
info!("Metadata heal completed successfully: {}/{} ({} drives)", bucket, object, result.after.drives.len());
|
||||
|
||||
info!(
|
||||
"Metadata heal completed successfully: {}/{} ({} drives)",
|
||||
bucket,
|
||||
object,
|
||||
result.after.drives.len()
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
@@ -697,7 +646,7 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal metadata {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -705,11 +654,11 @@ impl HealTask {
|
||||
|
||||
async fn heal_mrf(&self, meta_path: &str) -> Result<()> {
|
||||
info!("Healing MRF: {}", meta_path);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("mrf: {}", meta_path)));
|
||||
progress.set_current_object(Some(format!("mrf: {meta_path}")));
|
||||
progress.update_progress(0, 2, 0, 0);
|
||||
}
|
||||
|
||||
@@ -717,7 +666,7 @@ impl HealTask {
|
||||
let parts: Vec<&str> = meta_path.split('/').collect();
|
||||
if parts.len() < 2 {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Invalid meta path format: {}", meta_path),
|
||||
message: format!("Invalid meta path format: {meta_path}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -747,12 +696,12 @@ impl HealTask {
|
||||
progress.update_progress(2, 2, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal MRF {}: {}", meta_path, e),
|
||||
message: format!("Failed to heal MRF {meta_path}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
info!("MRF heal completed successfully: {} ({} drives)", meta_path, result.after.drives.len());
|
||||
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(2, 2, 0, 0);
|
||||
@@ -766,7 +715,7 @@ impl HealTask {
|
||||
progress.update_progress(2, 2, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal MRF {}: {}", meta_path, e),
|
||||
message: format!("Failed to heal MRF {meta_path}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -774,11 +723,11 @@ impl HealTask {
|
||||
|
||||
async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
info!("Healing EC decode: {}/{}", bucket, object);
|
||||
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("ec_decode: {}/{}", bucket, object)));
|
||||
progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
@@ -788,7 +737,7 @@ impl HealTask {
|
||||
if !object_exists {
|
||||
warn!("Object does not exist: {}/{}", bucket, object);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {}/{}", bucket, object),
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -820,14 +769,19 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal EC decode {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
let object_size = result.object_size as u64;
|
||||
info!("EC decode heal completed successfully: {}/{} ({} bytes, {} drives)",
|
||||
bucket, object, object_size, result.after.drives.len());
|
||||
|
||||
info!(
|
||||
"EC decode heal completed successfully: {}/{} ({} bytes, {} drives)",
|
||||
bucket,
|
||||
object,
|
||||
object_size,
|
||||
result.after.drives.len()
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, object_size, object_size);
|
||||
@@ -841,11 +795,86 @@ impl HealTask {
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal EC decode {}/{}: {}", bucket, object, e),
|
||||
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_fresh_disk(&self, endpoint: &Endpoint) -> Result<()> {
|
||||
// Locate disk via endpoint
|
||||
let disk = get_disk_via_endpoint(endpoint)
|
||||
.await
|
||||
.ok_or_else(|| Error::other(format!("Disk not found for endpoint: {endpoint}")))?;
|
||||
|
||||
// Skip if drive is root or other fatal errors
|
||||
if let Err(e) = disk.disk_info(&DiskInfoOptions::default()).await {
|
||||
match e {
|
||||
DiskError::DriveIsRoot => return Ok(()),
|
||||
DiskError::UnformattedDisk => { /* continue healing */ }
|
||||
_ => return Err(Error::other(e)),
|
||||
}
|
||||
}
|
||||
|
||||
// Load or init HealingTracker
|
||||
let mut tracker = match load_healing_tracker(&Some(disk.clone())).await {
|
||||
Ok(t) => t,
|
||||
Err(err) => match err {
|
||||
DiskError::FileNotFound => init_healing_tracker(disk.clone(), &Uuid::new_v4().to_string())
|
||||
.await
|
||||
.map_err(Error::other)?,
|
||||
_ => return Err(Error::other(err)),
|
||||
},
|
||||
};
|
||||
|
||||
// Build bucket list
|
||||
let mut buckets = self.storage.list_buckets().await.map_err(Error::other)?;
|
||||
buckets.push(BucketInfo {
|
||||
name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(RUSTFS_CONFIG_PREFIX)])
|
||||
.to_string_lossy()
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
buckets.push(BucketInfo {
|
||||
name: path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)])
|
||||
.to_string_lossy()
|
||||
.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Sort: system buckets first, others by creation time desc
|
||||
buckets.sort_by(|a, b| {
|
||||
let a_sys = a.name.starts_with(RUSTFS_META_BUCKET);
|
||||
let b_sys = b.name.starts_with(RUSTFS_META_BUCKET);
|
||||
match (a_sys, b_sys) {
|
||||
(true, false) => Ordering::Less,
|
||||
(false, true) => Ordering::Greater,
|
||||
_ => b.created.cmp(&a.created),
|
||||
}
|
||||
});
|
||||
|
||||
// Update tracker queue and persist
|
||||
tracker.set_queue_buckets(&buckets).await;
|
||||
tracker.save().await.map_err(Error::other)?;
|
||||
|
||||
// Prepare bucket names list
|
||||
let bucket_names: Vec<String> = buckets.iter().map(|b| b.name.clone()).collect();
|
||||
|
||||
// Run heal_erasure_set using underlying SetDisk
|
||||
let (pool_idx, set_idx) = (endpoint.pool_idx as usize, endpoint.set_idx as usize);
|
||||
let Some(store) = new_object_layer_fn() else {
|
||||
return Err(Error::other("errServerNotInitialized"));
|
||||
};
|
||||
let set_disk = store.pools[pool_idx].disk_set[set_idx].clone();
|
||||
|
||||
let tracker_arc = Arc::new(RwLock::new(tracker));
|
||||
set_disk
|
||||
.heal_erasure_set(&bucket_names, tracker_arc)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HealTask {
|
||||
@@ -857,4 +886,4 @@ impl std::fmt::Debug for HealTask {
|
||||
.field("created_at", &self.created_at)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user