mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
fix(heal): fence format repair during pool transitions
This commit is contained in:
@@ -231,6 +231,10 @@ impl HealTask {
|
||||
"Heal erasure set format repair skipped because no format heal was required"
|
||||
);
|
||||
} else {
|
||||
let error = e;
|
||||
if error.is_recoverable_heal() {
|
||||
return Err(error);
|
||||
}
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
@@ -239,7 +243,7 @@ impl HealTask {
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_failed",
|
||||
error = %e,
|
||||
error = %error,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
{
|
||||
@@ -247,7 +251,7 @@ impl HealTask {
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
@@ -284,6 +288,9 @@ impl HealTask {
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
if e.is_recoverable_heal() {
|
||||
return Err(e);
|
||||
}
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
|
||||
@@ -547,6 +547,7 @@ struct MockStorage {
|
||||
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
||||
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
replacement_format_calls: Mutex<Vec<(usize, usize, Vec<String>)>>,
|
||||
replacement_targets_ready: Mutex<bool>,
|
||||
@@ -867,6 +868,9 @@ impl HealStorageAPI for MockStorage {
|
||||
|
||||
async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
*self.global_format_calls.lock().unwrap() += 1;
|
||||
if let Some(error) = self.format_error.lock().unwrap().take() {
|
||||
return Err(error);
|
||||
}
|
||||
let no_heal_required = *self.format_no_heal_required.lock().unwrap();
|
||||
if no_heal_required {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::NoHealRequired))))
|
||||
@@ -2052,6 +2056,30 @@ async fn test_erasure_set_heal_continues_after_format_no_heal_required() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_format_slowdown_is_propagated() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
format_error: Mutex::new(Some(Error::Storage(EcstoreError::SlowDown))),
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("format SlowDown must remain recoverable for the task manager");
|
||||
|
||||
assert!(matches!(error, Error::Storage(EcstoreError::SlowDown)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
|
||||
Reference in New Issue
Block a user