mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
fix(heal): defer scoped repair on suspended pools (#5876)
This commit is contained in:
@@ -3842,6 +3842,36 @@ mod tests {
|
||||
assert!(retry_error.contains("Storage resources are insufficient"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_request_for_scoped_slowdown_preserves_scope() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
);
|
||||
let result = Err(Error::Storage(EcstoreError::SlowDown));
|
||||
|
||||
let (retry_request, retry_delay, _) =
|
||||
retry_request_for_result(&task, &result).expect("SlowDown should defer scoped heal");
|
||||
|
||||
assert_eq!(retry_request.options.pool_index, Some(0));
|
||||
assert_eq!(retry_request.options.set_index, Some(1));
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_request_for_typed_not_found_error_is_not_retryable() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -2517,6 +2517,7 @@ mod tests {
|
||||
OkWithOtherError(&'static str),
|
||||
ErrOther(&'static str),
|
||||
RetryableReadQuorum,
|
||||
RetryableSlowDown,
|
||||
PermanentOther(&'static str),
|
||||
}
|
||||
|
||||
@@ -2650,6 +2651,9 @@ mod tests {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
|
||||
MockHealObjectOutcome::OkWithOtherError(message) => {
|
||||
Ok((HealResultItem::default(), Some(Error::other(message))))
|
||||
@@ -2669,6 +2673,9 @@ mod tests {
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
};
|
||||
}
|
||||
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
|
||||
@@ -2761,6 +2768,42 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_object_heal_slowdown_is_not_treated_as_deleted() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
object_exists: Mutex::new(Some(true)),
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::RetryableSlowDown)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
let err = task.execute().await.expect_err("SlowDown must fail the current heal attempt");
|
||||
|
||||
assert!(matches!(err, Error::Storage(EcstoreError::SlowDown)));
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Failed { .. }));
|
||||
let opts = storage
|
||||
.object_heal_opts
|
||||
.lock()
|
||||
.expect("heal options lock should be available");
|
||||
assert_eq!(opts[0].pool, Some(0));
|
||||
assert_eq!(opts[0].set, Some(1));
|
||||
}
|
||||
|
||||
async fn make_resume_disk(temp: &TempDir) -> DiskStore {
|
||||
let disk_path = temp.path().join("test_disk");
|
||||
std::fs::create_dir_all(&disk_path).expect("test disk directory should be created");
|
||||
|
||||
Reference in New Issue
Block a user