fix(heal): defer scoped repair on suspended pools (#5876)

This commit is contained in:
cxymds
2026-08-09 11:50:17 +08:00
committed by GitHub
parent 10c7476883
commit 47369ff027
4 changed files with 227 additions and 2 deletions
+30
View File
@@ -3842,6 +3842,36 @@ mod tests {
assert!(retry_error.contains("Storage resources are insufficient"));
}
#[test]
fn test_retry_request_for_scoped_slowdown_preserves_scope() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let task = HealTask::from_request(
HealRequest::new(
HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
},
HealOptions {
pool_index: Some(0),
set_index: Some(1),
..Default::default()
},
HealPriority::Normal,
),
storage,
);
let result = Err(Error::Storage(EcstoreError::SlowDown));
let (retry_request, retry_delay, _) =
retry_request_for_result(&task, &result).expect("SlowDown should defer scoped heal");
assert_eq!(retry_request.options.pool_index, Some(0));
assert_eq!(retry_request.options.set_index, Some(1));
assert_eq!(retry_request.retry_attempts, 1);
assert!(retry_delay > Duration::ZERO);
}
#[test]
fn test_retry_request_for_typed_not_found_error_is_not_retryable() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+43
View File
@@ -2517,6 +2517,7 @@ mod tests {
OkWithOtherError(&'static str),
ErrOther(&'static str),
RetryableReadQuorum,
RetryableSlowDown,
PermanentOther(&'static str),
}
@@ -2650,6 +2651,9 @@ mod tests {
bucket.to_string(),
object.to_string(),
))),
MockHealObjectOutcome::RetryableSlowDown => {
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
}
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
MockHealObjectOutcome::OkWithOtherError(message) => {
Ok((HealResultItem::default(), Some(Error::other(message))))
@@ -2669,6 +2673,9 @@ mod tests {
bucket.to_string(),
object.to_string(),
))),
MockHealObjectOutcome::RetryableSlowDown => {
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
}
};
}
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
@@ -2761,6 +2768,42 @@ mod tests {
}
}
#[tokio::test]
async fn scoped_object_heal_slowdown_is_not_treated_as_deleted() {
let storage = Arc::new(MockStorage {
object_exists: Mutex::new(Some(true)),
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::RetryableSlowDown)),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
},
HealOptions {
pool_index: Some(0),
set_index: Some(1),
..Default::default()
},
HealPriority::Normal,
),
storage.clone(),
);
let err = task.execute().await.expect_err("SlowDown must fail the current heal attempt");
assert!(matches!(err, Error::Storage(EcstoreError::SlowDown)));
assert!(matches!(task.get_status().await, HealTaskStatus::Failed { .. }));
let opts = storage
.object_heal_opts
.lock()
.expect("heal options lock should be available");
assert_eq!(opts[0].pool, Some(0));
assert_eq!(opts[0].set, Some(1));
}
async fn make_resume_disk(temp: &TempDir) -> DiskStore {
let disk_path = temp.path().join("test_disk");
std::fs::create_dir_all(&disk_path).expect("test disk directory should be created");