From adfde0110b99a15f26c18be4f0bf7aa949e526cb Mon Sep 17 00:00:00 2001 From: houseme Date: Sat, 12 Sep 2026 18:07:33 +0800 Subject: [PATCH] test(heal): retry G14 deferred outage put (#7703) Treat SlowDownRead as a bounded retryable deferred PUT response after the target pool rejoins, and label terminal deferred outage PUT failures with stage context. Co-authored-by: zhi22915 --- .../src/heal_erasure_disk_rebuild_test.rs | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index c3fe0c54d..bd63572ba 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -685,6 +685,13 @@ mod tests { error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable") } + fn is_retryable_deferred_put_after_rejoin(error: &SdkError) -> bool { + matches!( + error.as_service_error().and_then(ProvideErrorMetadata::code), + Some("SlowDownRead" | "ServiceUnavailable") + ) + } + fn is_service_unavailable_delete(error: &SdkError) -> bool { error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable") } @@ -2181,11 +2188,21 @@ mod tests { .await; match put_result { Ok(Ok(_)) => break, - Ok(Err(error)) if is_service_unavailable_put(&error) && Instant::now() < deferred_deadline => { + Ok(Err(error)) if is_retryable_deferred_put_after_rejoin(&error) && Instant::now() < deferred_deadline => { sleep(Duration::from_secs(1)).await; } - Ok(Err(error)) => return Err(error.into()), - Err(error) => return Err(error.into()), + Ok(Err(error)) => { + return Err(format!( + "deferred outage PUT failed for {bucket}/{outage_key} after target rejoin and up to 60s wait: {error}" + ) + .into()); + } + Err(error) => { + return Err(format!( + "deferred outage PUT timed out for {bucket}/{outage_key} after 30s attempt: {error}" + ) + .into()); + } } } info!(