From c1d9a075ded902c0565ba24b80551d99004929ff Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 8 Sep 2026 07:21:51 +0800 Subject: [PATCH] test(heal): cover admin lock timeout progress (#7427) * test(heal): cover admin lock timeout progress Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(error): merge equivalent api message branches Co-Authored-By: heihutu Co-Authored-By: zhi22915 * fix(heal): cleanup consumed MRF replay journals Do not retain Accepted or Merged replay intents as startup anchors after they have been handed to the heal manager. Only refused or still-pending replay records keep the journal on disk until a successor snapshot can persist them. This keeps successor snapshots limited to the pending queue, which lets successful replay remove both authoritative and legacy journal paths and restores the crash-boundary tests around successor flush. Co-Authored-By: heihutu Co-Authored-By: zhi22915 (cherry picked from commit d5b8f49c9d63afafa8e86a1ac085f9af96ec6406) --------- Co-authored-by: zhi22915 Co-authored-by: Zhengchao An --- crates/heal/src/heal/task/tests.rs | 81 ++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 30ac040ba..01616f087 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -183,6 +183,72 @@ mod canonical_outcome { assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); } + #[tokio::test(start_paused = true)] + async fn admin_cluster_lock_timeout_exhaustion_keeps_progress_and_retry_outcome() { + let storage = Arc::new(MockStorage::default()); + storage.heal_object_outcomes.lock().expect("outcomes").insert( + "object-a".to_string(), + (0..4).map(|_| MockHealObjectOutcome::RetryableLockTimeout).collect(), + ); + let mut request = HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + request.source = HealRequestSource::Admin; + let task = HealTask::from_request(request, storage.clone()); + + let err = task + .execute() + .await + .expect_err("legacy adapter still returns the batch failure detail"); + assert!( + err.to_string() + .contains("Lock error: Lock acquisition timeout for resource 'object-a' after 5s"), + "lock timeout must remain actionable in the retained failure detail: {err}" + ); + + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!( + ( + outcome.counters.processed, + outcome.counters.failed, + outcome.counters.unknown, + outcome.counters.attempt_failures + ), + (2, 1, 1, 4) + ); + let failed = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("lock-contended object outcome"); + assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)); + assert!( + failed + .detail + .as_deref() + .is_some_and(|detail| detail.contains("Lock error: Lock acquisition timeout for resource 'object-a' after 5s")), + "exhausted lock detail stays observable" + ); + + let progress = task.get_progress().await; + assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); + let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None); + assert_eq!(legacy_summary, "stopped"); + assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects")); + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a", "object-b", "object-a", "object-a", "object-a"] + ); + } + #[tokio::test(start_paused = true)] async fn retry_success_counts_one_terminal_outcome() { let storage = Arc::new(MockStorage::default()); @@ -1293,6 +1359,7 @@ fn replacement_identity( enum MockHealObjectOutcome { RetryableLock, + RetryableLockTimeout, OkWithOtherError(&'static str), ErrOther(&'static str), DanglingGraceDeferred, @@ -1436,6 +1503,13 @@ impl HealStorageAPI for MockStorage { owner: "competing-writer".to_string(), }))), )), + MockHealObjectOutcome::RetryableLockTimeout => Ok(( + HealResultItem::default(), + Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::Timeout { + resource: object.to_string(), + timeout: Duration::from_secs(5), + }))), + )), MockHealObjectOutcome::RetryableSlowDown => { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown)))) } @@ -1468,6 +1542,13 @@ impl HealStorageAPI for MockStorage { owner: "competing-writer".to_string(), }))), )), + MockHealObjectOutcome::RetryableLockTimeout => Ok(( + HealResultItem::default(), + Some(Error::Storage(EcstoreError::Lock(rustfs_lock::LockError::Timeout { + resource: object.to_string(), + timeout: Duration::from_secs(5), + }))), + )), MockHealObjectOutcome::RetryableSlowDown => { Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown)))) }