fix(heal): recover replacement after transient disk errors (#7059)

* fix(heal): recover replacement after transient disk errors

* fix(heal): satisfy replacement status clippy lint

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -13,6 +13,13 @@
 // limitations under the License.
 /// erasure-set heal: drives the ErasureSetHealer across the set's buckets
 use super::*;
+use crate::heal::DiskStore;
+
+pub(super) async fn load_verified_replacement_resume(disk: &DiskStore, task_id: &str) -> Result<ResumeManager> {
+    let resume_manager = ResumeManager::load_replacement_intent(disk.clone(), task_id).await?;
+    resume_manager.ensure_replacement_completion_proof().await?;
+    Ok(resume_manager)
+}
 
 impl HealTask {
     pub(super) async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
@@ -445,14 +452,16 @@ impl HealTask {
                     self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "marker completion")
                         .await?;
                 }
+                let verified_replacement_resume = if let Some((disk, _, _)) = replacement_resume.as_ref() {
+                    Some((disk.clone(), load_verified_replacement_resume(disk, &self.id).await?))
+                } else {
+                    None
+                };
                 super::super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?;
-                if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
+                if let Some((disk, resume_manager)) = verified_replacement_resume {
                     resume_manager.mark_replacement_cleanup_pending().await?;
-                    if CheckpointManager::has_checkpoint(disk, &self.id).await {
-                        CheckpointManager::load_from_disk(disk.clone(), &self.id)
-                            .await?
-                            .cleanup()
-                            .await?;
+                    if CheckpointManager::has_checkpoint(&disk, &self.id).await {
+                        CheckpointManager::load_from_disk(disk, &self.id).await?.cleanup().await?;
                     }
                     resume_manager.cleanup().await?;
                 }

crates/heal/src/heal/task/tests.rs

@@ -469,6 +469,50 @@ async fn cleanup_pending_recovery_removes_checkpoint_without_rebuild_work() {
     assert!(!*storage.listed.lock().unwrap());
 }
 
+#[tokio::test]
+async fn replacement_cleanup_reloads_verified_state_from_survivor_anchor() {
+    let temp = TempDir::new().expect("temporary resume disk directory should be created");
+    let anchor = make_resume_disk(&temp).await;
+    let task_id = crate::heal::resume::ResumeUtils::generate_task_id();
+    let stale_resume = ResumeManager::new_replacement_intent(
+        anchor.clone(),
+        task_id.clone(),
+        "pool_0_set_0".to_string(),
+        vec!["bucket-a".to_string()],
+        vec!["replacement-a".to_string()],
+        vec![replacement_identity("replacement-a", "device-a", "filesystem-a")],
+    )
+    .await
+    .expect("replacement intent should persist on the survivor anchor");
+    let rebuilding_resume = ResumeManager::load_replacement_intent(anchor.clone(), &task_id)
+        .await
+        .expect("the inner healer should load the persisted replacement intent");
+    rebuilding_resume
+        .mark_replacement_completed_and_verified()
+        .await
+        .expect("the inner healer should persist verified completion");
+
+    assert!(
+        stale_resume.mark_replacement_cleanup_pending().await.is_err(),
+        "the original in-memory manager must remain stale after another manager persists verification"
+    );
+    let verified_resume = super::heal_erasure_set::load_verified_replacement_resume(&anchor, &task_id)
+        .await
+        .expect("cleanup should reload and verify durable replacement completion");
+    verified_resume
+        .mark_replacement_cleanup_pending()
+        .await
+        .expect("the freshly loaded verified state should enter cleanup");
+
+    let state = ResumeManager::load_replacement_intent(anchor, &task_id)
+        .await
+        .expect("cleanup-pending state should remain durable")
+        .get_state()
+        .await;
+    assert!(state.completed);
+    assert_eq!(state.replacement_phase, ReplacementPhase::CleanupPending);
+}
+
 #[tokio::test]
 async fn verified_recovery_keeps_state_when_marker_clear_fails() {
     let temp = TempDir::new().expect("temporary resume disk directory should be created");