fix(heal): reconcile dangling objects after node reconnect (#6567)

crates/ecstore/src/set_disk/ops/heal.rs

@@ -779,16 +779,11 @@ impl SetDisks {
                                 .await
                             {
                                 Ok(m) => {
-                                    let derr = if !version_id.is_empty() {
-                                        DiskError::FileVersionNotFound
-                                    } else {
-                                        DiskError::FileNotFound
-                                    };
                                     let mut t_errs = Vec::with_capacity(errs.len());
                                     for _ in 0..errs.len() {
                                         t_errs.push(None);
                                     }
-                                    Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
+                                    Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, None))
                                 }
                                 Err(err) => {
                                     error!(
@@ -798,10 +793,9 @@ impl SetDisks {
                                         object,
                                         version_id,
                                         error = %err,
-                                        returned_error = %cannot_heal_err,
                                         "Heal object dangling cleanup could not prove object deletion"
                                     );
-                                    Ok((result, Some(cannot_heal_err)))
+                                    Ok((result, Some(err)))
                                 }
                             };
                         }
@@ -1273,6 +1267,14 @@ impl SetDisks {
                     return Box::pin(self.heal_object_with_explicit_version_regen(bucket, object, version_id, opts, false)).await;
                 }
 
+                if opts.dry_run {
+                    return Ok((
+                        self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
+                            .await,
+                        Some(err),
+                    ));
+                }
+
                 if self
                     .dangling_delete_safety(bucket, object, &parts_metadata, &errs, &disks)
                     .await?
@@ -1300,18 +1302,11 @@ impl SetDisks {
                     )
                     .await
                 {
-                    Ok(m) => {
+                    Ok(m) => Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, None)),
-                        let err = if !version_id.is_empty() {
+                    Err(cleanup_err) => Ok((
-                            DiskError::FileVersionNotFound
-                        } else {
-                            DiskError::FileNotFound
-                        };
-                        Ok((self.default_heal_result(m, &errs, bucket, object, version_id).await, Some(err)))
-                    }
-                    Err(_) => Ok((
                         self.default_heal_result(FileInfo::default(), &errs, bucket, object, version_id)
                             .await,
-                        Some(err),
+                        Some(cleanup_err),
                     )),
                 }
             }
@@ -2123,7 +2118,7 @@ mod heal_result_report_tests {
     use crate::disk::endpoint::Endpoint;
     use crate::disk::error::DiskError;
     use crate::disk::format::FormatV3;
-    use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, new_disk};
+    use crate::disk::{DiskAPI as _, DiskOption, DiskStore, RUSTFS_META_TMP_BUCKET, ReadOptions, STORAGE_FORMAT_FILE, new_disk};
     use crate::error::Error;
     use crate::object_api::{ObjectOptions, PutObjReader};
     use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
@@ -2134,6 +2129,7 @@ mod heal_result_report_tests {
         config::storageclass,
         store::init_format::{load_format_erasure, save_format_file},
     };
+    use bytes::Bytes;
     use rustfs_common::heal_channel::{DriveState, HealOpts, HealScanMode};
     use rustfs_filemeta::{BLOCK_SIZE_V2, FileInfo, ObjectPartInfo, TRANSITION_COMPLETE};
     use std::sync::{Arc, Mutex};
@@ -2394,6 +2390,20 @@ mod heal_result_report_tests {
             .expect("test metadata should be written");
     }
 
+    async fn dangling_inline_test_fixture(
+        bucket: &str,
+        object: &str,
+        mod_time: OffsetDateTime,
+    ) -> (Vec<TempDir>, Arc<SetDisks>, Vec<Option<DiskStore>>) {
+        let (temp_dirs, set, disks) = meta_regen_test_set(bucket, object, &[]).await;
+        let mut metadata = meta_regen_test_fileinfo(object, Uuid::nil(), mod_time.unix_timestamp(), 0);
+        metadata.data_dir = None;
+        metadata.set_inline_data();
+        metadata.data = Some(Bytes::from_static(b"stale-inline-shard"));
+        seed_meta_regen_test_metadata(&disks, 0, bucket, object, &metadata).await;
+        (temp_dirs, set, disks)
+    }
+
     async fn formatted_single_disk_no_parity_set() -> (TempDir, Arc<SetDisks>) {
         let format = FormatV3::new(1, 1);
         let dir = tempfile::tempdir().expect("tempdir should be created");
@@ -3068,6 +3078,109 @@ mod heal_result_report_tests {
         );
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn heal_reports_success_after_dangling_inline_cleanup() {
+        temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async {
+            let bucket = "bucket-dangling-inline-cleanup";
+            let object = "stale.txt";
+            let (temp_dirs, set, _disks) =
+                dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await;
+
+            let (_, error) = set
+                .heal_object(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("dangling cleanup should complete");
+
+            assert!(error.is_none(), "successful dangling cleanup must not be reported as FileNotFound");
+            assert!(
+                temp_dirs.iter().all(|dir| !dir.path().join(bucket).join(object).exists()),
+                "successful dangling cleanup must remove the stale object from every disk"
+            );
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn heal_preserves_recent_dangling_inline_metadata_as_retryable() {
+        temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("3600"))], async {
+            let bucket = "bucket-dangling-inline-grace";
+            let object = "recent.txt";
+            let (temp_dirs, set, _disks) = dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc()).await;
+
+            let (_, error) = set
+                .heal_object(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("grace-protected dangling metadata should return a typed heal result");
+
+            assert_eq!(error, Some(DiskError::ErasureReadQuorum));
+            assert!(
+                temp_dirs[0]
+                    .path()
+                    .join(bucket)
+                    .join(object)
+                    .join(STORAGE_FORMAT_FILE)
+                    .is_file(),
+                "grace-protected metadata must remain for a later heal pass"
+            );
+        })
+        .await;
+    }
+
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn dry_run_never_deletes_dangling_inline_metadata() {
+        temp_env::async_with_vars([("RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS", Some("0"))], async {
+            let bucket = "bucket-dangling-inline-dry-run";
+            let object = "stale.txt";
+            let (temp_dirs, set, _disks) =
+                dangling_inline_test_fixture(bucket, object, OffsetDateTime::now_utc() - time::Duration::hours(2)).await;
+
+            let (_, error) = set
+                .heal_object(
+                    bucket,
+                    object,
+                    "",
+                    &HealOpts {
+                        dry_run: true,
+                        no_lock: true,
+                        ..Default::default()
+                    },
+                )
+                .await
+                .expect("dry-run should report the dangling candidate without mutation");
+
+            assert_eq!(error, Some(DiskError::FileNotFound));
+            assert!(
+                temp_dirs[0]
+                    .path()
+                    .join(bucket)
+                    .join(object)
+                    .join(STORAGE_FORMAT_FILE)
+                    .is_file(),
+                "dry-run must preserve dangling metadata even when grace is disabled"
+            );
+        })
+        .await;
+    }
+
     #[tokio::test]
     async fn heal_meta_quorum_failure_preserves_candidate_when_required_shard_disk_is_offline() {
         let bucket = "bucket-delete-guard-offline";

crates/ecstore/src/store/heal.rs

@@ -168,6 +168,60 @@ impl ECStore {
         }
     }
 
+    /// Return every live erasure set selected by an object-heal scope.
+    pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
+        let pools = self.get_pools_for_heal_object(opts)?;
+        let pool_meta = self.pool_meta.read().await;
+        let mut scopes = Vec::new();
+
+        for pool in pools {
+            let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
+                pool_meta
+                    .pools
+                    .get(pool.pool_idx)
+                    .and_then(|status| status.decommission.as_ref())
+                    .is_some_and(|decommission| decommission.complete)
+            });
+            if let Some(complete) = suspended_complete {
+                if opts.pool.is_some() {
+                    return Err(if complete {
+                        StorageError::InvalidArgument(
+                            "heal".to_string(),
+                            "pool".to_string(),
+                            format!("heal pool {} has completed decommission", pool.pool_idx),
+                        )
+                    } else {
+                        Error::SlowDown
+                    });
+                }
+                continue;
+            }
+
+            if let Some(set_idx) = opts.set {
+                if set_idx >= pool.disk_set.len() {
+                    return Err(StorageError::InvalidArgument(
+                        "heal".to_string(),
+                        "set".to_string(),
+                        format!(
+                            "invalid heal set index {set_idx} for pool {} with {} sets",
+                            pool.pool_idx,
+                            pool.disk_set.len()
+                        ),
+                    ));
+                }
+                scopes.push((pool.pool_idx, set_idx));
+            } else {
+                scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
+            }
+        }
+
+        if scopes.is_empty() {
+            return Err(Error::SlowDown);
+        }
+
+        Ok(scopes)
+    }
+
     #[instrument(skip(self))]
     pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
         let mut r = HealResultItem {
@@ -642,6 +696,40 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
+        let store = minimal_heal_store().await;
+
+        assert_eq!(
+            store
+                .heal_erasure_set_scopes(&HealOpts::default())
+                .await
+                .expect("unscoped heal should enumerate every live set"),
+            vec![(0, 0), (1, 0)]
+        );
+        assert_eq!(
+            store
+                .heal_erasure_set_scopes(&HealOpts {
+                    pool: Some(1),
+                    set: Some(0),
+                    ..Default::default()
+                })
+                .await
+                .expect("scoped heal should enumerate only its requested set"),
+            vec![(1, 0)]
+        );
+
+        let err = store
+            .heal_erasure_set_scopes(&HealOpts {
+                pool: Some(0),
+                set: Some(1),
+                ..Default::default()
+            })
+            .await
+            .expect_err("an invalid set scope must fail closed");
+        assert!(matches!(err, Error::InvalidArgument(..)));
+    }
+
     fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
         PoolMeta {
             pools: vec![PoolStatus {

crates/heal/src/heal/storage.rs

@@ -428,6 +428,16 @@ pub trait HealStorageAPI: Send + Sync {
         include_lifecycle_object_info: bool,
     ) -> Result<(Vec<HealListItem>, Option<String>, bool)>;
 
+    /// Return the live erasure sets selected by this heal request.
+    ///
+    /// Recursive admin heals use these scopes with the cross-disk union walk so
+    /// objects surviving on only one returning disk are still discovered. The
+    /// `None` default preserves the read-quorum listing for alternate backends;
+    /// `Some(Vec::new())` means the selected topology currently has no live set.
+    async fn heal_erasure_set_scopes(&self, _opts: &HealOpts) -> Result<Option<Vec<(usize, usize)>>> {
+        Ok(None)
+    }
+
     /// List versions for healing via a per-erasure-set DISK-WALK union enumerator
     /// (backlog#920). Unlike `list_objects_for_heal_page` (which reflects only the
     /// READ-QUORUM metadata view via `list_object_versions`), this surfaces every
@@ -1294,6 +1304,14 @@ impl HealStorageAPI for ECStoreHealStorage {
         Ok((page_objects, next_token, list_info.is_truncated))
     }
 
+    async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Option<Vec<(usize, usize)>>> {
+        self.ecstore
+            .heal_erasure_set_scopes(opts)
+            .await
+            .map(Some)
+            .map_err(Error::Storage)
+    }
+
     async fn list_versions_for_heal_page_disk_walk(
         &self,
         set_disk_id: &str,

crates/heal/src/heal/task/heal_bucket.rs

@@ -14,6 +14,7 @@
 /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
 use super::*;
 use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
+use crate::heal::utils::format_set_disk_id;
 
 impl HealTask {
     pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
@@ -242,7 +243,6 @@ impl HealTask {
 
     #[hotpath::measure]
     async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
-        let mut continuation_token: Option<String> = None;
         let mut scanned = 0u64;
         let mut healed = 0u64;
         let mut failed = 0u64;
@@ -266,108 +266,101 @@ impl HealTask {
             set: self.options.set_index,
         };
 
-        loop {
+        let erasure_set_scopes = self
-            self.check_control_flags().await?;
+            .await_with_control(self.storage.heal_erasure_set_scopes(&heal_opts))
-            let (objects, next_token, is_truncated) = self
+            .await?;
-                .await_with_control(
+        let listing_scopes = match erasure_set_scopes {
-                    self.storage
+            None => vec![(None, heal_opts)],
-                        .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
+            Some(erasure_set_scopes) => erasure_set_scopes
-                )
+                .into_iter()
-                .await?;
+                .map(|(pool_idx, set_idx)| {
+                    let mut scoped_opts = heal_opts;
+                    scoped_opts.pool = Some(pool_idx);
+                    scoped_opts.set = Some(set_idx);
+                    (Some(format_set_disk_id(pool_idx, set_idx)), scoped_opts)
+                })
+                .collect(),
+        };
 
-            let mut pending = objects;
+        for (set_disk_id, heal_opts) in listing_scopes {
-            let mut retry_attempt = 0_u32;
+            let mut continuation_token: Option<String> = None;
-            while !pending.is_empty() {
+            loop {
-                if retry_attempt > 0 {
+                self.check_control_flags().await?;
-                    self.await_with_control(async {
+                let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() {
-                        tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
+                    self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk(
-                        Ok(())
+                        set_disk_id,
-                    })
+                        bucket,
-                    .await?;
+                        prefix,
-                }
+                        continuation_token.as_deref(),
-                let mut retry = Vec::with_capacity(pending.len());
+                        false,
-                for item in pending {
+                    ))
-                    self.check_control_flags().await?;
+                    .await?
-                    let mut telemetry_unknown = false;
+                } else {
-                    let object = item.name.as_str();
+                    self.await_with_control(self.storage.list_objects_for_heal_page(
-                    {
+                        bucket,
-                        let mut progress = self.progress.write().await;
+                        prefix,
-                        progress.set_current_object(Some(format!("{bucket}/{object}")));
+                        continuation_token.as_deref(),
+                        false,
+                    ))
+                    .await?
+                };
+
+                let mut pending = objects;
+                let mut retry_attempt = 0_u32;
+                while !pending.is_empty() {
+                    if retry_attempt > 0 {
+                        self.await_with_control(async {
+                            tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
+                            Ok(())
+                        })
+                        .await?;
                     }
-
+                    let mut retry = Vec::with_capacity(pending.len());
-                    let mut terminal_outcome = true;
+                    for item in pending {
-                    let error = match self
+                        self.check_control_flags().await?;
-                        .await_with_control(
+                        let mut telemetry_unknown = false;
-                            self.storage
+                        let object = item.name.as_str();
-                                .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
+                        {
-                        )
+                            let mut progress = self.progress.write().await;
-                        .await
+                            progress.set_current_object(Some(format!("{bucket}/{object}")));
-                    {
-                        Ok((result, None)) => {
-                            telemetry_unknown |= !increment_counter(&mut healed);
-                            telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
-                            self.record_result_item(result).await;
-                            None
                         }
-                        Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
-                            telemetry_unknown |= !increment_counter(&mut healed);
-                            debug!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object,
-                                result = "object_dir_not_found_skipped",
-                                "Heal bucket object-dir candidate skipped after not-found result"
-                            );
-                            None
-                        }
-                        Ok((_, Some(err))) | Err(err) => Some(err),
-                    };
 
-                    if let Some(err) = error {
+                        let mut terminal_outcome = true;
-                        if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                        let error = match self
-                            telemetry_unknown |= !increment_counter(&mut skipped);
+                            .await_with_control(
-                            warn!(
+                                self.storage
-                                target: "rustfs::heal::task",
+                                    .heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
-                                event = EVENT_HEAL_BUCKET_RESULT,
+                            )
-                                component = LOG_COMPONENT_HEAL,
+                            .await
-                                subsystem = LOG_SUBSYSTEM_TASK,
+                        {
-                                task_id = %self.id,
+                            Ok((result, None)) => {
-                                bucket,
+                                telemetry_unknown |= !increment_counter(&mut healed);
-                                object,
+                                telemetry_unknown |=
-                                result = "transient_skip",
+                                    !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
-                                error = %err,
+                                self.record_result_item(result).await;
-                                "Heal bucket object repair skipped due to transient metadata error"
+                                None
-                            );
-                        } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
-                            terminal_outcome = false;
-                            debug!(
-                                target: "rustfs::heal::task",
-                                event = EVENT_HEAL_BUCKET_RESULT,
-                                component = LOG_COMPONENT_HEAL,
-                                subsystem = LOG_SUBSYSTEM_TASK,
-                                task_id = %self.id,
-                                bucket,
-                                object,
-                                retry_attempt = retry_attempt.saturating_add(1),
-                                error = %err,
-                                result = "object_retry_scheduled",
-                                "Heal bucket object retry scheduled"
-                            );
-                            retry.push(item);
-                        } else {
-                            telemetry_unknown |= !increment_counter(&mut failed);
-                            if err.is_recoverable_heal() {
-                                retryable_failed = retryable_failed.saturating_add(1);
-                            } else {
-                                permanent_failed = permanent_failed.saturating_add(1);
                             }
-                            first_failed_object.get_or_insert_with(|| object.to_string());
+                            Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
-                            first_error.get_or_insert_with(|| err.to_string());
+                                telemetry_unknown |= !increment_counter(&mut healed);
-                            if take_failure_log_sample(&mut failure_samples_logged) {
+                                debug!(
+                                    target: "rustfs::heal::task",
+                                    event = EVENT_HEAL_BUCKET_RESULT,
+                                    component = LOG_COMPONENT_HEAL,
+                                    subsystem = LOG_SUBSYSTEM_TASK,
+                                    task_id = %self.id,
+                                    bucket,
+                                    object,
+                                    result = "object_dir_not_found_skipped",
+                                    "Heal bucket object-dir candidate skipped after not-found result"
+                                );
+                                None
+                            }
+                            Ok((_, Some(err))) | Err(err) => Some(err),
+                        };
+
+                        if let Some(err) = error {
+                            if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
+                                telemetry_unknown |= !increment_counter(&mut skipped);
                                 warn!(
                                     target: "rustfs::heal::task",
                                     event = EVENT_HEAL_BUCKET_RESULT,
@@ -376,41 +369,80 @@ impl HealTask {
                                     task_id = %self.id,
                                     bucket,
                                     object,
-                                    retry_attempt,
+                                    result = "transient_skip",
                                     error = %err,
-                                    result = "object_failed",
+                                    "Heal bucket object repair skipped due to transient metadata error"
-                                    "Heal bucket object repair failed"
                                 );
+                            } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
+                                terminal_outcome = false;
+                                debug!(
+                                    target: "rustfs::heal::task",
+                                    event = EVENT_HEAL_BUCKET_RESULT,
+                                    component = LOG_COMPONENT_HEAL,
+                                    subsystem = LOG_SUBSYSTEM_TASK,
+                                    task_id = %self.id,
+                                    bucket,
+                                    object,
+                                    retry_attempt = retry_attempt.saturating_add(1),
+                                    error = %err,
+                                    result = "object_retry_scheduled",
+                                    "Heal bucket object retry scheduled"
+                                );
+                                retry.push(item);
+                            } else {
+                                telemetry_unknown |= !increment_counter(&mut failed);
+                                if err.is_recoverable_heal() {
+                                    retryable_failed = retryable_failed.saturating_add(1);
+                                } else {
+                                    permanent_failed = permanent_failed.saturating_add(1);
+                                }
+                                first_failed_object.get_or_insert_with(|| object.to_string());
+                                first_error.get_or_insert_with(|| err.to_string());
+                                if take_failure_log_sample(&mut failure_samples_logged) {
+                                    warn!(
+                                        target: "rustfs::heal::task",
+                                        event = EVENT_HEAL_BUCKET_RESULT,
+                                        component = LOG_COMPONENT_HEAL,
+                                        subsystem = LOG_SUBSYSTEM_TASK,
+                                        task_id = %self.id,
+                                        bucket,
+                                        object,
+                                        retry_attempt,
+                                        error = %err,
+                                        result = "object_failed",
+                                        "Heal bucket object repair failed"
+                                    );
+                                }
                             }
                         }
-                    }
 
-                    if terminal_outcome {
+                        if terminal_outcome {
-                        telemetry_unknown |= !increment_counter(&mut scanned);
+                            telemetry_unknown |= !increment_counter(&mut scanned);
-                    }
+                        }
 
-                    if !terminal_outcome {
+                        if !terminal_outcome {
-                        continue;
+                            continue;
-                    }
+                        }
 
-                    let mut progress = self.progress.write().await;
+                        let mut progress = self.progress.write().await;
-                    progress.update_object_progress(scanned, healed, failed, skipped, bytes);
+                        progress.update_object_progress(scanned, healed, failed, skipped, bytes);
-                    if telemetry_unknown {
+                        if telemetry_unknown {
-                        progress.mark_unknown();
+                            progress.mark_unknown();
+                        }
                     }
+                    pending = retry;
+                    retry_attempt = retry_attempt.saturating_add(1);
                 }
-                pending = retry;
-                retry_attempt = retry_attempt.saturating_add(1);
-            }
 
-            if !is_truncated {
+                if !is_truncated {
-                break;
+                    break;
-            }
+                }
 
-            continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
+                continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
-            if continuation_token.is_none() {
+                if continuation_token.is_none() {
-                // Truncated but no continuation token: end of listing.
+                    // Truncated but no continuation token: end of listing.
-                break;
+                    break;
+                }
             }
         }
 

crates/heal/src/heal/task/tests.rs

@@ -564,6 +564,8 @@ struct MockStorage {
     replacement_resume_disk: Mutex<Option<DiskStore>>,
     usage_baseline: Mutex<Option<HealBucketUsageBaseline>>,
     usage_baseline_error: Mutex<bool>,
+    erasure_set_scopes: Mutex<Vec<(usize, usize)>>,
+    disk_walk_calls: Mutex<Vec<String>>,
 }
 
 #[test]
@@ -946,6 +948,26 @@ impl HealStorageAPI for MockStorage {
         }
     }
 
+    async fn heal_erasure_set_scopes(&self, _opts: &HealOpts) -> Result<Option<Vec<(usize, usize)>>> {
+        let scopes = self.erasure_set_scopes.lock().unwrap().clone();
+        Ok((!scopes.is_empty()).then_some(scopes))
+    }
+
+    async fn list_versions_for_heal_page_disk_walk(
+        &self,
+        set_disk_id: &str,
+        _bucket: &str,
+        _prefix: &str,
+        continuation_token: Option<&str>,
+        _include_lifecycle_object_info: bool,
+    ) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
+        self.disk_walk_calls.lock().unwrap().push(set_disk_id.to_string());
+        if continuation_token.is_some() {
+            return Ok((Vec::new(), None, false));
+        }
+        Ok((vec![heal_item(&format!("{set_disk_id}-object"))], None, false))
+    }
+
     async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
         self.resume_disk
             .lock()
@@ -1081,6 +1103,47 @@ async fn test_recursive_bucket_heal_visits_objects() {
     assert_eq!(result_items.iter().filter(|item| item.object_size == 1).count(), 2);
 }
 
+#[tokio::test]
+async fn recursive_bucket_heal_uses_each_erasure_set_union_scope() {
+    let storage = Arc::new(MockStorage {
+        erasure_set_scopes: Mutex::new(vec![(0, 0), (1, 2)]),
+        ..Default::default()
+    });
+    let request = HealRequest::new(
+        HealType::Bucket {
+            bucket: "bucket-a".to_string(),
+        },
+        HealOptions {
+            recursive: true,
+            timeout: None,
+            ..Default::default()
+        },
+        HealPriority::Normal,
+    );
+    let task = HealTask::from_request(request, storage.clone());
+
+    task.heal_bucket("bucket-a")
+        .await
+        .expect("recursive bucket heal should consume every selected set union");
+
+    assert_eq!(
+        storage.disk_walk_calls.lock().unwrap().as_slice(),
+        ["pool_0_set_0".to_string(), "pool_1_set_2".to_string()]
+    );
+    assert_eq!(
+        storage.healed_objects.lock().unwrap().as_slice(),
+        ["pool_0_set_0-object".to_string(), "pool_1_set_2-object".to_string()]
+    );
+    assert!(
+        !*storage.listed.lock().unwrap(),
+        "read-quorum listing must not hide returning-disk candidates"
+    );
+    let object_opts = storage.object_heal_opts.lock().unwrap();
+    assert_eq!(object_opts.len(), 2);
+    assert_eq!((object_opts[0].pool, object_opts[0].set), (Some(0), Some(0)));
+    assert_eq!((object_opts[1].pool, object_opts[1].set), (Some(1), Some(2)));
+}
+
 #[tokio::test]
 async fn result_items_are_bounded_and_report_truncation() {
     let storage = Arc::new(MockStorage::default());

crates/heal/tests/heal_b920_subquorum_union_test.rs

@@ -354,9 +354,9 @@ mod serial_tests {
             with_dangling_grace_disabled(heal_storage.heal_object(bucket, object, Some(&v1), &deep_heal_opts()))
                 .await
                 .expect("heal_object call must not itself error");
-        // A dangling delete reports the version as gone (FileVersionNotFound),
+        // Successful cleanup is a successful heal outcome. The on-disk checks
-        // proving the destructive path fired for a genuinely torn write.
+        // below prove that the torn version was deleted rather than healed.
-        assert!(error.is_some(), "a torn (< data_blocks) version must NOT be silently treated as healed");
+        assert!(error.is_none(), "successful torn-version cleanup must not report a heal error");
         // Destructive-path PROOF: the stale minority copy on disk0 was purged by
         // the dangling delete (the guard correctly did NOT rescue a torn write).
         assert_eq!(