fix(heal): harden resumable set repair failures (#5693)

* fix(heal): enforce resumable task control * fix(ecstore): surface bucket and metadata heal errors * chore: refresh guardrail path references --------- Signed-off-by: cxymds <cxymds@gmail.com>
2026-08-19 02:56:18 +00:00 · 2026-08-04 20:35:58 +08:00
parent 3c8bd5b929
commit 31959b90db
15 changed files with 1276 additions and 191 deletions
@@ -99,6 +99,69 @@ impl DeleteBucketEmptyScanBarrier {
 #[cfg(test)]
 static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);

+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+enum HealBucketOperation {
+    Make,
+    Delete,
+}
+
+#[cfg(test)]
+struct HealBucketOperationFailure {
+    bucket: String,
+    disk_index: usize,
+    operation: HealBucketOperation,
+}
+
+#[cfg(test)]
+type HealBucketOperationFailureKey = (String, usize, HealBucketOperation);
+
+#[cfg(test)]
+fn heal_bucket_operation_failures() -> &'static StdMutex<HashMap<HealBucketOperationFailureKey, Error>> {
+    static FAILURES: std::sync::OnceLock<StdMutex<HashMap<HealBucketOperationFailureKey, Error>>> = std::sync::OnceLock::new();
+    FAILURES.get_or_init(|| StdMutex::new(HashMap::new()))
+}
+
+#[cfg(test)]
+impl HealBucketOperationFailure {
+    fn install(bucket: &str, disk_index: usize, operation: HealBucketOperation, error: Error) -> Self {
+        let key = (bucket.to_string(), disk_index, operation);
+        let previous = heal_bucket_operation_failures()
+            .lock()
+            .expect("heal bucket failure registry should not poison")
+            .insert(key, error);
+        assert!(previous.is_none(), "heal bucket operation failure already installed");
+        Self {
+            bucket: bucket.to_string(),
+            disk_index,
+            operation,
+        }
+    }
+}
+
+#[cfg(test)]
+impl Drop for HealBucketOperationFailure {
+    fn drop(&mut self) {
+        heal_bucket_operation_failures()
+            .lock()
+            .expect("heal bucket failure registry should not poison")
+            .remove(&(self.bucket.clone(), self.disk_index, self.operation));
+    }
+}
+
+#[cfg(test)]
+fn injected_heal_bucket_operation_error(bucket: &str, disk_index: usize, operation: HealBucketOperation) -> Option<Error> {
+    heal_bucket_operation_failures()
+        .lock()
+        .expect("heal bucket failure registry should not poison")
+        .get(&(bucket.to_string(), disk_index, operation))
+        .cloned()
+}
+
+#[cfg(not(test))]
+fn injected_heal_bucket_operation_error(_bucket: &str, _disk_index: usize, _operation: HealBucketOperation) -> Option<Error> {
+    None
+}
+
 #[cfg(test)]
 pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
    let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
@@ -1207,10 +1270,6 @@ pub(crate) async fn heal_bucket_local_on_disks(
        ..Default::default()
    };

-    if opts.dry_run {
-        return Ok(res);
-    }
-
    for (disk, state) in disks.iter().zip(before_state.read().await.iter()) {
        res.before.drives.push(HealDriveInfo {
            uuid: "".to_string(),
@@ -1219,35 +1278,68 @@ pub(crate) async fn heal_bucket_local_on_disks(
        });
    }

+    if opts.dry_run {
+        for (disk, state) in disks.iter().zip(after_state.read().await.iter()) {
+            res.after.drives.push(HealDriveInfo {
+                uuid: "".to_string(),
+                endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
+                state: state.to_string(),
+            });
+        }
+        return Ok(res);
+    }
+
+    let mut operation_error = errs
+        .iter()
+        .filter_map(|err| match err {
+            Some(Error::VolumeNotFound) | None => None,
+            Some(err) => Some(err.clone()),
+        })
+        .next();
+
    if opts.remove && !bucket.starts_with(disk::RUSTFS_META_BUCKET) && !is_all_buckets_not_found(&errs) {
        let mut futures = Vec::new();
-        for disk in disks.iter() {
-            let disk = disk.clone();
+        for (index, disk) in disks.iter().enumerate() {
+            if matches!(errs[index].as_ref(), Some(Error::DiskNotFound | Error::VolumeNotFound)) {
+                continue;
+            }
+            let Some(disk) = disk.clone() else {
+                continue;
+            };
            let bucket = bucket.to_string();
-            info!("heal_bucket_local, errs: {:?}, opts: {:?}", errs, opts);
            futures.push(async move {
-                match disk {
-                    Some(disk) => {
-                        // Non-force: a bucket that still holds object data refuses
-                        // deletion (VolumeNotEmpty) instead of being recursively
-                        // wiped, so a misclassified "dangling" bucket cannot lose
-                        // data (backlog#799 B1). Surface that refusal instead of
-                        // discarding it — it signals the bucket is not dangling.
-                        match disk.delete_volume(&bucket, false).await {
-                            Ok(()) => None,
-                            Err(Error::VolumeNotEmpty) => {
-                                warn!("heal declined to remove non-empty bucket {bucket} (not dangling)");
-                                None
-                            }
-                            Err(e) => Some(e),
-                        }
-                    }
-                    None => Some(Error::DiskNotFound),
+                if let Some(err) = injected_heal_bucket_operation_error(&bucket, index, HealBucketOperation::Delete) {
+                    return (index, Err(err));
                }
+                (index, disk.delete_volume(&bucket, false).await)
            });
        }

-        let _ = join_all(futures).await;
+        for (index, result) in join_all(futures).await {
+            match result {
+                Ok(()) | Err(Error::VolumeNotFound) => {
+                    after_state.write().await[index] = DriveState::Missing.to_string();
+                }
+                Err(Error::VolumeNotEmpty) => {
+                    warn!(
+                        bucket,
+                        operation = "heal_bucket_delete_volume",
+                        result = "preserved_non_empty_bucket",
+                        "heal declined to remove non-empty bucket"
+                    );
+                    after_state.write().await[index] = DriveState::Ok.to_string();
+                }
+                Err(err) => {
+                    after_state.write().await[index] = match &err {
+                        Error::DiskNotFound => DriveState::Offline.to_string(),
+                        _ => DriveState::Corrupt.to_string(),
+                    };
+                    if operation_error.is_none() {
+                        operation_error = Some(err);
+                    }
+                }
+            }
+        }
    }

    if !opts.remove {
@@ -1256,41 +1348,56 @@ pub(crate) async fn heal_bucket_local_on_disks(
            let disk = disk.clone();
            let bucket = bucket.to_string();
            let bs_clone = before_state.clone();
-            let as_clone = after_state.clone();
-            let errs_clone = errs.to_vec();
            futures.push(async move {
                if bs_clone.read().await[idx] == DriveState::Missing.to_string() {
                    let Some(disk) = disk.as_ref() else {
-                        return Some(Error::DiskNotFound);
+                        return (idx, Some(Error::DiskNotFound));
                    };

-                    info!("bucket not find, will recreate");
+                    if let Some(err) = injected_heal_bucket_operation_error(&bucket, idx, HealBucketOperation::Make) {
+                        return (idx, Some(err));
+                    }
                    match disk.make_volume(&bucket).await {
-                        Ok(_) => {
-                            as_clone.write().await[idx] = DriveState::Ok.to_string();
-                            return None;
-                        }
-                        Err(err) => {
-                            return Some(err);
-                        }
+                        Ok(()) | Err(Error::VolumeExists) => return (idx, None),
+                        Err(err) => return (idx, Some(err)),
                    }
                }
-                errs_clone[idx].clone()
+                (idx, None)
            });
        }

-        let _ = join_all(futures).await;
+        for (index, result) in join_all(futures).await {
+            match result {
+                None => {
+                    if before_state.read().await[index] == DriveState::Missing.to_string() {
+                        after_state.write().await[index] = DriveState::Ok.to_string();
+                    }
+                }
+                Some(err) => {
+                    after_state.write().await[index] = match &err {
+                        Error::DiskNotFound => DriveState::Offline.to_string(),
+                        _ => DriveState::Corrupt.to_string(),
+                    };
+                    if operation_error.is_none() {
+                        operation_error = Some(err);
+                    }
+                }
+            }
+        }
    }

    for (disk, state) in disks.iter().zip(after_state.read().await.iter()) {
-        res.before.drives.push(HealDriveInfo {
+        res.after.drives.push(HealDriveInfo {
            uuid: "".to_string(),
            endpoint: disk.clone().map(|s| s.to_string()).unwrap_or_default(),
            state: state.to_string(),
        });
    }

-    Ok(res)
+    match operation_error {
+        Some(err) => Err(err),
+        None => Ok(res),
+    }
 }

 async fn clone_drives() -> Vec<Option<DiskStore>> {
@@ -1756,7 +1863,7 @@ mod tests {
            .await
            .expect_err("second disk should start missing the bucket");

-        heal_bucket_local(
+        let result = heal_bucket_local(
            bucket,
            &HealOpts {
                recreate: true,
@@ -1766,6 +1873,25 @@ mod tests {
        .await
        .expect("bucket heal should recreate missing volumes");

+        assert_eq!(result.before.drives.len(), 2);
+        assert_eq!(result.after.drives.len(), 2);
+        assert!(
+            result
+                .before
+                .drives
+                .iter()
+                .any(|drive| drive.state == DriveState::Missing.to_string()),
+            "one bucket volume must be reported missing before heal"
+        );
+        assert!(
+            result
+                .after
+                .drives
+                .iter()
+                .all(|drive| drive.state == DriveState::Ok.to_string()),
+            "all bucket volumes must be reported healthy after heal"
+        );
+
        for disk in disks {
            disk.stat_volume(bucket).await.expect("bucket should exist after heal");
        }
@@ -1773,6 +1899,166 @@ mod tests {
        reset_local_disk_test_state().await;
    }

+    #[tokio::test]
+    #[serial]
+    async fn heal_bucket_local_dry_run_reports_discovered_drive_states() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for bucket heal dry-run regression");
+        let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-dry-run-reports-state").await;
+        let bucket = "dry-run-healed-bucket";
+        disks[0]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should exist on the first disk");
+
+        let result = heal_bucket_local_on_disks(
+            bucket,
+            &HealOpts {
+                dry_run: true,
+                ..Default::default()
+            },
+            vec![Some(disks[0].clone()), Some(disks[1].clone()), None],
+        )
+        .await
+        .expect("dry-run bucket heal should inspect disks");
+
+        assert_eq!(result.before.drives.len(), 3);
+        assert_eq!(result.after.drives.len(), 3);
+        assert_eq!(result.before.drives[0].state, DriveState::Ok.to_string());
+        assert_eq!(result.before.drives[1].state, DriveState::Missing.to_string());
+        assert_eq!(result.before.drives[2].state, DriveState::Offline.to_string());
+        for (before, after) in result.before.drives.iter().zip(&result.after.drives) {
+            assert_eq!(after.endpoint, before.endpoint);
+            assert_eq!(after.state, before.state);
+        }
+        assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
+
+        reset_local_disk_test_state().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn heal_bucket_local_propagates_recreate_failure() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for bucket recreate failure regression");
+        let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-propagates-recreate-failure").await;
+        let bucket = "recreate-failure-bucket";
+        disks[0]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should exist on the first disk");
+        let _failure = HealBucketOperationFailure::install(bucket, 1, HealBucketOperation::Make, Error::DiskAccessDenied);
+
+        let error = heal_bucket_local_on_disks(
+            bucket,
+            &HealOpts {
+                recreate: true,
+                ..Default::default()
+            },
+            disks.iter().cloned().map(Some).collect(),
+        )
+        .await
+        .expect_err("failed volume recreation must fail bucket heal");
+
+        assert_eq!(error, Error::DiskAccessDenied);
+        assert!(matches!(disks[1].stat_volume(bucket).await, Err(Error::VolumeNotFound)));
+
+        reset_local_disk_test_state().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn heal_bucket_local_propagates_delete_failure() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for bucket delete failure regression");
+        let disks = init_test_local_disks(&temp_dir, 2, "heal-bucket-local-propagates-delete-failure").await;
+        let bucket = "delete-failure-bucket";
+        disks[0]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should exist on the first disk");
+        let _failure = HealBucketOperationFailure::install(bucket, 0, HealBucketOperation::Delete, Error::DiskAccessDenied);
+
+        let error = heal_bucket_local_on_disks(
+            bucket,
+            &HealOpts {
+                remove: true,
+                ..Default::default()
+            },
+            disks.iter().cloned().map(Some).collect(),
+        )
+        .await
+        .expect_err("failed volume deletion must fail bucket heal");
+
+        assert_eq!(error, Error::DiskAccessDenied);
+        disks[0]
+            .stat_volume(bucket)
+            .await
+            .expect("failed deletion must leave the bucket volume present");
+
+        reset_local_disk_test_state().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn heal_bucket_local_preserves_non_empty_bucket() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for non-empty bucket heal regression");
+        let disks = init_test_local_disks(&temp_dir, 1, "heal-bucket-local-preserves-non-empty").await;
+        let bucket = "non-empty-bucket";
+        disks[0]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should exist on the first disk");
+        let _failure = HealBucketOperationFailure::install(bucket, 0, HealBucketOperation::Delete, Error::VolumeNotEmpty);
+
+        let result = heal_bucket_local_on_disks(
+            bucket,
+            &HealOpts {
+                remove: true,
+                ..Default::default()
+            },
+            disks.iter().cloned().map(Some).collect(),
+        )
+        .await
+        .expect("a non-empty bucket refusal is an expected safety result");
+
+        assert_eq!(result.after.drives.len(), 1);
+        assert_eq!(result.after.drives[0].state, DriveState::Ok.to_string());
+        disks[0]
+            .stat_volume(bucket)
+            .await
+            .expect("the non-empty bucket must remain present");
+
+        reset_local_disk_test_state().await;
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn heal_bucket_local_propagates_preexisting_offline_disk() {
+        reset_local_disk_test_state().await;
+
+        let temp_dir = TempDir::new().expect("create temp dir for offline bucket heal regression");
+        let disks = init_test_local_disks(&temp_dir, 1, "heal-bucket-local-preexisting-offline").await;
+        let bucket = "offline-disk-bucket";
+        disks[0]
+            .make_volume(bucket)
+            .await
+            .expect("bucket should exist on the online disk");
+
+        let error = heal_bucket_local_on_disks(bucket, &HealOpts::default(), vec![Some(disks[0].clone()), None])
+            .await
+            .expect_err("a prepass offline disk must keep the bucket heal incomplete");
+
+        assert_eq!(error, Error::DiskNotFound);
+
+        reset_local_disk_test_state().await;
+    }
+
    #[test]
    fn test_reduce_pool_write_quorum_uses_only_pool_participants() {
        let clients = vec![
@@ -155,6 +155,60 @@ fn injected_dangling_check_parts_error(bucket: &str, object: &str, disk_index: u
        .cloned()
 }

+#[cfg(test)]
+struct DanglingDeleteFailure {
+    key: DanglingDeleteFailureKey,
+}
+
+#[cfg(test)]
+type DanglingDeleteFailureKey = (String, String, usize);
+
+#[cfg(test)]
+type DanglingDeleteFailures = HashMap<DanglingDeleteFailureKey, DiskError>;
+
+#[cfg(test)]
+fn dangling_delete_failures() -> &'static std::sync::Mutex<DanglingDeleteFailures> {
+    static FAILURES: std::sync::OnceLock<std::sync::Mutex<DanglingDeleteFailures>> = std::sync::OnceLock::new();
+    FAILURES.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
+}
+
+#[cfg(test)]
+impl DanglingDeleteFailure {
+    fn install(bucket: &str, object: &str, disk_index: usize, error: DiskError) -> Self {
+        let key = (bucket.to_string(), object.to_string(), disk_index);
+        let previous = dangling_delete_failures()
+            .lock()
+            .expect("dangling delete failure registry should not poison")
+            .insert(key.clone(), error);
+        assert!(previous.is_none(), "dangling delete failure already installed");
+        Self { key }
+    }
+}
+
+#[cfg(test)]
+impl Drop for DanglingDeleteFailure {
+    fn drop(&mut self) {
+        dangling_delete_failures()
+            .lock()
+            .expect("dangling delete failure registry should not poison")
+            .remove(&self.key);
+    }
+}
+
+#[cfg(test)]
+fn injected_dangling_delete_error(bucket: &str, object: &str, disk_index: usize) -> Option<DiskError> {
+    dangling_delete_failures()
+        .lock()
+        .expect("dangling delete failure registry should not poison")
+        .get(&(bucket.to_string(), object.to_string(), disk_index))
+        .cloned()
+}
+
+#[cfg(not(test))]
+fn injected_dangling_delete_error(_bucket: &str, _object: &str, _disk_index: usize) -> Option<DiskError> {
+    None
+}
+
 fn first_unhealthy_part_summary(
    data_errs_by_part: &HashMap<usize, Vec<usize>>,
    parts: &[ObjectPartInfo],
@@ -1181,30 +1235,37 @@ impl SetDisks {

        let errs = stat_all_dirs(&disks, bucket, object).await;
        let dangling_object = is_object_dir_dangling(&errs);
-        if dangling_object && !dry_run && remove {
+        let delete_errs = if dangling_object && !dry_run && remove {
            let mut futures = Vec::with_capacity(disks.len());
-            for disk in disks.iter().flatten() {
+            for (disk_index, disk) in disks.iter().enumerate() {
                let disk = disk.clone();
-                let bucket = bucket.to_string();
-                let object = object.to_string();
-                futures.push(tokio::spawn(async move {
-                    let _ = disk
-                        .delete(
-                            &bucket,
-                            &object,
+                futures.push(async move {
+                    let Some(disk) = disk else {
+                        return (disk_index, Some(DiskError::DiskNotFound));
+                    };
+                    if let Some(error) = injected_dangling_delete_error(bucket, object, disk_index) {
+                        return (disk_index, Some(error));
+                    }
+                    (
+                        disk_index,
+                        disk.delete(
+                            bucket,
+                            object,
                            DeleteOptions {
                                recursive: false,
                                immediate: false,
                                ..Default::default()
                            },
                        )
-                        .await;
-                }));
+                        .await
+                        .err(),
+                    )
+                });
            }
-
-            // ignore errors
-            let _ = join_all(futures).await;
-        }
+            Some(join_all(futures).await)
+        } else {
+            None
+        };

        for (err, drive) in errs.iter().zip(self.set_endpoints.iter()) {
            let endpoint = drive.to_string();
@@ -1229,6 +1290,28 @@ impl SetDisks {
            });
        }

+        if let Some(delete_errs) = delete_errs {
+            let mut delete_failure = None;
+            for (index, err) in delete_errs {
+                match err {
+                    None | Some(DiskError::FileNotFound) => {
+                        result.after.drives[index].state = DriveState::Missing.to_string();
+                    }
+                    Some(err) => {
+                        result.after.drives[index].state = if matches!(&err, DiskError::DiskNotFound) {
+                            DriveState::Offline.to_string()
+                        } else {
+                            DriveState::Corrupt.to_string()
+                        };
+                        if delete_failure.is_none() {
+                            delete_failure = Some(err);
+                        }
+                    }
+                }
+            }
+            return Ok((result, Some(delete_failure.unwrap_or(DiskError::FileNotFound))));
+        }
+
        if dangling_object || DiskError::is_all_not_found(&errs) {
            return Ok((result, Some(DiskError::FileNotFound)));
        }
@@ -1553,7 +1636,7 @@ impl crate::storage_api_contracts::heal::HealOperations for SetDisks {

 #[cfg(test)]
 mod heal_result_report_tests {
-    use super::{DanglingCheckPartsFailure, DanglingDeleteSafety, SetDisks};
+    use super::{DanglingCheckPartsFailure, DanglingDeleteFailure, DanglingDeleteSafety, SetDisks};
    use super::{HEAL_RENAME_INCOMPLETE, HealRenameFailureScope};
    use crate::disk::endpoint::Endpoint;
    use crate::disk::error::DiskError;
@@ -2020,6 +2103,67 @@ mod heal_result_report_tests {
        assert_eq!(result.before.drives[3].state, DriveState::Ok.to_string());
    }

+    #[tokio::test]
+    async fn dangling_object_dir_delete_preserves_results_and_propagates_failure() {
+        let bucket = "bucket-dangling-dir-delete";
+        let object = "dangling__XLDIR__";
+        let mut temp_dirs = Vec::new();
+        let mut endpoints = Vec::new();
+        let mut disks = Vec::new();
+        for _ in 0..8 {
+            let (temp_dir, endpoint, disk) = real_disk().await;
+            disk.make_volume(bucket).await.expect("test bucket should be created");
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+        disks[0] = None;
+        let set = set_disks_with(disks, endpoints, 4).await;
+        for disk_index in [1, 2] {
+            tokio::fs::create_dir_all(temp_dirs[disk_index].path().join(bucket).join(object))
+                .await
+                .expect("dangling object directory should be created");
+        }
+
+        let _delete_failure = DanglingDeleteFailure::install(bucket, object, 2, DiskError::DiskAccessDenied);
+        let _file_missing = DanglingDeleteFailure::install(bucket, object, 3, DiskError::FileNotFound);
+        let _version_missing = DanglingDeleteFailure::install(bucket, object, 4, DiskError::FileVersionNotFound);
+        let _path_missing = DanglingDeleteFailure::install(bucket, object, 5, DiskError::PathNotFound);
+        let _volume_missing = DanglingDeleteFailure::install(bucket, object, 6, DiskError::VolumeNotFound);
+        let _disk_missing = DanglingDeleteFailure::install(bucket, object, 7, DiskError::DiskNotFound);
+
+        let (result, err) = set
+            .heal_object_dir_locked(bucket, object, false, true)
+            .await
+            .expect("dangling directory heal should report its per-disk delete results");
+
+        assert_eq!(err, Some(DiskError::DiskNotFound));
+        assert_eq!(result.before.drives.len(), 8);
+        assert_eq!(result.after.drives.len(), 8);
+        assert_eq!(result.before.drives[0].state, DriveState::Offline.to_string());
+        assert_eq!(result.after.drives[0].state, DriveState::Offline.to_string());
+        assert_eq!(result.before.drives[1].state, DriveState::Ok.to_string());
+        assert_eq!(result.after.drives[1].state, DriveState::Missing.to_string());
+        assert_eq!(result.before.drives[2].state, DriveState::Ok.to_string());
+        assert_eq!(result.after.drives[2].state, DriveState::Corrupt.to_string());
+        assert_eq!(result.before.drives[3].state, DriveState::Missing.to_string());
+        assert_eq!(result.after.drives[3].state, DriveState::Missing.to_string());
+        for disk_index in [4, 5, 6] {
+            assert_eq!(result.before.drives[disk_index].state, DriveState::Missing.to_string());
+            assert_eq!(result.after.drives[disk_index].state, DriveState::Corrupt.to_string());
+        }
+        assert_eq!(result.before.drives[7].state, DriveState::Missing.to_string());
+        assert_eq!(result.after.drives[7].state, DriveState::Offline.to_string());
+        assert!(
+            !temp_dirs[1].path().join(bucket).join(object).exists(),
+            "successful delete must remove the dangling directory"
+        );
+        assert!(
+            temp_dirs[2].path().join(bucket).join(object).is_dir(),
+            "failed delete must leave the dangling directory for retry"
+        );
+    }
+
    #[tokio::test]
    async fn dangling_delete_guard_preserves_conflicting_identities_without_writing_metadata() {
        let bucket = "bucket-delete-guard-conflict";
@@ -64,6 +64,7 @@ struct HealWalkCollector {
    batch_objects: usize,
    version_budget: usize,
    objects: Mutex<Vec<HealWalkObject>>,
+    decode_error: Mutex<Option<DiskError>>,
    version_total: AtomicUsize,
    truncated: AtomicBool,
    cancel: CancellationToken,
@@ -77,6 +78,22 @@ impl HealWalkCollector {
        })
    }

+    fn record_decode_error(&self, error: rustfs_filemeta::Error) {
+        if let Ok(mut first_error) = self.decode_error.lock()
+            && first_error.is_none()
+        {
+            *first_error = Some(error.into());
+        }
+        self.cancel.cancel();
+    }
+
+    fn take_decode_error(&self) -> disk::error::Result<Option<DiskError>> {
+        self.decode_error.lock().map(|mut error| error.take()).map_err(|_| {
+            self.cancel.cancel();
+            DiskError::FileCorrupt
+        })
+    }
+
    /// Expand one resolved entry into its versions and record it. Cancels the
    /// walk once EITHER page bound (distinct object names OR expanded versions)
    /// is met — always at a sorted object-key boundary so a heavily-versioned
@@ -92,7 +109,7 @@ impl HealWalkCollector {
        let fiv = match entry.file_info_versions_with_free_versions(&self.bucket) {
            Ok(fiv) => fiv,
            Err(err) => {
-                debug!(entry = %entry.name, error = ?err, "heal disk-walk skipped entry with unreadable versions");
+                self.record_decode_error(err);
                return;
            }
        };
@@ -248,6 +265,7 @@ impl SetDisks {
            batch_objects,
            version_budget: version_budget.max(1),
            objects: Mutex::new(Vec::new()),
+            decode_error: Mutex::new(None),
            version_total: AtomicUsize::new(0),
            truncated: AtomicBool::new(false),
            cancel: CancellationToken::new(),
@@ -294,7 +312,12 @@ impl SetDisks {

        // Drive the walk. A tolerated missing-path / not-found is treated as an
        // empty page rather than an error (nothing to heal on this prefix).
-        match list_path_raw(collector.cancel.clone(), opts).await {
+        let walk_result = list_path_raw(collector.cancel.clone(), opts).await;
+        if let Some(err) = collector.take_decode_error()? {
+            return Err(err);
+        }
+
+        match walk_result {
            Ok(()) => {}
            Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {
                debug!(bucket, prefix, "heal disk-walk treated missing path as empty page");
@@ -314,6 +337,52 @@ impl SetDisks {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::set_disk::ops::object::hermetic_set_disks_support::hermetic_set_disks_isolated;
+    use rustfs_filemeta::{ChecksumAlgo, ErasureAlgo, FileMetaVersion, MetaObject, VersionType};
+    use time::OffsetDateTime;
+    use uuid::Uuid;
+
+    fn test_collector() -> Arc<HealWalkCollector> {
+        Arc::new(HealWalkCollector {
+            bucket: "bucket".to_string(),
+            batch_objects: 2,
+            version_budget: 2,
+            objects: Mutex::new(Vec::new()),
+            decode_error: Mutex::new(None),
+            version_total: AtomicUsize::new(0),
+            truncated: AtomicBool::new(false),
+            cancel: CancellationToken::new(),
+        })
+    }
+
+    fn crc_valid_semantically_corrupt_entry(name: &str) -> MetaCacheEntry {
+        let mut metadata = FileMeta::new();
+        metadata
+            .add_version_filemata(FileMetaVersion {
+                version_type: VersionType::Object,
+                object: Some(MetaObject {
+                    version_id: Some(Uuid::new_v4()),
+                    erasure_algorithm: ErasureAlgo::ReedSolomon,
+                    erasure_m: 2,
+                    erasure_n: 2,
+                    erasure_block_size: 1 << 20,
+                    bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
+                    part_numbers: vec![1, 2],
+                    part_sizes: vec![10],
+                    part_actual_sizes: vec![10, 20],
+                    mod_time: Some(OffsetDateTime::now_utc()),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            })
+            .expect("corrupt test version should be accepted before semantic decoding");
+
+        MetaCacheEntry {
+            name: name.to_string(),
+            metadata: metadata.marshal_msg().expect("test metadata should encode with a valid CRC"),
+            ..Default::default()
+        }
+    }

    fn version(name: &str, id: &str, dm: bool) -> HealWalkVersion {
        HealWalkVersion {
@@ -449,15 +518,7 @@ mod tests {

    #[test]
    fn poisoned_collector_state_cancels_the_walk() {
-        let collector = Arc::new(HealWalkCollector {
-            bucket: "bucket".to_string(),
-            batch_objects: 2,
-            version_budget: 2,
-            objects: Mutex::new(Vec::new()),
-            version_total: AtomicUsize::new(0),
-            truncated: AtomicBool::new(false),
-            cancel: CancellationToken::new(),
-        });
+        let collector = test_collector();
        let poison_target = Arc::clone(&collector);
        let _ = std::thread::spawn(move || {
            let _guard = poison_target.objects.lock().expect("fresh mutex should lock");
@@ -470,4 +531,45 @@ mod tests {
        assert!(collector.cancel.is_cancelled(), "a poisoned page collector must cancel its walk");
        assert!(collector.objects.lock().is_err(), "poisoned state must remain fail-closed");
    }
+
+    #[test]
+    fn semantic_decode_failure_records_error_and_cancels_walk() {
+        let collector = test_collector();
+        let entry = crc_valid_semantically_corrupt_entry("corrupt-object");
+
+        collector.ingest(entry);
+
+        let error = collector
+            .take_decode_error()
+            .expect("decode error state should remain readable")
+            .expect("semantic metadata corruption must be recorded");
+        assert_eq!(error, DiskError::FileCorrupt);
+        assert!(collector.cancel.is_cancelled(), "semantic metadata corruption must cancel the disk walk");
+    }
+
+    #[tokio::test]
+    async fn heal_walk_returns_crc_valid_semantic_decode_failure() {
+        let bucket = "bucket";
+        let object = "corrupt-object";
+        let (temp_dirs, disks, set_disks) = hermetic_set_disks_isolated(1).await;
+        disks[0].make_volume(bucket).await.expect("test bucket should be created");
+
+        let object_dir = temp_dirs[0].path().join(bucket).join(object);
+        tokio::fs::create_dir_all(&object_dir)
+            .await
+            .expect("test object directory should be created");
+        tokio::fs::write(
+            object_dir.join(crate::disk::STORAGE_FORMAT_FILE),
+            crc_valid_semantically_corrupt_entry(object).metadata,
+        )
+        .await
+        .expect("corrupt test metadata should be written");
+
+        let error = set_disks
+            .heal_walk_versions_page(bucket, "", None, 2, 2)
+            .await
+            .expect_err("semantic metadata corruption must fail the heal disk walk");
+
+        assert_eq!(error, DiskError::FileCorrupt);
+    }
 }