fix(heal): align scanner repair safety with quorum errors (#3887)

* fix(heal): avoid task-level delete on repair failure

* fix(scanner): avoid recreate for scanner heal checks

* fix(heal): preserve typed quorum failures

* test(heal): satisfy clippy bool assertion

crates/ecstore/src/erasure_coding/heal.rs

@@ -19,7 +19,7 @@ use crate::erasure_coding::decode::ParallelReader;
 use crate::erasure_coding::encode::MultiWriter;
 use bytes::Bytes;
 use tokio::io::AsyncRead;
-use tracing::info;
+use tracing::{info, warn};
 
 impl super::Erasure {
     pub async fn heal<R>(
@@ -60,10 +60,14 @@ impl super::Erasure {
             // We need at least data_shards available shards (data + parity combined)
             let available_shards = errs.iter().filter(|e| e.is_none()).count();
             if available_shards < self.data_shards {
-                return Err(Error::other(format!(
-                    "can not reconstruct data: not enough available shards (need {}, have {}) {errs:?}",
-                    self.data_shards, available_shards
-                )));
+                warn!(
+                    required_data_shards = self.data_shards,
+                    available_shards,
+                    total_shards = errs.len(),
+                    errors = ?errs,
+                    "Erasure heal read quorum unavailable"
+                );
+                return Err(Error::ErasureReadQuorum);
             }
 
             if self.parity_shards > 0 {
@@ -193,4 +197,36 @@ mod tests {
             .expect("inline writer should retain data");
         assert_eq!(healed, encoded[missing_data].to_vec());
     }
+
+    #[tokio::test]
+    async fn heal_returns_read_quorum_when_available_shards_are_insufficient() {
+        let erasure = Erasure::new(3, 2, 64);
+        let data = b"heal should fail before decode when too few shards are readable";
+        let encoded = erasure.encode_data(data).expect("encode should succeed");
+
+        let readers = encoded
+            .iter()
+            .enumerate()
+            .map(|(index, shard)| {
+                if index < 2 {
+                    Some(BitrotReader::new(
+                        Cursor::new(shard.to_vec()),
+                        erasure.shard_size(),
+                        HashAlgorithm::None,
+                        false,
+                    ))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+        let mut writers = (0..erasure.total_shard_count()).map(|_| None).collect::<Vec<_>>();
+
+        let err = erasure
+            .heal(&mut writers, readers, data.len(), &[])
+            .await
+            .expect_err("heal should fail when available shards are below data shards");
+
+        assert!(matches!(err, Error::ErasureReadQuorum));
+    }
 }

crates/ecstore/src/set_disk.rs

@@ -4763,7 +4763,9 @@ impl rustfs_storage_api::HealOperations for SetDisks {
         let disks = self.disks.read().await;
 
         let disks = disks.clone();
-        let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
+        let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false)
+            .await
+            .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
         if DiskError::is_all_not_found(&errs) {
             debug!(
                 event = EVENT_SET_DISK_HEAL,
@@ -4791,14 +4793,20 @@ impl rustfs_storage_api::HealOperations for SetDisks {
         // Pass no_lock=true since we already obtained write lock (or are already called with no_lock=true)
         let mut inner_opts = *opts;
         inner_opts.no_lock = true;
-        let (result, err) = self.heal_object(bucket, object, version_id, &inner_opts).await?;
+        let (result, err) = self
+            .heal_object(bucket, object, version_id, &inner_opts)
+            .await
+            .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
         if let Some(err) = err.as_ref() {
             match err {
                 &DiskError::FileCorrupt if opts.scan_mode != HealScanMode::Deep => {
                     // Instead of returning an error when a bitrot error is detected
                     // during a normal heal scan, heal again with bitrot flag enabled.
                     inner_opts.scan_mode = HealScanMode::Deep;
-                    let (result, err) = self.heal_object(bucket, object, version_id, &inner_opts).await?;
+                    let (result, err) = self
+                        .heal_object(bucket, object, version_id, &inner_opts)
+                        .await
+                        .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
                     return Ok((result, err.map(|e| e.into())));
                 }
                 _ => {}

crates/ecstore/src/set_disk/heal.rs

@@ -215,13 +215,14 @@ impl SetDisks {
                             let required_data = total_disks.saturating_sub(latest_meta.erasure.parity_blocks);
 
                             error!(
-                                "Data corruption detected for {}/{}: Insufficient healthy shards. Need at least {} data shards, but found only {} healthy disks. (Missing/Corrupt: {}, Parity: {})",
                                 bucket,
                                 object,
-                                required_data,
-                                healthy_count,
-                                disks_to_heal_count,
-                                latest_meta.erasure.parity_blocks
+                                version_id,
+                                required_data_shards = required_data,
+                                healthy_shards = healthy_count,
+                                missing_or_corrupt_shards = disks_to_heal_count,
+                                parity_shards = latest_meta.erasure.parity_blocks,
+                                "Heal object cannot reconstruct with available shards"
                             );
 
                             // Allow for dangling deletes, on versions that have DataDir missing etc.
@@ -253,16 +254,23 @@ impl SetDisks {
                                     Ok((self.default_heal_result(m, &t_errs, bucket, object, version_id).await, Some(derr)))
                                 }
                                 Err(err) => {
-                                    // t_errs = vec![Some(err.clone()]; errs.len());
+                                    error!(
+                                        bucket,
+                                        object,
+                                        version_id,
+                                        error = %err,
+                                        "Heal object dangling cleanup could not prove object deletion"
+                                    );
+                                    let quorum_err = DiskError::ErasureReadQuorum;
                                     let mut t_errs = Vec::with_capacity(errs.len());
                                     for _ in 0..errs.len() {
-                                        t_errs.push(Some(err.clone()));
+                                        t_errs.push(Some(quorum_err.clone()));
                                     }
 
                                     Ok((
                                         self.default_heal_result(FileInfo::default(), &t_errs, bucket, object, version_id)
                                             .await,
-                                        Some(err),
+                                        Some(quorum_err),
                                     ))
                                 }
                             };