fix(heal): skip dangling delete grace failures (#6799)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/ecstore/src/disk/error.rs

@@ -22,6 +22,7 @@ pub type Error = DiskError;
 pub type Result<T> = core::result::Result<T, Error>;
 
 const METACACHE_OUTPUT_STREAM_CLOSED: &str = "metacache output stream closed";
+pub(crate) const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
 
 /// Marker carried by a shard-read `io::Error` when the underlying reader can
 /// no longer be realigned after a fresh remote open failed.  The marker is
@@ -33,6 +34,12 @@ pub(crate) struct TerminalReadError {
     source: DiskError,
 }
 
+#[derive(Debug)]
+struct DanglingDeleteGraceError {
+    retry_after_secs: i64,
+    grace_secs: i64,
+}
+
 // DiskError == StorageErr
 #[derive(Debug, thiserror::Error)]
 pub enum DiskError {
@@ -200,6 +207,18 @@ impl StdError for TerminalReadError {
     }
 }
 
+impl std::fmt::Display for DanglingDeleteGraceError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{HEAL_DANGLING_DELETE_GRACE_MESSAGE}; retry_after_secs={}; grace_secs={}",
+            self.retry_after_secs, self.grace_secs
+        )
+    }
+}
+
+impl StdError for DanglingDeleteGraceError {}
+
 fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
     if error.is_remote_file_not_found() {
         return Some(DiskError::FileNotFound);
@@ -253,6 +272,24 @@ impl DiskError {
         DiskError::Io(std::io::Error::other(error))
     }
 
+    pub(crate) fn dangling_delete_grace(retry_after_secs: i64, grace_secs: i64) -> Self {
+        DiskError::other(DanglingDeleteGraceError {
+            retry_after_secs,
+            grace_secs,
+        })
+    }
+
+    pub fn is_dangling_delete_grace(&self) -> bool {
+        matches!(self, DiskError::Io(io_error) if Self::io_error_is_dangling_delete_grace(io_error))
+    }
+
+    pub fn io_error_is_dangling_delete_grace(io_error: &io::Error) -> bool {
+        io_error
+            .get_ref()
+            .is_some_and(|source| source.downcast_ref::<DanglingDeleteGraceError>().is_some())
+            || io_error.to_string().contains(HEAL_DANGLING_DELETE_GRACE_MESSAGE)
+    }
+
     pub(crate) fn metacache_output_stream_closed() -> Self {
         DiskError::Io(std::io::Error::new(std::io::ErrorKind::BrokenPipe, METACACHE_OUTPUT_STREAM_CLOSED))
     }

crates/ecstore/src/error/mod.rs

@@ -277,6 +277,10 @@ impl StorageError {
                 | StorageError::NamespaceLockQuorumUnavailable { .. }
         )
     }
+
+    pub fn is_dangling_delete_grace(&self) -> bool {
+        matches!(self, StorageError::Io(io_error) if DiskError::io_error_is_dangling_delete_grace(io_error))
+    }
 }
 
 impl From<HTTPRangeError> for StorageError {

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -5696,6 +5696,8 @@ impl SetDisks {
         {
             let grace = dangling_delete_grace();
             if !grace.is_zero() && OffsetDateTime::now_utc() - mod_time < grace {
+                let elapsed = OffsetDateTime::now_utc() - mod_time;
+                let retry_after_secs = grace.saturating_sub(elapsed).whole_seconds().max(0);
                 info!(
                     bucket = bucket,
                     object = object,
@@ -5703,7 +5705,7 @@ impl SetDisks {
                     grace_secs = grace.whole_seconds(),
                     "skipping dangling-object deletion within grace window"
                 );
-                return Err(DiskError::ErasureReadQuorum);
+                return Err(DiskError::dangling_delete_grace(retry_after_secs, grace.whole_seconds()));
             }
         }
 
@@ -6799,6 +6801,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier {
 
 #[cfg(test)]
 mod tests {
+    use crate::disk::error::HEAL_DANGLING_DELETE_GRACE_MESSAGE;
     use crate::disk::local::{DurabilityMode, durability_mode_override};
 
     use super::*;
@@ -10746,7 +10749,13 @@ mod tests {
         let object = "object";
         let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
         let set = io_primitives_test_set(vec![Some(disk.clone()), None, None], 1).await;
-        let mut fi = metadata_test_fileinfo(object);
+        let mut fi = FileInfo::new(object, 2, 1);
+        fi.volume = bucket.to_string();
+        fi.name = object.to_string();
+        fi.size = 1;
+        fi.erasure.index = 1;
+        fi.metadata.insert("etag".to_string(), "etag-1".to_string());
+        fi.add_object_part(1, "part-etag-1".to_string(), 1, None, 1, None, None);
         fi.mod_time = Some(OffsetDateTime::now_utc());
         disk.write_metadata(bucket, bucket, object, fi.clone())
             .await
@@ -10764,7 +10773,15 @@ mod tests {
             .await
             .expect_err("recent dangling metadata must stay protected by grace");
 
-        assert_eq!(err, DiskError::ErasureReadQuorum);
+        let message = err.to_string();
+        assert!(
+            message.contains(HEAL_DANGLING_DELETE_GRACE_MESSAGE),
+            "grace-protected dangling cleanup must explain the deferred delete: {message}"
+        );
+        assert!(
+            message.contains("retry_after_secs="),
+            "grace-protected dangling cleanup must include retry timing: {message}"
+        );
         disk.read_all(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
             .await
             .expect("metadata should remain during dangling grace");

crates/ecstore/src/set_disk/ops/heal.rs

@@ -3543,7 +3543,20 @@ mod heal_result_report_tests {
                 .await
                 .expect("grace-protected dangling metadata should return a typed heal result");
 
-            assert_eq!(error, Some(DiskError::ErasureReadQuorum));
+            let error = error.expect("grace-protected dangling metadata should be reported as deferred");
+            assert!(
+                error.is_dangling_delete_grace(),
+                "grace-protected dangling metadata should keep a typed deferred-cleanup marker: {error}"
+            );
+            let message = error.to_string();
+            assert!(
+                message.contains("dangling object deletion deferred by heal grace window"),
+                "grace-protected dangling metadata should explain that cleanup was deferred: {message}"
+            );
+            assert!(
+                message.contains("retry_after_secs="),
+                "grace-protected dangling metadata should include retry timing: {message}"
+            );
             assert!(
                 temp_dirs[0]
                     .path()