fix(heal): bound read-repair object commit locks (#6839)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/heal/src/heal/storage.rs

@@ -689,6 +689,7 @@ impl HealStorageAPI for ECStoreHealStorage {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/heal_bucket.rs

@@ -95,6 +95,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -262,6 +263,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };

crates/heal/src/heal/task/heal_erasure_set.rs

@@ -356,6 +356,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -411,6 +412,7 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };

crates/heal/src/heal/task/heal_metadata.rs

@@ -97,6 +97,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: false,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
@@ -259,6 +260,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/heal_object.rs

@@ -158,13 +158,22 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: self.options.no_lock,
+            read_repair: self.source == HealRequestSource::ReadRepair,
             pool: self.options.pool_index,
             set: self.options.set_index,
         };
 
-        let heal_result = self
-            .await_with_control(self.storage.heal_object(bucket, object, version_id, &heal_opts))
-            .await;
+        let heal_fut = self.storage.heal_object(bucket, object, version_id, &heal_opts);
+        let heal_result = if self.source == HealRequestSource::ReadRepair {
+            let result = heal_fut.await;
+            if self.cancel_token.is_cancelled() {
+                Err(Error::TaskCancelled)
+            } else {
+                result
+            }
+        } else {
+            self.await_with_control(heal_fut).await
+        };
 
         match heal_result {
             Ok((result, error)) => {
@@ -375,6 +384,7 @@ impl HealTask {
             scan_mode: HealScanMode::Deep,
             update_parity: true,
             no_lock: self.options.no_lock,
+            read_repair: false,
             pool: None,
             set: None,
         };

crates/heal/src/heal/task/tests.rs

@@ -1061,6 +1061,72 @@ async fn scoped_object_heal_slowdown_is_not_treated_as_deleted() {
         .expect("heal options lock should be available");
     assert_eq!(opts[0].pool, Some(0));
     assert_eq!(opts[0].set, Some(1));
+    assert!(!opts[0].read_repair);
+}
+
+#[tokio::test]
+async fn read_repair_object_heal_sets_read_repair_option() {
+    let storage = Arc::new(MockStorage::default());
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), Some("version-a".to_string()));
+    request.source = HealRequestSource::ReadRepair;
+    let task = HealTask::from_request(request, storage.clone());
+
+    task.execute().await.expect("read-repair object heal should complete");
+
+    let opts = storage.object_heal_opts.lock().unwrap();
+    assert_eq!(opts.len(), 1);
+    assert!(opts[0].read_repair);
+    assert!(!opts[0].no_lock);
+}
+
+#[tokio::test]
+async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
+    let storage = Arc::new(MockStorage {
+        block_heal_object: Mutex::new(true),
+        ..Default::default()
+    });
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+    request.source = HealRequestSource::ReadRepair;
+    request.options.timeout = Some(Duration::from_millis(1));
+    let task = Arc::new(HealTask::from_request(request, storage.clone()));
+    let execution = tokio::spawn({
+        let task = task.clone();
+        async move { task.execute().await }
+    });
+
+    tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if !storage.object_heal_opts.lock().unwrap().is_empty() {
+                break;
+            }
+            tokio::task::yield_now().await;
+        }
+    })
+    .await
+    .expect("read-repair object heal should start");
+    tokio::time::sleep(Duration::from_millis(20)).await;
+    assert!(!execution.is_finished(), "read repair must not be failed by the flat task timeout");
+
+    execution.abort();
+    assert!(execution.await.is_err(), "aborted mock execution should not join successfully");
+    assert!(storage.object_heal_opts.lock().unwrap()[0].read_repair);
+}
+
+#[tokio::test]
+async fn non_read_repair_object_heal_still_uses_flat_timeout() {
+    let storage = Arc::new(MockStorage {
+        block_heal_object: Mutex::new(true),
+        ..Default::default()
+    });
+    let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
+    request.options.timeout = Some(Duration::from_millis(1));
+    let task = HealTask::from_request(request, storage);
+
+    let result = tokio::time::timeout(Duration::from_secs(1), task.execute())
+        .await
+        .expect("flat timeout should finish the task");
+
+    assert!(matches!(result, Err(Error::TaskTimeout)));
 }
 
 async fn make_resume_disk(temp: &TempDir) -> DiskStore {

crates/heal/tests/heal_integration_test.rs

@@ -497,6 +497,7 @@ mod serial_tests {
             scan_mode: HealScanMode::Normal,
             update_parity: false,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };
@@ -518,6 +519,7 @@ mod serial_tests {
             scan_mode: HealScanMode::Normal,
             update_parity: false,
             no_lock: false,
+            read_repair: false,
             pool: None,
             set: None,
         };