fix(heal): retry recoverable bucket heal failures (#3509)

* fix(heal): retry recoverable bucket heal failures

* fix(heal): keep merged heal tokens cancellable

* fix(heal): group queue processing context

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

rustfs/src/admin/handlers/heal.rs

@@ -247,6 +247,11 @@ fn encode_heal_task_status(
 }
 
 fn build_heal_channel_request(hip: &HealInitParams) -> HealChannelRequest {
+    let recursive = if !hip.bucket.is_empty() && hip.obj_prefix.is_empty() {
+        true
+    } else {
+        hip.hs.recursive
+    };
     let mut heal_request = rustfs_common::heal_channel::create_heal_request(
         hip.bucket.clone(),
         if hip.obj_prefix.is_empty() {
@@ -264,7 +269,7 @@ fn build_heal_channel_request(hip: &HealInitParams) -> HealChannelRequest {
     heal_request.remove_corrupted = Some(hip.hs.remove);
     heal_request.recreate_missing = Some(hip.hs.recreate);
     heal_request.update_parity = Some(hip.hs.update_parity);
-    heal_request.recursive = Some(hip.hs.recursive);
+    heal_request.recursive = Some(recursive);
     heal_request.dry_run = Some(hip.hs.dry_run);
     heal_request.source = HealRequestSource::Admin;
     heal_request
@@ -831,6 +836,27 @@ mod tests {
         assert_eq!(request.set_index, Some(2));
     }
 
+    #[test]
+    fn test_bucket_heal_channel_request_defaults_to_recursive() {
+        let hip = HealInitParams {
+            bucket: "bucket".to_string(),
+            obj_prefix: String::new(),
+            hs: HealOpts {
+                recursive: false,
+                scan_mode: HealScanMode::Deep,
+                recreate: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let request = build_heal_channel_request(&hip);
+
+        assert_eq!(request.bucket, "bucket");
+        assert_eq!(request.object_prefix, None);
+        assert_eq!(request.recursive, Some(true));
+    }
+
     #[test]
     fn test_extract_heal_init_params_allows_root_heal_target() {
         let uri: Uri = "/rustfs/admin/v3/heal/".parse().expect("uri should parse");