fix(heal): supervise scheduler task panics

crates/heal/src/heal/manager/tests.rs

@@ -110,7 +110,10 @@ impl HealStorageAPI for MockStorage {
         Ok(Vec::new())
     }
 
-    async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>> {
+    async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
+        if bucket == "panic" {
+            panic!("test-only panic payload must not escape the scheduler");
+        }
         Ok(None)
     }
 
@@ -1021,6 +1024,231 @@ async fn test_task_alias_is_removed_after_terminal_completion() {
     assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
 }
 
+#[tokio::test]
+#[serial_test::serial]
+async fn scheduler_panic_releases_active_slot_and_allows_same_target_readmission() {
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let request = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    let task_id = request.id.clone();
+
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("panic request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    let duplicate = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    let duplicate_id = duplicate.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(duplicate)
+            .await
+            .expect("same target should merge while active is queued"),
+        HealAdmissionResult::Merged
+    );
+    assert_eq!(manager.canonical_task_id(&duplicate_id).await, task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Failed { .. })
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("panic task should reach a terminal status");
+    assert_eq!(
+        status,
+        HealTaskStatus::Failed {
+            error: PANICKED_HEAL_TASK_ERROR.to_string()
+        }
+    );
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert_eq!(manager.get_queue_length().await, 0);
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert!(manager.completed_heals.lock().await.contains_key(&task_id));
+    assert_eq!(manager.canonical_task_id(&duplicate_id).await, duplicate_id);
+
+    let readmitted = bucket_request("panic", HealPriority::Normal, HealRequestSource::Admin);
+    assert_eq!(
+        manager
+            .submit_heal_request(readmitted)
+            .await
+            .expect("same target should be re-admitted after a panic"),
+        HealAdmissionResult::Accepted
+    );
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn retry_child_panic_finishes_parent_once() {
+    clear_scheduler_panic();
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("retry request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    arm_scheduler_panic(SchedulerPanicPoint::RetryChild, &task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Failed { .. })
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("retry child panic should finish the parent");
+    clear_scheduler_panic();
+    assert_eq!(
+        status,
+        HealTaskStatus::Failed {
+            error: PANICKED_HEAL_TASK_ERROR.to_string()
+        }
+    );
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert_eq!(manager.get_queue_length().await, 0);
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert_eq!(manager.completed_heals.lock().await.len(), 1);
+    assert_eq!(manager.get_statistics().await.failed_tasks, 1);
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn cleanup_panic_is_supervised() {
+    clear_scheduler_panic();
+    let notice_bucket = "cleanup-panic-mrf";
+    let notice_object = "object";
+    let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
+    let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
+    let manager = HealManager::new(storage, None);
+    let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
+    request.source = HealRequestSource::Admin;
+    let task_id = request.id.clone();
+    assert_eq!(
+        manager
+            .submit_heal_request(request)
+            .await
+            .expect("cleanup request should be admitted"),
+        HealAdmissionResult::Accepted
+    );
+    manager
+        .mrf_repair_notice_targets
+        .lock()
+        .expect("mrf repair notice registry poisoned")
+        .insert(
+            task_id.clone(),
+            vec![MrfRepairNoticeTarget {
+                bucket: Arc::from(notice_bucket),
+                object: Arc::from(notice_object),
+                version_id: None,
+            }],
+        );
+    arm_scheduler_panic(SchedulerPanicPoint::Cleanup, &task_id);
+    process_manager_queue_once(&manager).await;
+
+    let status = tokio::time::timeout(Duration::from_secs(1), async {
+        loop {
+            if let Ok(status) = manager.get_task_status(&task_id).await
+                && matches!(status, HealTaskStatus::Completed)
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("cleanup panic should leave a terminal status");
+    clear_scheduler_panic();
+    assert_eq!(status, HealTaskStatus::Completed);
+    assert_eq!(manager.get_active_task_count().await, 0);
+    assert!(manager.task_aliases.lock().await.is_empty());
+    assert_eq!(manager.completed_heals.lock().await.len(), 1);
+    assert_eq!(manager.get_statistics().await.successful_tasks, 1);
+    let events = rustfs_common::mrf_channel::take_mrf_repaired_events_for(notice_bucket);
+    assert_eq!(events.len(), 1, "cleanup panic must preserve successful MRF notice delivery");
+    assert_eq!(events[0].object.as_ref(), notice_object);
+}
+
+#[tokio::test]
+#[serial_test::serial]
+async fn cancelled_retry_child_panic_does_not_rearchive_failed_status() {
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
+    let task_id = request.id.clone();
+    let retry_cancel_token = insert_retrying_request(&manager, request.clone()).await;
+
+    manager
+        .cancel_task(&task_id)
+        .await
+        .expect("retry cancellation should succeed");
+    assert!(retry_cancel_token.is_cancelled());
+
+    let state = PanicCleanupState {
+        active_heals: manager.active_heals.clone(),
+        heal_queue: manager.heal_queue.clone(),
+        completed_heals: manager.completed_heals.clone(),
+        task_aliases: manager.task_aliases.clone(),
+        retrying_heals: manager.retrying_heals.clone(),
+        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
+        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
+        statistics: manager.statistics.clone(),
+    };
+    finish_panicked_retry_child(task_id.clone(), request.heal_type, retry_cancel_token, state).await;
+
+    assert!(manager.retrying_heals.lock().await.is_empty());
+    assert!(manager.completed_heals.lock().await.is_empty());
+    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
+}
+
+#[tokio::test]
+async fn active_cancel_wins_parent_panic_cleanup_without_completed_status() {
+    let manager = HealManager::new(Arc::new(MockStorage), None);
+    let request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
+    let task_id = request.id.clone();
+    let task = Arc::new(HealTask::from_request(request, Arc::new(MockStorage)));
+    manager.active_heals.lock().await.insert(task_id.clone(), task.clone());
+
+    manager
+        .cancel_task(&task_id)
+        .await
+        .expect("active task cancellation should win");
+    assert_eq!(task.get_status().await, HealTaskStatus::Cancelled);
+
+    let state = PanicCleanupState {
+        active_heals: manager.active_heals.clone(),
+        heal_queue: manager.heal_queue.clone(),
+        completed_heals: manager.completed_heals.clone(),
+        task_aliases: manager.task_aliases.clone(),
+        retrying_heals: manager.retrying_heals.clone(),
+        mrf_repair_notice_targets: manager.mrf_repair_notice_targets.clone(),
+        replacement_recovery_anchors: manager.replacement_recovery_anchors.clone(),
+        statistics: manager.statistics.clone(),
+    };
+    finish_panicked_heal_task(task, task_id, state).await;
+
+    assert!(manager.completed_heals.lock().await.is_empty());
+    assert_eq!(manager.get_statistics().await.failed_tasks, 0);
+}
+
 #[tokio::test]
 async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
     let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);