feat: implement heal channel mechanism for admin-ahm communication

- Add global unbounded channel in common crate for heal requests
- Implement channel processor in ahm to handle heal commands
- Add Start/Query/Cancel commands support via channel
- Integrate heal manager initialization in main.rs
- Replace direct MRF calls with channel-based heal requests in ecstore
- Support advanced heal options including pool_index and set_index
- Enable admin handlers to send heal requests via channel

crates/ahm/src/heal/channel.rs

@@ -0,0 +1,232 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::error::Result;
+use crate::heal::{
+    manager::HealManager,
+    task::{HealOptions, HealPriority, HealRequest, HealType},
+};
+
+use rustfs_common::heal_channel::{
+    HealChannelCommand, HealChannelPriority, HealChannelReceiver, HealChannelRequest, HealChannelResponse,
+};
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use tracing::{error, info};
+
+/// Heal channel processor
+pub struct HealChannelProcessor {
+    /// Heal manager
+    heal_manager: Arc<HealManager>,
+    /// Response sender
+    response_sender: mpsc::UnboundedSender<HealChannelResponse>,
+    /// Response receiver
+    response_receiver: mpsc::UnboundedReceiver<HealChannelResponse>,
+}
+
+impl HealChannelProcessor {
+    /// Create new HealChannelProcessor
+    pub fn new(heal_manager: Arc<HealManager>) -> Self {
+        let (response_tx, response_rx) = mpsc::unbounded_channel();
+        Self {
+            heal_manager,
+            response_sender: response_tx,
+            response_receiver: response_rx,
+        }
+    }
+
+    /// Start processing heal channel requests
+    pub async fn start(&mut self, mut receiver: HealChannelReceiver) -> Result<()> {
+        info!("Starting heal channel processor");
+
+        loop {
+            tokio::select! {
+                command = receiver.recv() => {
+                    match command {
+                        Some(command) => {
+                            if let Err(e) = self.process_command(command).await {
+                                error!("Failed to process heal command: {}", e);
+                            }
+                        }
+                        None => {
+                            info!("Heal channel receiver closed, stopping processor");
+                            break;
+                        }
+                    }
+                }
+                response = self.response_receiver.recv() => {
+                    if let Some(response) = response {
+                        // Handle response if needed
+                        info!("Received heal response for request: {}", response.request_id);
+                    }
+                }
+            }
+        }
+
+        info!("Heal channel processor stopped");
+        Ok(())
+    }
+
+    /// Process heal command
+    async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
+        match command {
+            HealChannelCommand::Start(request) => self.process_start_request(request).await,
+            HealChannelCommand::Query { heal_path, client_token } => self.process_query_request(heal_path, client_token).await,
+            HealChannelCommand::Cancel { heal_path } => self.process_cancel_request(heal_path).await,
+        }
+    }
+
+    /// Process start request
+    async fn process_start_request(&self, request: HealChannelRequest) -> Result<()> {
+        info!("Processing heal start request: {} for bucket: {}", request.id, request.bucket);
+
+        // Convert channel request to heal request
+        let heal_request = self.convert_to_heal_request(request.clone())?;
+
+        // Submit to heal manager
+        match self.heal_manager.submit_heal_request(heal_request).await {
+            Ok(task_id) => {
+                info!("Successfully submitted heal request: {} as task: {}", request.id, task_id);
+
+                // Send success response
+                let response = HealChannelResponse {
+                    request_id: request.id,
+                    success: true,
+                    data: Some(format!("Task ID: {}", task_id).into_bytes()),
+                    error: None,
+                };
+
+                if let Err(e) = self.response_sender.send(response) {
+                    error!("Failed to send heal response: {}", e);
+                }
+            }
+            Err(e) => {
+                error!("Failed to submit heal request: {} - {}", request.id, e);
+
+                // Send error response
+                let response = HealChannelResponse {
+                    request_id: request.id,
+                    success: false,
+                    data: None,
+                    error: Some(e.to_string()),
+                };
+
+                if let Err(e) = self.response_sender.send(response) {
+                    error!("Failed to send heal error response: {}", e);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Process query request
+    async fn process_query_request(&self, heal_path: String, client_token: String) -> Result<()> {
+        info!("Processing heal query request for path: {}", heal_path);
+
+        // TODO: Implement query logic based on heal_path and client_token
+        // For now, return a placeholder response
+        let response = HealChannelResponse {
+            request_id: client_token,
+            success: true,
+            data: Some(format!("Query result for path: {}", heal_path).into_bytes()),
+            error: None,
+        };
+
+        if let Err(e) = self.response_sender.send(response) {
+            error!("Failed to send query response: {}", e);
+        }
+
+        Ok(())
+    }
+
+    /// Process cancel request
+    async fn process_cancel_request(&self, heal_path: String) -> Result<()> {
+        info!("Processing heal cancel request for path: {}", heal_path);
+
+        // TODO: Implement cancel logic based on heal_path
+        // For now, return a placeholder response
+        let response = HealChannelResponse {
+            request_id: heal_path.clone(),
+            success: true,
+            data: Some(format!("Cancel request for path: {}", heal_path).into_bytes()),
+            error: None,
+        };
+
+        if let Err(e) = self.response_sender.send(response) {
+            error!("Failed to send cancel response: {}", e);
+        }
+
+        Ok(())
+    }
+
+    /// Convert channel request to heal request
+    fn convert_to_heal_request(&self, request: HealChannelRequest) -> Result<HealRequest> {
+        let heal_type = match &request.object_prefix {
+            Some(prefix) if !prefix.is_empty() => HealType::Object {
+                bucket: request.bucket.clone(),
+                object: prefix.clone(),
+                version_id: None,
+            },
+            _ => HealType::Bucket {
+                bucket: request.bucket.clone(),
+            },
+        };
+
+        let priority = match request.priority {
+            HealChannelPriority::Low => HealPriority::Low,
+            HealChannelPriority::Normal => HealPriority::Normal,
+            HealChannelPriority::High => HealPriority::High,
+            HealChannelPriority::Critical => HealPriority::Urgent,
+        };
+
+        // Convert scan mode
+        let scan_mode = match request.scan_mode {
+            Some(rustfs_common::heal_channel::HealChannelScanMode::Normal) => {
+                rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN
+            }
+            Some(rustfs_common::heal_channel::HealChannelScanMode::Deep) => {
+                rustfs_ecstore::heal::heal_commands::HEAL_DEEP_SCAN
+            }
+            None => rustfs_ecstore::heal::heal_commands::HEAL_NORMAL_SCAN,
+        };
+
+        // Build HealOptions with all available fields
+        let mut options = HealOptions {
+            scan_mode,
+            remove_corrupted: request.remove_corrupted.unwrap_or(false),
+            recreate_missing: request.recreate_missing.unwrap_or(true),
+            update_parity: request.update_parity.unwrap_or(true),
+            recursive: request.recursive.unwrap_or(false),
+            dry_run: request.dry_run.unwrap_or(false),
+            timeout: request.timeout_seconds.map(|secs| std::time::Duration::from_secs(secs)),
+            pool_index: request.pool_index,
+            set_index: request.set_index,
+        };
+
+        // Apply force_start overrides
+        if request.force_start {
+            options.remove_corrupted = true;
+            options.recreate_missing = true;
+            options.update_parity = true;
+        }
+
+        Ok(HealRequest::new(heal_type, options, priority))
+    }
+
+    /// Get response sender for external use
+    pub fn get_response_sender(&self) -> mpsc::UnboundedSender<HealChannelResponse> {
+        self.response_sender.clone()
+    }
+}

crates/ahm/src/heal/event.rs

@@ -246,9 +246,7 @@ impl HealEvent {
                 actual_checksum,
                 ..
             } => {
-                format!(
-                    "Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}"
-                )
+                format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}")
             }
             HealEvent::BucketMetadataCorruption {
                 bucket, corruption_type, ..

crates/ahm/src/heal/manager.rs

@@ -52,7 +52,7 @@ impl Default for HealConfig {
     fn default() -> Self {
         Self {
             enable_auto_heal: true,
-            heal_interval: Duration::from_secs(60), // 1 minute
+            heal_interval: Duration::from_secs(10), // 10 seconds
             max_concurrent_heals: 4,
             task_timeout: Duration::from_secs(300), // 5 minutes
             queue_size: 1000,

crates/ahm/src/heal/mod.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod channel;
 pub mod event;
 pub mod manager;
 pub mod progress;

crates/ahm/src/heal/task.rs

@@ -94,6 +94,10 @@ pub struct HealOptions {
     pub dry_run: bool,
     /// Timeout
     pub timeout: Option<Duration>,
+    /// pool index
+    pub pool_index: Option<usize>,
+    /// set index
+    pub set_index: Option<usize>,
 }
 
 impl Default for HealOptions {
@@ -106,6 +110,8 @@ impl Default for HealOptions {
             recursive: false,
             dry_run: false,
             timeout: Some(Duration::from_secs(300)), // 5 minutes default timeout
+            pool_index: None,
+            set_index: None,
         }
     }
 }
@@ -339,8 +345,8 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: false,
-            pool: None,
-            set: None,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
         };
 
         match self.storage.heal_object(bucket, object, version_id, &heal_opts).await {
@@ -491,8 +497,8 @@ impl HealTask {
             scan_mode: self.options.scan_mode,
             update_parity: self.options.update_parity,
             no_lock: false,
-            pool: None,
-            set: None,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
         };
 
         match self.storage.heal_bucket(bucket, &heal_opts).await {
@@ -609,8 +615,8 @@ impl HealTask {
             scan_mode: rustfs_ecstore::heal::heal_commands::HEAL_DEEP_SCAN,
             update_parity: false,
             no_lock: false,
-            pool: None,
-            set: None,
+            pool: self.options.pool_index,
+            set: self.options.set_index,
         };
 
         match self.storage.heal_object(bucket, object, None, &heal_opts).await {

crates/ahm/src/lib.rs

@@ -12,15 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::OnceLock;
+use std::sync::{Arc, OnceLock};
 use tokio_util::sync::CancellationToken;
+use tracing::{error, info};
 
 pub mod error;
 pub mod heal;
 pub mod scanner;
 
 pub use error::{Error, Result};
-pub use heal::{HealManager, HealOptions, HealPriority, HealRequest, HealType};
+pub use heal::{channel::HealChannelProcessor, HealManager, HealOptions, HealPriority, HealRequest, HealType};
 pub use scanner::{
     BucketTargetUsageInfo, BucketUsageInfo, DataUsageInfo, Scanner, ScannerMetrics, load_data_usage_from_backend,
     store_data_usage_in_backend,
@@ -54,3 +55,61 @@ pub fn shutdown_ahm_services() {
         cancel_token.cancel();
     }
 }
+
+/// Global heal manager instance
+static GLOBAL_HEAL_MANAGER: OnceLock<Arc<HealManager>> = OnceLock::new();
+
+/// Global heal channel processor instance
+static GLOBAL_HEAL_CHANNEL_PROCESSOR: OnceLock<Arc<tokio::sync::Mutex<HealChannelProcessor>>> = OnceLock::new();
+
+/// Initialize and start heal manager with channel processor
+pub async fn init_heal_manager_with_channel(
+    storage: Arc<dyn heal::storage::HealStorageAPI>,
+    config: Option<heal::manager::HealConfig>,
+) -> Result<()> {
+    // Create heal manager
+    let heal_manager = Arc::new(HealManager::new(storage, config));
+
+    // Start heal manager
+    heal_manager.start().await?;
+
+    // Store global instance
+    GLOBAL_HEAL_MANAGER
+        .set(heal_manager.clone())
+        .map_err(|_| Error::Config("Heal manager already initialized".to_string()))?;
+
+    // Initialize heal channel
+    let channel_receiver = rustfs_common::heal_channel::init_heal_channel();
+
+    // Create channel processor
+    let channel_processor = HealChannelProcessor::new(heal_manager);
+
+    // Store channel processor instance first
+    GLOBAL_HEAL_CHANNEL_PROCESSOR
+        .set(Arc::new(tokio::sync::Mutex::new(channel_processor)))
+        .map_err(|_| Error::Config("Heal channel processor already initialized".to_string()))?;
+
+    // Start channel processor in background
+    let receiver = channel_receiver;
+    tokio::spawn(async move {
+        if let Some(processor_guard) = GLOBAL_HEAL_CHANNEL_PROCESSOR.get() {
+            let mut processor = processor_guard.lock().await;
+            if let Err(e) = processor.start(receiver).await {
+                error!("Heal channel processor failed: {}", e);
+            }
+        }
+    });
+
+    info!("Heal manager with channel processor initialized successfully");
+    Ok(())
+}
+
+/// Get global heal manager instance
+pub fn get_heal_manager() -> Option<&'static Arc<HealManager>> {
+    GLOBAL_HEAL_MANAGER.get()
+}
+
+/// Get global heal channel processor instance
+pub fn get_heal_channel_processor() -> Option<&'static Arc<tokio::sync::Mutex<HealChannelProcessor>>> {
+    GLOBAL_HEAL_CHANNEL_PROCESSOR.get()
+}