fix(heal): coordinate cluster-wide control operations (#5003)

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -99,6 +99,11 @@ fn validate_heal_control_capability_proof(canonical_ack: &[u8], proof: &[u8]) ->
         .map_err(|_| Error::other("peer returned an invalid heal control capability proof"))
 }
 
+fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8]) -> Result<()> {
+    verify_tonic_rpc_response_proof(canonical_response, proof)
+        .map_err(|_| Error::other("peer returned an invalid heal control response proof"))
+}
+
 #[derive(Clone, Debug)]
 pub struct PeerLiveEventsBatch {
     pub events: Vec<u8>,
@@ -737,6 +742,7 @@ impl PeerRestClient {
         if command.len() > HEAL_CONTROL_PAYLOAD_MAX_SIZE {
             return Err(Error::other("heal control command exceeds size limit"));
         }
+        let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
         self.finalize_result(
             async {
                 let mut client = self
@@ -748,9 +754,10 @@ impl PeerRestClient {
                     .map_err(|_| Error::other("heal control request length cannot be represented"))?;
                 let mut request = Request::new(HealControlRequest {
                     version,
-                    topology_fingerprint,
-                    command: command.into(),
+                    topology_fingerprint: topology_fingerprint.clone(),
+                    command: command.clone().into(),
                 });
+                request.set_timeout(rustfs_protos::heal_control_execution_timeout());
                 set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
                 let response = client.heal_control(request).await?.into_inner();
                 if !response.success {
@@ -760,6 +767,16 @@ impl PeerRestClient {
                             .unwrap_or_else(|| "peer heal control failed without an error".to_string()),
                     ));
                 }
+                if !capability_probe {
+                    let canonical_response = rustfs_protos::canonical_heal_control_response_body(
+                        version,
+                        &topology_fingerprint,
+                        &command,
+                        &response.result,
+                    )
+                    .map_err(|_| Error::other("heal control response length cannot be represented"))?;
+                    validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
+                }
                 Ok(response.result.to_vec())
             }
             .await,
@@ -767,7 +784,8 @@ impl PeerRestClient {
         .await
     }
 
-    /// Confirms that a peer supports heal-control v1 and has the same storage
+    /// Confirms that a peer supports the current heal-control coordination
+    /// contract and has the same storage
     /// topology. Every non-success response is an error so old or divergent
     /// peers cannot be mistaken for compatible ones.
     pub async fn probe_heal_control(&self, topology_fingerprint: String) -> Result<()> {
@@ -1407,6 +1425,24 @@ mod tests {
         assert!(err.to_string().contains("invalid heal control capability proof"));
     }
 
+    #[test]
+    fn heal_control_response_proof_binds_command_and_result() {
+        runtime_sources::ensure_test_rpc_secret();
+        let canonical = rustfs_protos::canonical_heal_control_response_body(2, "fingerprint", b"query", b"result")
+            .expect("small response should encode");
+        let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("test proof should sign");
+        assert!(validate_heal_control_response_proof(&canonical, &proof).is_ok());
+
+        for tampered in [
+            rustfs_protos::canonical_heal_control_response_body(2, "fingerprint", b"cancel", b"result").unwrap(),
+            rustfs_protos::canonical_heal_control_response_body(2, "fingerprint", b"query", b"tampered").unwrap(),
+        ] {
+            let err = validate_heal_control_response_proof(&tampered, &proof)
+                .expect_err("proof must not authenticate a different command or result");
+            assert!(err.to_string().contains("invalid heal control response proof"));
+        }
+    }
+
     #[tokio::test]
     async fn peer_rest_client_prepare_retry_clears_offline_gate() {
         // finalize_result sets the offline gate on a network error; without

crates/heal/src/heal/channel.rs

@@ -34,6 +34,7 @@ const LOG_SUBSYSTEM_CHANNEL: &str = "channel";
 const EVENT_HEAL_CHANNEL_STATE: &str = "heal_channel_state";
 const EVENT_HEAL_CHANNEL_REQUEST: &str = "heal_channel_request";
 const EVENT_HEAL_CHANNEL_RESPONSE: &str = "heal_channel_response";
+const MAX_HEAL_STATUS_PAYLOAD_SIZE: usize = 8 * 1024 * 1024;
 
 fn admission_response(request_id: String, admission: HealAdmissionResult) -> HealChannelResponse {
     let (success, error) = match admission {
@@ -61,11 +62,56 @@ pub struct HealChannelProcessor {
 }
 
 #[derive(Serialize)]
-struct HealTaskStatusPayload {
-    summary: String,
-    items: Vec<HealResultItem>,
+struct HealTaskStatusPayload<'a> {
+    summary: &'a str,
+    items: &'a [HealResultItem],
+    truncated: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
-    progress: Option<HealProgress>,
+    progress: Option<&'a HealProgress>,
+}
+
+fn encode_heal_task_status_payload(
+    summary: &str,
+    mut items: Vec<HealResultItem>,
+    progress: Option<&HealProgress>,
+    mut truncated: bool,
+) -> Result<(Vec<u8>, bool)> {
+    loop {
+        let data = serde_json::to_vec(&HealTaskStatusPayload {
+            summary,
+            items: &items,
+            truncated,
+            progress,
+        })
+        .map_err(|e| Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
+        if data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE {
+            return Ok((data, truncated));
+        }
+        if items.is_empty() {
+            return Err(Error::Serialization("heal task status metadata exceeds size limit".to_string()));
+        }
+        truncated = true;
+        items.truncate(items.len() / 2);
+    }
+}
+
+fn heal_status_detail(detail: Option<String>, truncated: bool) -> Option<String> {
+    if !truncated {
+        return detail;
+    }
+    let truncation = "heal result items were truncated";
+    Some(detail.map_or_else(|| truncation.to_string(), |detail| format!("{detail}; {truncation}")))
+}
+
+fn encode_heal_status_response(
+    summary: &str,
+    items: Vec<HealResultItem>,
+    progress: Option<&HealProgress>,
+    detail: Option<String>,
+    truncated: bool,
+) -> Result<(Vec<u8>, Option<String>)> {
+    let (data, truncated) = encode_heal_task_status_payload(summary, items, progress, truncated)?;
+    Ok((data, heal_status_detail(detail, truncated)))
 }
 
 impl HealChannelProcessor {
@@ -79,6 +125,37 @@ impl HealChannelProcessor {
         }
     }
 
+    /// Execute a start directly against the manager without entering the
+    /// process-global unbounded command queue.
+    pub async fn execute_start_request(&self, request: HealChannelRequest) -> Result<HealAdmissionReceipt> {
+        let (response_tx, response_rx) = oneshot::channel();
+        self.process_start_request(request, false, true, response_tx).await?;
+        response_rx
+            .await
+            .map_err(|err| Error::other(format!("heal receipt channel closed: {err}")))?
+            .map_err(Error::other)
+    }
+
+    /// Execute a token query directly against the manager.
+    pub async fn execute_query_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
+        let (response_tx, response_rx) = oneshot::channel();
+        self.process_query_request(heal_path, client_token, response_tx).await?;
+        response_rx
+            .await
+            .map_err(|err| Error::other(format!("heal query channel closed: {err}")))?
+            .map_err(Error::other)
+    }
+
+    /// Execute cancellation directly against the manager.
+    pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
+        let (response_tx, response_rx) = oneshot::channel();
+        self.process_cancel_request(heal_path, client_token, response_tx).await?;
+        response_rx
+            .await
+            .map_err(|err| Error::other(format!("heal cancel channel closed: {err}")))?
+            .map_err(Error::other)
+    }
+
     /// Start processing legacy heal channel requests.
     pub async fn start(&mut self, receiver: HealChannelReceiver) -> Result<()> {
         let (receipt_sender, receipt_receiver) = mpsc::unbounded_channel();
@@ -326,46 +403,66 @@ impl HealChannelProcessor {
             self.heal_manager.get_task_report_for_path(&heal_path, &client_token).await
         };
 
-        let (summary, detail, items, progress) = match report {
+        let (summary, detail, items, truncated, progress) = match report {
             Ok(HealTaskReport {
                 status: HealTaskStatus::Pending | HealTaskStatus::Running,
                 result_items,
+                result_items_truncated,
                 progress,
-            }) => ("running".to_string(), None, result_items, progress),
+            }) => ("running".to_string(), None, result_items, result_items_truncated, progress),
             Ok(HealTaskReport {
                 status: HealTaskStatus::Retrying { error, retry_attempt },
                 result_items,
+                result_items_truncated,
                 progress,
             }) => (
                 "running".to_string(),
                 Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")),
                 result_items,
+                result_items_truncated,
                 progress,
             ),
             Ok(HealTaskReport {
                 status: HealTaskStatus::Completed,
                 result_items,
+                result_items_truncated,
                 progress,
-            }) => ("finished".to_string(), None, result_items, progress),
+            }) => ("finished".to_string(), None, result_items, result_items_truncated, progress),
             Ok(HealTaskReport {
                 status: HealTaskStatus::Cancelled,
                 result_items,
+                result_items_truncated,
                 progress,
-            }) => ("stopped".to_string(), Some("heal task cancelled".to_string()), result_items, progress),
+            }) => (
+                "stopped".to_string(),
+                Some("heal task cancelled".to_string()),
+                result_items,
+                result_items_truncated,
+                progress,
+            ),
             Ok(HealTaskReport {
                 status: HealTaskStatus::Timeout,
                 result_items,
+                result_items_truncated,
                 progress,
-            }) => ("stopped".to_string(), Some("heal task timed out".to_string()), result_items, progress),
+            }) => (
+                "stopped".to_string(),
+                Some("heal task timed out".to_string()),
+                result_items,
+                result_items_truncated,
+                progress,
+            ),
             Ok(HealTaskReport {
                 status: HealTaskStatus::Failed { error },
                 result_items,
+                result_items_truncated,
                 progress,
-            }) => ("stopped".to_string(), Some(error), result_items, progress),
+            }) => ("stopped".to_string(), Some(error), result_items, result_items_truncated, progress),
             Err(crate::Error::TaskNotFound { .. }) => (
                 "notFound".to_string(),
                 Some("heal task not found or expired".to_string()),
                 Vec::new(),
+                false,
                 None,
             ),
             Err(crate::Error::InvalidClientToken) => {
@@ -393,12 +490,7 @@ impl HealChannelProcessor {
             }
         };
 
-        let data = serde_json::to_vec(&HealTaskStatusPayload {
-            summary,
-            items,
-            progress,
-        })
-        .map_err(|e| crate::Error::Serialization(format!("failed to serialize heal task status: {e}")))?;
+        let (data, detail) = encode_heal_status_response(&summary, items, progress.as_ref(), detail, truncated)?;
 
         let response = HealChannelResponse {
             request_id: client_token,
@@ -697,6 +789,22 @@ mod tests {
         // If we can get the sender, processor was created correctly
     }
 
+    #[test]
+    fn oversized_status_items_are_truncated_before_transport() {
+        let items = vec![HealResultItem {
+            detail: "x".repeat(MAX_HEAL_STATUS_PAYLOAD_SIZE + 1),
+            ..Default::default()
+        }];
+
+        let (data, detail) = encode_heal_status_response("running", items, None, None, false).unwrap();
+
+        assert!(data.len() <= MAX_HEAL_STATUS_PAYLOAD_SIZE);
+        let payload: serde_json::Value = serde_json::from_slice(&data).unwrap();
+        assert_eq!(payload["truncated"], true);
+        assert!(payload["items"].as_array().unwrap().is_empty());
+        assert_eq!(detail.as_deref(), Some("heal result items were truncated"));
+    }
+
     #[test]
     fn admission_response_preserves_all_admission_outcomes() {
         let cases = [
@@ -1339,6 +1447,70 @@ mod tests {
             .expect("receipt processor should stop cleanly");
     }
 
+    #[tokio::test]
+    async fn direct_control_execution_preserves_target_dedup_and_token_ownership() {
+        let manager = create_test_heal_manager();
+        let processor = HealChannelProcessor::new(manager);
+        let original_id = uuid::Uuid::new_v4().to_string();
+        let request = HealChannelRequest {
+            id: original_id.clone(),
+            bucket: "bucket-a".to_string(),
+            object_prefix: Some("object".to_string()),
+            priority: HealChannelPriority::High,
+            source: HealRequestSource::Admin,
+            ..Default::default()
+        };
+
+        let accepted = processor
+            .execute_start_request(request.clone())
+            .await
+            .expect("first direct start should be admitted");
+        assert_eq!(accepted.result, HealAdmissionResult::Accepted);
+        assert_eq!(accepted.task_id, original_id);
+
+        let mut duplicate = request;
+        duplicate.id = uuid::Uuid::new_v4().to_string();
+        let merged = processor
+            .execute_start_request(duplicate)
+            .await
+            .expect("duplicate direct start should merge");
+        assert_eq!(merged.result, HealAdmissionResult::Merged);
+        assert_eq!(merged.task_id, accepted.task_id);
+
+        let other = processor
+            .execute_start_request(HealChannelRequest {
+                id: uuid::Uuid::new_v4().to_string(),
+                bucket: "bucket-b".to_string(),
+                object_prefix: Some("object".to_string()),
+                priority: HealChannelPriority::High,
+                source: HealRequestSource::Admin,
+                ..Default::default()
+            })
+            .await
+            .expect("different target should be admitted independently");
+        assert_eq!(other.result, HealAdmissionResult::Accepted);
+        assert_ne!(other.task_id, accepted.task_id);
+
+        let status = processor
+            .execute_query_request(String::new(), accepted.task_id.clone())
+            .await
+            .expect("canonical token should be queryable");
+        assert!(status.success);
+        let cancelled = processor
+            .execute_cancel_request(String::new(), accepted.task_id.clone())
+            .await
+            .expect("canonical token should be cancellable");
+        assert!(cancelled.success);
+        let stopped = processor
+            .execute_query_request(String::new(), accepted.task_id)
+            .await
+            .expect("cancelled task status should remain queryable");
+        assert!(stopped.success);
+        assert_eq!(stopped.error.as_deref(), Some("heal task not found or expired"));
+        let status: serde_json::Value = serde_json::from_slice(stopped.data.as_deref().unwrap()).unwrap();
+        assert_eq!(status["summary"], "notFound");
+    }
+
     #[tokio::test]
     async fn test_process_start_request_returns_error_on_invalid_request() {
         let heal_manager = create_test_heal_manager();

crates/heal/src/heal/manager.rs

@@ -179,6 +179,7 @@ struct CompletedHealStatus {
     heal_type: HealType,
     status: HealTaskStatus,
     result_items: Vec<HealResultItem>,
+    result_items_truncated: bool,
     completed_at: SystemTime,
 }
 
@@ -198,6 +199,7 @@ struct RetryingHeal {
 pub struct HealTaskReport {
     pub status: HealTaskStatus,
     pub result_items: Vec<HealResultItem>,
+    pub result_items_truncated: bool,
     pub progress: Option<HealProgress>,
 }
 
@@ -1600,6 +1602,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: task.get_status().await,
                     result_items: task.get_result_items().await,
+                    result_items_truncated: task.result_items_truncated(),
                     progress: Some(task.get_progress().await),
                 });
             }
@@ -1611,6 +1614,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: retrying.status(),
                     result_items: Vec::new(),
+                    result_items_truncated: false,
                     progress: None,
                 });
             }
@@ -1625,6 +1629,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: completed.status.clone(),
                     result_items: completed.result_items.clone(),
+                    result_items_truncated: completed.result_items_truncated,
                     progress: None,
                 });
             }
@@ -1636,6 +1641,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: HealTaskStatus::Pending,
                     result_items: Vec::new(),
+                    result_items_truncated: false,
                     progress: None,
                 });
             }
@@ -1647,6 +1653,7 @@ impl HealManager {
             return Ok(HealTaskReport {
                 status: completed.status.clone(),
                 result_items: completed.result_items.clone(),
+                result_items_truncated: completed.result_items_truncated,
                 progress: None,
             });
         }
@@ -1666,6 +1673,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: task.get_status().await,
                     result_items: task.get_result_items().await,
+                    result_items_truncated: task.result_items_truncated(),
                     progress: Some(task.get_progress().await),
                 });
             }
@@ -1679,6 +1687,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: retrying.status(),
                     result_items: Vec::new(),
+                    result_items_truncated: false,
                     progress: None,
                 });
             }
@@ -1694,6 +1703,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: completed.status.clone(),
                     result_items: completed.result_items.clone(),
+                    result_items_truncated: completed.result_items_truncated,
                     progress: None,
                 });
             }
@@ -1705,6 +1715,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: HealTaskStatus::Pending,
                     result_items: Vec::new(),
+                    result_items_truncated: false,
                     progress: None,
                 });
             }
@@ -1719,6 +1730,7 @@ impl HealManager {
                 return Ok(HealTaskReport {
                     status: completed.status.clone(),
                     result_items: completed.result_items.clone(),
+                    result_items_truncated: completed.result_items_truncated,
                     progress: None,
                 });
             }
@@ -2618,6 +2630,7 @@ impl HealManager {
                             heal_type: completed_task.heal_type.clone(),
                             status: completed_status.clone(),
                             result_items: completed_task.get_result_items().await,
+                            result_items_truncated: completed_task.result_items_truncated(),
                             completed_at: SystemTime::now(),
                         };
                         let mut completed_heals_guard = completed_heals_clone.lock().await;
@@ -3848,6 +3861,7 @@ mod tests {
                     retry_attempt: request.retry_attempts,
                 },
                 result_items: Vec::new(),
+                result_items_truncated: false,
                 completed_at: SystemTime::now(),
             },
         );
@@ -4411,6 +4425,7 @@ mod tests {
                 },
                 status: HealTaskStatus::Completed,
                 result_items: Vec::new(),
+                result_items_truncated: false,
                 completed_at: SystemTime::now(),
             },
         );
@@ -4444,6 +4459,7 @@ mod tests {
                     object_size: 1024,
                     ..Default::default()
                 }],
+                result_items_truncated: true,
                 completed_at: SystemTime::now(),
             },
         );
@@ -4452,6 +4468,7 @@ mod tests {
             .get_task_report_for_path("bucket/object", "completed-token")
             .await
             .expect("recent completed task report should be queryable");
+        assert!(report.result_items_truncated);
 
         assert_eq!(report.status, HealTaskStatus::Completed);
         assert_eq!(report.result_items.len(), 1);

crates/heal/src/heal/task.rs

@@ -43,6 +43,7 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object";
 const EVENT_HEAL_TASK_STATE: &str = "heal_task_state";
 const EVENT_HEAL_OBJECT_STAGE: &str = "heal_object_stage";
 const EVENT_HEAL_OBJECT_MISSING: &str = "heal_object_missing";
+const MAX_RETAINED_HEAL_RESULT_ITEMS: usize = 1024;
 const EVENT_HEAL_OBJECT_RESULT: &str = "heal_object_result";
 const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3;
 const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5;
@@ -306,6 +307,7 @@ pub struct HealTask {
     pub progress: Arc<RwLock<HealProgress>>,
     /// Result items collected from storage heal calls.
     pub result_items: Arc<RwLock<Vec<HealResultItem>>>,
+    result_items_truncated: Arc<AtomicBool>,
     batch_failure: Arc<RwLock<Option<BatchHealFailure>>>,
     batch_failure_recorded: Arc<AtomicBool>,
     /// Created time
@@ -337,6 +339,7 @@ impl HealTask {
             status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
             progress: Arc::new(RwLock::new(HealProgress::new())),
             result_items: Arc::new(RwLock::new(Vec::new())),
+            result_items_truncated: Arc::new(AtomicBool::new(false)),
             batch_failure: Arc::new(RwLock::new(None)),
             batch_failure_recorded: Arc::new(AtomicBool::new(false)),
             created_at: request.created_at,
@@ -741,8 +744,17 @@ impl HealTask {
         self.result_items.read().await.clone()
     }
 
+    pub fn result_items_truncated(&self) -> bool {
+        self.result_items_truncated.load(Ordering::Relaxed)
+    }
+
     async fn record_result_item(&self, result: HealResultItem) {
-        self.result_items.write().await.push(result);
+        let mut result_items = self.result_items.write().await;
+        if result_items.len() < MAX_RETAINED_HEAL_RESULT_ITEMS {
+            result_items.push(result);
+        } else {
+            self.result_items_truncated.store(true, Ordering::Relaxed);
+        }
     }
 
     // specific heal implementation method
@@ -2656,6 +2668,19 @@ mod tests {
         assert_eq!(result_items.iter().filter(|item| item.object_size == 1).count(), 2);
     }
 
+    #[tokio::test]
+    async fn result_items_are_bounded_and_report_truncation() {
+        let storage = Arc::new(MockStorage::default());
+        let task = HealTask::from_request(HealRequest::bucket("bucket-a".to_string()), storage);
+
+        for _ in 0..=MAX_RETAINED_HEAL_RESULT_ITEMS {
+            task.record_result_item(HealResultItem::default()).await;
+        }
+
+        assert_eq!(task.get_result_items().await.len(), MAX_RETAINED_HEAL_RESULT_ITEMS);
+        assert!(task.result_items_truncated());
+    }
+
     #[tokio::test]
     async fn test_recursive_bucket_heal_skips_object_dir_candidates() {
         let storage = Arc::new(MockStorage {

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1114,6 +1114,8 @@ pub struct HealControlResponse {
     pub result: ::prost::bytes::Bytes,
     #[prost(string, optional, tag = "3")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub response_proof: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetMetacacheListingRequest {

crates/protos/src/heal_control.rs

@@ -29,8 +29,10 @@ use std::{fmt, io::Cursor, io::Write};
 
 const ENVELOPE_VERSION: u8 = 1;
 pub const ENVELOPE_MAX_SIZE: usize = 64 * 1024;
+pub const RESULT_MAX_SIZE: usize = 16 * 1024 * 1024;
 pub const NONCE_SIZE: usize = 16;
 pub const MAX_LIFETIME_MS: i64 = 30_000;
+const MAX_CLOCK_SKEW_MS: i64 = 5_000;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct RequestMetadata {
@@ -136,6 +138,38 @@ impl TryFrom<HealChannelRequest> for StartCommand {
     }
 }
 
+impl StartCommand {
+    fn into_channel_request(self, request_id: String) -> Result<HealChannelRequest, String> {
+        Ok(HealChannelRequest {
+            id: request_id,
+            disk: self.disk,
+            bucket: self.bucket,
+            object_prefix: self.object_prefix,
+            object_version_id: self.object_version_id,
+            force_start: self.force_start,
+            priority: self.priority.into(),
+            pool_index: self
+                .pool_index
+                .map(usize::try_from)
+                .transpose()
+                .map_err(|_| "heal pool index exceeds platform range".to_string())?,
+            set_index: self
+                .set_index
+                .map(usize::try_from)
+                .transpose()
+                .map_err(|_| "heal set index exceeds platform range".to_string())?,
+            scan_mode: self.scan_mode,
+            remove_corrupted: self.remove_corrupted,
+            recreate_missing: self.recreate_missing,
+            update_parity: self.update_parity,
+            recursive: self.recursive,
+            dry_run: self.dry_run,
+            timeout_seconds: self.timeout_seconds,
+            source: self.source,
+        })
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
 pub enum Command {
@@ -144,6 +178,13 @@ pub enum Command {
     Cancel { heal_path: String, client_token: String },
 }
 
+#[derive(Debug)]
+pub enum ExecutableCommand {
+    Start { request: HealChannelRequest },
+    Query { heal_path: String, client_token: String },
+    Cancel { heal_path: String, client_token: String },
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct Envelope {
@@ -214,13 +255,42 @@ impl Envelope {
         }
         match &self.command {
             Command::Start { .. } => {}
-            Command::Query { client_token, .. } | Command::Cancel { client_token, .. } if client_token.is_empty() => {
+            Command::Query { client_token, .. } if client_token.is_empty() => {
                 return Err("heal control client token is empty".to_string());
             }
             Command::Query { .. } | Command::Cancel { .. } => {}
         }
         Ok(())
     }
+
+    pub fn validate_execution(&self, now_unix_ms: i64, expected_coordinator_epoch: u64) -> Result<(), String> {
+        self.validate()?;
+        if self.coordinator_epoch != expected_coordinator_epoch {
+            return Err("heal control coordinator epoch does not match".to_string());
+        }
+        if self.issued_at_unix_ms > now_unix_ms.saturating_add(MAX_CLOCK_SKEW_MS) {
+            return Err("heal control request was issued in the future".to_string());
+        }
+        if self.expires_at_unix_ms <= now_unix_ms {
+            return Err("heal control request expired".to_string());
+        }
+        Ok(())
+    }
+
+    pub fn into_execution(self) -> Result<(String, u64, ExecutableCommand), String> {
+        let command = match self.command {
+            Command::Start { request } => ExecutableCommand::Start {
+                request: request.into_channel_request(self.request_id.clone())?,
+            },
+            Command::Query { heal_path, client_token } => ExecutableCommand::Query { heal_path, client_token },
+            Command::Cancel { heal_path, client_token } => ExecutableCommand::Cancel { heal_path, client_token },
+        };
+        Ok((self.request_id, self.coordinator_epoch, command))
+    }
+
+    pub const fn expires_at_unix_ms(&self) -> i64 {
+        self.expires_at_unix_ms
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -249,6 +319,16 @@ impl Admission {
     pub const fn is_admitted(self) -> bool {
         matches!(self, Self::Accepted | Self::Merged)
     }
+
+    pub const fn into_heal_admission_result(self) -> HealAdmissionResult {
+        match self {
+            Self::Accepted => HealAdmissionResult::Accepted,
+            Self::Merged => HealAdmissionResult::Merged,
+            Self::Full => HealAdmissionResult::Full,
+            Self::DroppedQueueFull => HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull),
+            Self::DroppedPolicy => HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped),
+        }
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -274,20 +354,20 @@ impl<'de> Visitor<'de> for BoundedBytesVisitor {
     type Value = BoundedBytes;
 
     fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(formatter, "at most {ENVELOPE_MAX_SIZE} bytes")
+        write!(formatter, "at most {RESULT_MAX_SIZE} bytes")
     }
 
     fn visit_seq<A>(self, mut sequence: A) -> Result<Self::Value, A::Error>
     where
         A: SeqAccess<'de>,
     {
-        if sequence.size_hint().is_some_and(|length| length > ENVELOPE_MAX_SIZE) {
+        if sequence.size_hint().is_some_and(|length| length > RESULT_MAX_SIZE) {
             return Err(serde::de::Error::custom("heal control response data exceeds size limit"));
         }
 
         let mut bytes = Vec::new();
         while let Some(byte) = sequence.next_element()? {
-            if bytes.len() == ENVELOPE_MAX_SIZE {
+            if bytes.len() == RESULT_MAX_SIZE {
                 return Err(serde::de::Error::custom("heal control response data exceeds size limit"));
             }
             bytes.push(byte);
@@ -358,13 +438,6 @@ impl ResultEnvelope {
         validate_uuid(&self.request_id, "result request")?;
         match &self.outcome {
             Outcome::Start { task_id, .. } => validate_uuid(task_id, "result task")?,
-            Outcome::Channel {
-                success: true,
-                error: Some(_),
-                ..
-            } => {
-                return Err("successful heal control result contains an error".to_string());
-            }
             Outcome::Channel {
                 success: false,
                 error: None,
@@ -390,6 +463,17 @@ impl ResultEnvelope {
         }
         Ok(())
     }
+
+    pub fn into_outcome(self, expected_request_id: &str, expected_epoch: u64) -> Result<Outcome, String> {
+        self.validate()?;
+        if self.request_id != expected_request_id {
+            return Err("heal control result request ID does not match".to_string());
+        }
+        if self.coordinator_epoch != expected_epoch {
+            return Err("heal control result coordinator epoch does not match".to_string());
+        }
+        Ok(self.outcome)
+    }
 }
 
 fn validate_uuid(value: &str, field: &str) -> Result<(), String> {
@@ -400,8 +484,8 @@ fn validate_uuid(value: &str, field: &str) -> Result<(), String> {
     Ok(())
 }
 
-fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str) -> Result<T, String> {
-    if data.len() > ENVELOPE_MAX_SIZE {
+fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str, max_size: usize) -> Result<T, String> {
+    if data.len() > max_size {
         return Err(format!("{value_name} exceeds size limit"));
     }
     let mut deserializer = Deserializer::new(Cursor::new(data));
@@ -414,35 +498,36 @@ fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str) -> Result
 
 pub fn encode_envelope(envelope: &Envelope) -> Result<Vec<u8>, String> {
     envelope.validate()?;
-    encode_bounded(envelope, "heal control envelope")
+    encode_bounded(envelope, "heal control envelope", ENVELOPE_MAX_SIZE)
 }
 
 pub fn decode_envelope(data: &[u8]) -> Result<Envelope, String> {
-    let envelope: Envelope = decode(data, "heal control envelope")?;
+    let envelope: Envelope = decode(data, "heal control envelope", ENVELOPE_MAX_SIZE)?;
     envelope.validate()?;
     Ok(envelope)
 }
 
 pub fn encode_result(result: &ResultEnvelope) -> Result<Vec<u8>, String> {
     result.validate()?;
-    encode_bounded(result, "heal control result")
+    encode_bounded(result, "heal control result", RESULT_MAX_SIZE)
 }
 
 pub fn decode_result(data: &[u8]) -> Result<ResultEnvelope, String> {
-    let result: ResultEnvelope = decode(data, "heal control result")?;
+    let result: ResultEnvelope = decode(data, "heal control result", RESULT_MAX_SIZE)?;
     result.validate()?;
     Ok(result)
 }
 
-fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, String> {
+fn encode_bounded(value: &impl Serialize, value_name: &str, max_size: usize) -> Result<Vec<u8>, String> {
     struct BoundedWriter {
         bytes: Vec<u8>,
         exceeded: bool,
+        max_size: usize,
     }
 
     impl Write for BoundedWriter {
         fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
-            let remaining = ENVELOPE_MAX_SIZE.saturating_sub(self.bytes.len());
+            let remaining = self.max_size.saturating_sub(self.bytes.len());
             if data.len() > remaining {
                 self.exceeded = true;
                 return Err(std::io::Error::other("heal control value exceeds size limit"));
@@ -459,6 +544,7 @@ fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, S
     let mut writer = BoundedWriter {
         bytes: Vec::with_capacity(1024),
         exceeded: false,
+        max_size,
     };
     let result = value.serialize(&mut rmp_serde::Serializer::new(&mut writer).with_struct_map());
     if writer.exceeded {
@@ -471,7 +557,8 @@ fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, S
 #[cfg(test)]
 mod tests {
     use super::{
-        Admission, ENVELOPE_MAX_SIZE, Envelope, Outcome, RequestMetadata, ResultEnvelope, decode_envelope, decode_result,
+        Admission, ENVELOPE_MAX_SIZE, Envelope, Outcome, RESULT_MAX_SIZE, RequestMetadata, ResultEnvelope, decode_envelope,
+        decode_result, encode_result,
     };
     use rustfs_common::heal_channel::{HealChannelRequest, HealChannelResponse, HealRequestSource};
     use serde::de::{DeserializeSeed, SeqAccess, Visitor, value::Error as ValueError};
@@ -640,11 +727,7 @@ mod tests {
         assert!(Envelope::start(test_request(request_id.clone()), metadata(1, 0)).is_err());
         assert!(Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 1_000, 31_001, 7),).is_err());
         assert!(Envelope::query(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_err());
-        assert!(
-            Envelope::cancel(request_id.clone(), metadata(1, 7), String::new(), String::new())
-                .unwrap_err()
-                .contains("token is empty")
-        );
+        assert!(Envelope::cancel(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_ok());
 
         let mut noncanonical_request = test_request(request_id.to_uppercase());
         assert!(Envelope::start(noncanonical_request.clone(), metadata(1, 7)).is_err());
@@ -689,6 +772,15 @@ mod tests {
         let unknown = rmp_serde::to_vec_named(&unknown).unwrap();
         assert!(decode_envelope(&unknown).unwrap_err().contains("unknown field"));
 
+        let executable =
+            Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 10_000, 20_000, 7)).unwrap();
+        assert!(executable.validate_execution(15_000, 7).is_ok());
+        assert!(executable.validate_execution(20_000, 7).unwrap_err().contains("expired"));
+        assert!(executable.validate_execution(4_999, 7).unwrap_err().contains("future"));
+        let wrong_epoch =
+            Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 10_000, 20_000, 8)).unwrap();
+        assert!(wrong_epoch.validate_execution(15_000, 7).unwrap_err().contains("epoch"));
+
         assert!(
             ResultEnvelope::channel(
                 uuid::Uuid::new_v4().to_string(),
@@ -711,11 +803,25 @@ mod tests {
                 Outcome::Channel {
                     success: true,
                     data: None,
-                    error: Some("unexpected".to_string()),
+                    error: Some("status detail".to_string()),
                 },
             )
-            .is_err()
+            .is_ok()
         );
+
+        let large_result = ResultEnvelope::new(
+            request_id.clone(),
+            7,
+            Outcome::Channel {
+                success: true,
+                data: Some(vec![7; ENVELOPE_MAX_SIZE + 1]),
+                error: None,
+            },
+        )
+        .unwrap();
+        let large_result = encode_result(&large_result).expect("status results may exceed the request envelope limit");
+        assert!(large_result.len() > ENVELOPE_MAX_SIZE);
+        assert!(decode_result(&large_result).is_ok());
         assert!(
             ResultEnvelope::new(
                 request_id.clone(),
@@ -823,7 +929,7 @@ mod tests {
             }
 
             fn size_hint(&self) -> Option<usize> {
-                Some(ENVELOPE_MAX_SIZE + 1)
+                Some(RESULT_MAX_SIZE + 1)
             }
         }
 

crates/protos/src/lib.rs

@@ -97,10 +97,31 @@ fn internode_http2_keep_alive_timeout() -> Duration {
 }
 
 fn internode_rpc_timeout() -> Duration {
-    Duration::from_secs(rustfs_utils::get_env_u64(
+    normalize_internode_rpc_timeout(Duration::from_secs(rustfs_utils::get_env_u64(
         rustfs_config::ENV_INTERNODE_RPC_TIMEOUT_SECS,
         rustfs_config::DEFAULT_INTERNODE_RPC_TIMEOUT_SECS,
-    ))
+    )))
+}
+
+fn normalize_internode_rpc_timeout(timeout: Duration) -> Duration {
+    timeout.max(Duration::from_secs(1))
+}
+
+/// Budget for one heal-control execution, kept below the transport timeout so
+/// the coordinator stops waiting for admission before the caller gives up.
+pub fn heal_control_execution_timeout() -> Duration {
+    heal_control_execution_timeout_for(internode_rpc_timeout())
+}
+
+fn heal_control_execution_timeout_for(transport_timeout: Duration) -> Duration {
+    const TRANSPORT_GUARD: Duration = Duration::from_secs(1);
+    let transport_timeout = transport_timeout.max(Duration::from_secs(1));
+    transport_timeout
+        .saturating_sub(TRANSPORT_GUARD.min(transport_timeout / 2))
+        .max(Duration::from_millis(1))
+        .min(Duration::from_millis(
+            u64::try_from(heal_control::MAX_LIFETIME_MS).expect("positive heal control lifetime must fit u64"),
+        ))
 }
 
 fn internode_rpc_tcp_nodelay() -> bool {
@@ -141,9 +162,20 @@ pub fn internode_rpc_max_message_size() -> usize {
     rustfs_utils::get_env_usize(rustfs_config::ENV_INTERNODE_RPC_MAX_MESSAGE_SIZE, DEFAULT_GRPC_SERVER_MESSAGE_LEN)
 }
 
-pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = 65 * 1024;
-pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 1;
-pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v1\0";
+pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
+pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
+pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
+
+pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
+    let prefix = topology_fingerprint
+        .get(..16)
+        .ok_or("heal control topology fingerprint is too short")?;
+    let epoch = u64::from_str_radix(prefix, 16).map_err(|_| "heal control topology fingerprint is not hexadecimal")?;
+    if epoch == 0 {
+        return Err("heal control topology epoch is zero");
+    }
+    Ok(epoch)
+}
 
 pub fn heal_control_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
     let mut probe = Vec::with_capacity(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
@@ -165,7 +197,7 @@ pub fn canonical_heal_control_request_body(
     topology_fingerprint: &str,
     command: &[u8],
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-heal-control-v1\0";
+    const DOMAIN: &[u8] = b"rustfs-heal-control-v2\0";
 
     let fingerprint = topology_fingerprint.as_bytes();
     let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len());
@@ -185,7 +217,7 @@ pub fn canonical_heal_control_capability_ack(
     topology_fingerprint: &str,
     probe: &[u8],
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v1\0";
+    const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v2\0";
 
     let fingerprint = topology_fingerprint.as_bytes();
     let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + probe.len());
@@ -198,17 +230,42 @@ pub fn canonical_heal_control_capability_ack(
     Ok(body)
 }
 
+pub fn canonical_heal_control_response_body(
+    version: u32,
+    topology_fingerprint: &str,
+    command: &[u8],
+    result: &[u8],
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    const DOMAIN: &[u8] = b"rustfs-heal-control-response-v2\0";
+
+    let fingerprint = topology_fingerprint.as_bytes();
+    let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len() + 8 + result.len());
+    body.extend_from_slice(DOMAIN);
+    body.extend_from_slice(&version.to_be_bytes());
+    body.extend_from_slice(&u64::try_from(fingerprint.len())?.to_be_bytes());
+    body.extend_from_slice(fingerprint);
+    body.extend_from_slice(&u64::try_from(command.len())?.to_be_bytes());
+    body.extend_from_slice(command);
+    body.extend_from_slice(&u64::try_from(result.len())?.to_be_bytes());
+    body.extend_from_slice(result);
+    Ok(body)
+}
+
 #[cfg(test)]
 mod heal_control_tests {
     use super::{
-        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack, canonical_heal_control_request_body,
-        heal_control_capability_probe, is_heal_control_capability_probe,
+        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
+        canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
+        heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
+        internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
     };
+    use crate::heal_control;
+    use std::time::Duration;
 
     #[test]
     fn canonical_heal_control_body_binds_every_field_and_boundary() {
         let baseline = canonical_heal_control_request_body(1, "ab", b"c").expect("small request should encode");
-        let mut golden = b"rustfs-heal-control-v1\0".to_vec();
+        let mut golden = b"rustfs-heal-control-v2\0".to_vec();
         golden.extend_from_slice(&1_u32.to_be_bytes());
         golden.extend_from_slice(&2_u64.to_be_bytes());
         golden.extend_from_slice(b"ab");
@@ -236,9 +293,11 @@ mod heal_control_tests {
 
     #[test]
     fn canonical_capability_ack_binds_version_and_topology() {
+        assert_eq!(HEAL_CONTROL_PROTOCOL_VERSION, 2);
+        assert!(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.starts_with(b"rustfs-heal-control-capability-v2"));
         let probe = heal_control_capability_probe(&[7; 16]);
         let ack = canonical_heal_control_capability_ack(1, "ab", &probe).expect("small acknowledgement should encode");
-        let mut golden = b"rustfs-heal-control-capability-ack-v1\0".to_vec();
+        let mut golden = b"rustfs-heal-control-capability-ack-v2\0".to_vec();
         golden.extend_from_slice(&1_u32.to_be_bytes());
         golden.extend_from_slice(&2_u64.to_be_bytes());
         golden.extend_from_slice(b"ab");
@@ -254,6 +313,54 @@ mod heal_control_tests {
         assert!(is_heal_control_capability_probe(&probe));
         assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
     }
+
+    #[test]
+    fn canonical_response_binds_request_and_result() {
+        let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
+        assert_ne!(baseline, canonical_heal_control_response_body(1, "abcdef", b"query", b"result").unwrap());
+        assert_ne!(baseline, canonical_heal_control_response_body(2, "bbcdef", b"query", b"result").unwrap());
+        assert_ne!(baseline, canonical_heal_control_response_body(2, "abcdef", b"cancel", b"result").unwrap());
+        assert_ne!(
+            baseline,
+            canonical_heal_control_response_body(2, "abcdef", b"query", b"tampered").unwrap()
+        );
+    }
+
+    #[test]
+    fn coordinator_epoch_is_stable_and_rejects_invalid_fingerprints() {
+        assert_eq!(heal_control_coordinator_epoch("0123456789abcdefextra"), Ok(0x0123_4567_89ab_cdef));
+        assert_eq!(
+            heal_control_coordinator_epoch("0000000000000000"),
+            Err("heal control topology epoch is zero")
+        );
+        assert_eq!(
+            heal_control_coordinator_epoch("short"),
+            Err("heal control topology fingerprint is too short")
+        );
+        assert_eq!(
+            heal_control_coordinator_epoch("not-hex-value!!!!"),
+            Err("heal control topology fingerprint is not hexadecimal")
+        );
+    }
+
+    #[test]
+    fn execution_budget_precedes_transport_timeout() {
+        let execution = heal_control_execution_timeout();
+        assert!(!execution.is_zero());
+        assert!(execution < internode_rpc_timeout());
+        assert!(execution <= std::time::Duration::from_millis(heal_control::MAX_LIFETIME_MS as u64));
+    }
+
+    #[test]
+    fn execution_budget_is_nonzero_for_zero_transport_configuration() {
+        let normalized_transport = Duration::from_secs(1);
+        assert_eq!(normalize_internode_rpc_timeout(Duration::ZERO), normalized_transport);
+        for configured_transport in [Duration::ZERO, normalized_transport] {
+            let execution = heal_control_execution_timeout_for(configured_transport);
+            assert!(execution > Duration::ZERO);
+            assert!(execution < normalized_transport);
+        }
+    }
 }
 
 /// Whether internode metadata RPCs should send only the msgpack `_bin` payloads and leave the JSON

crates/protos/src/node.proto

@@ -783,6 +783,7 @@ message HealControlResponse {
   bool success = 1;
   bytes result = 2;
   optional string error_info = 3;
+  bytes response_proof = 4;
 }
 
 message GetMetacacheListingRequest {