fix(heal): coordinate cluster-wide control operations (#5003)

crates/protos/src/generated/proto_gen/node_service.rs

@@ -1114,6 +1114,8 @@ pub struct HealControlResponse {
     pub result: ::prost::bytes::Bytes,
     #[prost(string, optional, tag = "3")]
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+    #[prost(bytes = "bytes", tag = "4")]
+    pub response_proof: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetMetacacheListingRequest {

crates/protos/src/heal_control.rs

@@ -29,8 +29,10 @@ use std::{fmt, io::Cursor, io::Write};
 
 const ENVELOPE_VERSION: u8 = 1;
 pub const ENVELOPE_MAX_SIZE: usize = 64 * 1024;
+pub const RESULT_MAX_SIZE: usize = 16 * 1024 * 1024;
 pub const NONCE_SIZE: usize = 16;
 pub const MAX_LIFETIME_MS: i64 = 30_000;
+const MAX_CLOCK_SKEW_MS: i64 = 5_000;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct RequestMetadata {
@@ -136,6 +138,38 @@ impl TryFrom<HealChannelRequest> for StartCommand {
     }
 }
 
+impl StartCommand {
+    fn into_channel_request(self, request_id: String) -> Result<HealChannelRequest, String> {
+        Ok(HealChannelRequest {
+            id: request_id,
+            disk: self.disk,
+            bucket: self.bucket,
+            object_prefix: self.object_prefix,
+            object_version_id: self.object_version_id,
+            force_start: self.force_start,
+            priority: self.priority.into(),
+            pool_index: self
+                .pool_index
+                .map(usize::try_from)
+                .transpose()
+                .map_err(|_| "heal pool index exceeds platform range".to_string())?,
+            set_index: self
+                .set_index
+                .map(usize::try_from)
+                .transpose()
+                .map_err(|_| "heal set index exceeds platform range".to_string())?,
+            scan_mode: self.scan_mode,
+            remove_corrupted: self.remove_corrupted,
+            recreate_missing: self.recreate_missing,
+            update_parity: self.update_parity,
+            recursive: self.recursive,
+            dry_run: self.dry_run,
+            timeout_seconds: self.timeout_seconds,
+            source: self.source,
+        })
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
 pub enum Command {
@@ -144,6 +178,13 @@ pub enum Command {
     Cancel { heal_path: String, client_token: String },
 }
 
+#[derive(Debug)]
+pub enum ExecutableCommand {
+    Start { request: HealChannelRequest },
+    Query { heal_path: String, client_token: String },
+    Cancel { heal_path: String, client_token: String },
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct Envelope {
@@ -214,13 +255,42 @@ impl Envelope {
         }
         match &self.command {
             Command::Start { .. } => {}
-            Command::Query { client_token, .. } | Command::Cancel { client_token, .. } if client_token.is_empty() => {
+            Command::Query { client_token, .. } if client_token.is_empty() => {
                 return Err("heal control client token is empty".to_string());
             }
             Command::Query { .. } | Command::Cancel { .. } => {}
         }
         Ok(())
     }
+
+    pub fn validate_execution(&self, now_unix_ms: i64, expected_coordinator_epoch: u64) -> Result<(), String> {
+        self.validate()?;
+        if self.coordinator_epoch != expected_coordinator_epoch {
+            return Err("heal control coordinator epoch does not match".to_string());
+        }
+        if self.issued_at_unix_ms > now_unix_ms.saturating_add(MAX_CLOCK_SKEW_MS) {
+            return Err("heal control request was issued in the future".to_string());
+        }
+        if self.expires_at_unix_ms <= now_unix_ms {
+            return Err("heal control request expired".to_string());
+        }
+        Ok(())
+    }
+
+    pub fn into_execution(self) -> Result<(String, u64, ExecutableCommand), String> {
+        let command = match self.command {
+            Command::Start { request } => ExecutableCommand::Start {
+                request: request.into_channel_request(self.request_id.clone())?,
+            },
+            Command::Query { heal_path, client_token } => ExecutableCommand::Query { heal_path, client_token },
+            Command::Cancel { heal_path, client_token } => ExecutableCommand::Cancel { heal_path, client_token },
+        };
+        Ok((self.request_id, self.coordinator_epoch, command))
+    }
+
+    pub const fn expires_at_unix_ms(&self) -> i64 {
+        self.expires_at_unix_ms
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -249,6 +319,16 @@ impl Admission {
     pub const fn is_admitted(self) -> bool {
         matches!(self, Self::Accepted | Self::Merged)
     }
+
+    pub const fn into_heal_admission_result(self) -> HealAdmissionResult {
+        match self {
+            Self::Accepted => HealAdmissionResult::Accepted,
+            Self::Merged => HealAdmissionResult::Merged,
+            Self::Full => HealAdmissionResult::Full,
+            Self::DroppedQueueFull => HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull),
+            Self::DroppedPolicy => HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped),
+        }
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -274,20 +354,20 @@ impl<'de> Visitor<'de> for BoundedBytesVisitor {
     type Value = BoundedBytes;
 
     fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(formatter, "at most {ENVELOPE_MAX_SIZE} bytes")
+        write!(formatter, "at most {RESULT_MAX_SIZE} bytes")
     }
 
     fn visit_seq<A>(self, mut sequence: A) -> Result<Self::Value, A::Error>
     where
         A: SeqAccess<'de>,
     {
-        if sequence.size_hint().is_some_and(|length| length > ENVELOPE_MAX_SIZE) {
+        if sequence.size_hint().is_some_and(|length| length > RESULT_MAX_SIZE) {
             return Err(serde::de::Error::custom("heal control response data exceeds size limit"));
         }
 
         let mut bytes = Vec::new();
         while let Some(byte) = sequence.next_element()? {
-            if bytes.len() == ENVELOPE_MAX_SIZE {
+            if bytes.len() == RESULT_MAX_SIZE {
                 return Err(serde::de::Error::custom("heal control response data exceeds size limit"));
             }
             bytes.push(byte);
@@ -358,13 +438,6 @@ impl ResultEnvelope {
         validate_uuid(&self.request_id, "result request")?;
         match &self.outcome {
             Outcome::Start { task_id, .. } => validate_uuid(task_id, "result task")?,
-            Outcome::Channel {
-                success: true,
-                error: Some(_),
-                ..
-            } => {
-                return Err("successful heal control result contains an error".to_string());
-            }
             Outcome::Channel {
                 success: false,
                 error: None,
@@ -390,6 +463,17 @@ impl ResultEnvelope {
         }
         Ok(())
     }
+
+    pub fn into_outcome(self, expected_request_id: &str, expected_epoch: u64) -> Result<Outcome, String> {
+        self.validate()?;
+        if self.request_id != expected_request_id {
+            return Err("heal control result request ID does not match".to_string());
+        }
+        if self.coordinator_epoch != expected_epoch {
+            return Err("heal control result coordinator epoch does not match".to_string());
+        }
+        Ok(self.outcome)
+    }
 }
 
 fn validate_uuid(value: &str, field: &str) -> Result<(), String> {
@@ -400,8 +484,8 @@ fn validate_uuid(value: &str, field: &str) -> Result<(), String> {
     Ok(())
 }
 
-fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str) -> Result<T, String> {
-    if data.len() > ENVELOPE_MAX_SIZE {
+fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str, max_size: usize) -> Result<T, String> {
+    if data.len() > max_size {
         return Err(format!("{value_name} exceeds size limit"));
     }
     let mut deserializer = Deserializer::new(Cursor::new(data));
@@ -414,35 +498,36 @@ fn decode<T: for<'de> Deserialize<'de>>(data: &[u8], value_name: &str) -> Result
 
 pub fn encode_envelope(envelope: &Envelope) -> Result<Vec<u8>, String> {
     envelope.validate()?;
-    encode_bounded(envelope, "heal control envelope")
+    encode_bounded(envelope, "heal control envelope", ENVELOPE_MAX_SIZE)
 }
 
 pub fn decode_envelope(data: &[u8]) -> Result<Envelope, String> {
-    let envelope: Envelope = decode(data, "heal control envelope")?;
+    let envelope: Envelope = decode(data, "heal control envelope", ENVELOPE_MAX_SIZE)?;
     envelope.validate()?;
     Ok(envelope)
 }
 
 pub fn encode_result(result: &ResultEnvelope) -> Result<Vec<u8>, String> {
     result.validate()?;
-    encode_bounded(result, "heal control result")
+    encode_bounded(result, "heal control result", RESULT_MAX_SIZE)
 }
 
 pub fn decode_result(data: &[u8]) -> Result<ResultEnvelope, String> {
-    let result: ResultEnvelope = decode(data, "heal control result")?;
+    let result: ResultEnvelope = decode(data, "heal control result", RESULT_MAX_SIZE)?;
     result.validate()?;
     Ok(result)
 }
 
-fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, String> {
+fn encode_bounded(value: &impl Serialize, value_name: &str, max_size: usize) -> Result<Vec<u8>, String> {
     struct BoundedWriter {
         bytes: Vec<u8>,
         exceeded: bool,
+        max_size: usize,
     }
 
     impl Write for BoundedWriter {
         fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
-            let remaining = ENVELOPE_MAX_SIZE.saturating_sub(self.bytes.len());
+            let remaining = self.max_size.saturating_sub(self.bytes.len());
             if data.len() > remaining {
                 self.exceeded = true;
                 return Err(std::io::Error::other("heal control value exceeds size limit"));
@@ -459,6 +544,7 @@ fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, S
     let mut writer = BoundedWriter {
         bytes: Vec::with_capacity(1024),
         exceeded: false,
+        max_size,
     };
     let result = value.serialize(&mut rmp_serde::Serializer::new(&mut writer).with_struct_map());
     if writer.exceeded {
@@ -471,7 +557,8 @@ fn encode_bounded(value: &impl Serialize, value_name: &str) -> Result<Vec<u8>, S
 #[cfg(test)]
 mod tests {
     use super::{
-        Admission, ENVELOPE_MAX_SIZE, Envelope, Outcome, RequestMetadata, ResultEnvelope, decode_envelope, decode_result,
+        Admission, ENVELOPE_MAX_SIZE, Envelope, Outcome, RESULT_MAX_SIZE, RequestMetadata, ResultEnvelope, decode_envelope,
+        decode_result, encode_result,
     };
     use rustfs_common::heal_channel::{HealChannelRequest, HealChannelResponse, HealRequestSource};
     use serde::de::{DeserializeSeed, SeqAccess, Visitor, value::Error as ValueError};
@@ -640,11 +727,7 @@ mod tests {
         assert!(Envelope::start(test_request(request_id.clone()), metadata(1, 0)).is_err());
         assert!(Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 1_000, 31_001, 7),).is_err());
         assert!(Envelope::query(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_err());
-        assert!(
-            Envelope::cancel(request_id.clone(), metadata(1, 7), String::new(), String::new())
-                .unwrap_err()
-                .contains("token is empty")
-        );
+        assert!(Envelope::cancel(request_id.clone(), metadata(1, 7), String::new(), String::new()).is_ok());
 
         let mut noncanonical_request = test_request(request_id.to_uppercase());
         assert!(Envelope::start(noncanonical_request.clone(), metadata(1, 7)).is_err());
@@ -689,6 +772,15 @@ mod tests {
         let unknown = rmp_serde::to_vec_named(&unknown).unwrap();
         assert!(decode_envelope(&unknown).unwrap_err().contains("unknown field"));
 
+        let executable =
+            Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 10_000, 20_000, 7)).unwrap();
+        assert!(executable.validate_execution(15_000, 7).is_ok());
+        assert!(executable.validate_execution(20_000, 7).unwrap_err().contains("expired"));
+        assert!(executable.validate_execution(4_999, 7).unwrap_err().contains("future"));
+        let wrong_epoch =
+            Envelope::start(test_request(request_id.clone()), RequestMetadata::new([1; 16], 10_000, 20_000, 8)).unwrap();
+        assert!(wrong_epoch.validate_execution(15_000, 7).unwrap_err().contains("epoch"));
+
         assert!(
             ResultEnvelope::channel(
                 uuid::Uuid::new_v4().to_string(),
@@ -711,11 +803,25 @@ mod tests {
                 Outcome::Channel {
                     success: true,
                     data: None,
-                    error: Some("unexpected".to_string()),
+                    error: Some("status detail".to_string()),
                 },
             )
-            .is_err()
+            .is_ok()
         );
+
+        let large_result = ResultEnvelope::new(
+            request_id.clone(),
+            7,
+            Outcome::Channel {
+                success: true,
+                data: Some(vec![7; ENVELOPE_MAX_SIZE + 1]),
+                error: None,
+            },
+        )
+        .unwrap();
+        let large_result = encode_result(&large_result).expect("status results may exceed the request envelope limit");
+        assert!(large_result.len() > ENVELOPE_MAX_SIZE);
+        assert!(decode_result(&large_result).is_ok());
         assert!(
             ResultEnvelope::new(
                 request_id.clone(),
@@ -823,7 +929,7 @@ mod tests {
             }
 
             fn size_hint(&self) -> Option<usize> {
-                Some(ENVELOPE_MAX_SIZE + 1)
+                Some(RESULT_MAX_SIZE + 1)
             }
         }
 

crates/protos/src/lib.rs

@@ -97,10 +97,31 @@ fn internode_http2_keep_alive_timeout() -> Duration {
 }
 
 fn internode_rpc_timeout() -> Duration {
-    Duration::from_secs(rustfs_utils::get_env_u64(
+    normalize_internode_rpc_timeout(Duration::from_secs(rustfs_utils::get_env_u64(
         rustfs_config::ENV_INTERNODE_RPC_TIMEOUT_SECS,
         rustfs_config::DEFAULT_INTERNODE_RPC_TIMEOUT_SECS,
-    ))
+    )))
+}
+
+fn normalize_internode_rpc_timeout(timeout: Duration) -> Duration {
+    timeout.max(Duration::from_secs(1))
+}
+
+/// Budget for one heal-control execution, kept below the transport timeout so
+/// the coordinator stops waiting for admission before the caller gives up.
+pub fn heal_control_execution_timeout() -> Duration {
+    heal_control_execution_timeout_for(internode_rpc_timeout())
+}
+
+fn heal_control_execution_timeout_for(transport_timeout: Duration) -> Duration {
+    const TRANSPORT_GUARD: Duration = Duration::from_secs(1);
+    let transport_timeout = transport_timeout.max(Duration::from_secs(1));
+    transport_timeout
+        .saturating_sub(TRANSPORT_GUARD.min(transport_timeout / 2))
+        .max(Duration::from_millis(1))
+        .min(Duration::from_millis(
+            u64::try_from(heal_control::MAX_LIFETIME_MS).expect("positive heal control lifetime must fit u64"),
+        ))
 }
 
 fn internode_rpc_tcp_nodelay() -> bool {
@@ -141,9 +162,20 @@ pub fn internode_rpc_max_message_size() -> usize {
     rustfs_utils::get_env_usize(rustfs_config::ENV_INTERNODE_RPC_MAX_MESSAGE_SIZE, DEFAULT_GRPC_SERVER_MESSAGE_LEN)
 }
 
-pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = 65 * 1024;
-pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 1;
-pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v1\0";
+pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
+pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
+pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
+
+pub fn heal_control_coordinator_epoch(topology_fingerprint: &str) -> Result<u64, &'static str> {
+    let prefix = topology_fingerprint
+        .get(..16)
+        .ok_or("heal control topology fingerprint is too short")?;
+    let epoch = u64::from_str_radix(prefix, 16).map_err(|_| "heal control topology fingerprint is not hexadecimal")?;
+    if epoch == 0 {
+        return Err("heal control topology epoch is zero");
+    }
+    Ok(epoch)
+}
 
 pub fn heal_control_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
     let mut probe = Vec::with_capacity(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
@@ -165,7 +197,7 @@ pub fn canonical_heal_control_request_body(
     topology_fingerprint: &str,
     command: &[u8],
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-heal-control-v1\0";
+    const DOMAIN: &[u8] = b"rustfs-heal-control-v2\0";
 
     let fingerprint = topology_fingerprint.as_bytes();
     let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len());
@@ -185,7 +217,7 @@ pub fn canonical_heal_control_capability_ack(
     topology_fingerprint: &str,
     probe: &[u8],
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
-    const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v1\0";
+    const DOMAIN: &[u8] = b"rustfs-heal-control-capability-ack-v2\0";
 
     let fingerprint = topology_fingerprint.as_bytes();
     let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + probe.len());
@@ -198,17 +230,42 @@ pub fn canonical_heal_control_capability_ack(
     Ok(body)
 }
 
+pub fn canonical_heal_control_response_body(
+    version: u32,
+    topology_fingerprint: &str,
+    command: &[u8],
+    result: &[u8],
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    const DOMAIN: &[u8] = b"rustfs-heal-control-response-v2\0";
+
+    let fingerprint = topology_fingerprint.as_bytes();
+    let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len() + 8 + result.len());
+    body.extend_from_slice(DOMAIN);
+    body.extend_from_slice(&version.to_be_bytes());
+    body.extend_from_slice(&u64::try_from(fingerprint.len())?.to_be_bytes());
+    body.extend_from_slice(fingerprint);
+    body.extend_from_slice(&u64::try_from(command.len())?.to_be_bytes());
+    body.extend_from_slice(command);
+    body.extend_from_slice(&u64::try_from(result.len())?.to_be_bytes());
+    body.extend_from_slice(result);
+    Ok(body)
+}
+
 #[cfg(test)]
 mod heal_control_tests {
     use super::{
-        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, canonical_heal_control_capability_ack, canonical_heal_control_request_body,
-        heal_control_capability_probe, is_heal_control_capability_probe,
+        HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
+        canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
+        heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
+        internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
     };
+    use crate::heal_control;
+    use std::time::Duration;
 
     #[test]
     fn canonical_heal_control_body_binds_every_field_and_boundary() {
         let baseline = canonical_heal_control_request_body(1, "ab", b"c").expect("small request should encode");
-        let mut golden = b"rustfs-heal-control-v1\0".to_vec();
+        let mut golden = b"rustfs-heal-control-v2\0".to_vec();
         golden.extend_from_slice(&1_u32.to_be_bytes());
         golden.extend_from_slice(&2_u64.to_be_bytes());
         golden.extend_from_slice(b"ab");
@@ -236,9 +293,11 @@ mod heal_control_tests {
 
     #[test]
     fn canonical_capability_ack_binds_version_and_topology() {
+        assert_eq!(HEAL_CONTROL_PROTOCOL_VERSION, 2);
+        assert!(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.starts_with(b"rustfs-heal-control-capability-v2"));
         let probe = heal_control_capability_probe(&[7; 16]);
         let ack = canonical_heal_control_capability_ack(1, "ab", &probe).expect("small acknowledgement should encode");
-        let mut golden = b"rustfs-heal-control-capability-ack-v1\0".to_vec();
+        let mut golden = b"rustfs-heal-control-capability-ack-v2\0".to_vec();
         golden.extend_from_slice(&1_u32.to_be_bytes());
         golden.extend_from_slice(&2_u64.to_be_bytes());
         golden.extend_from_slice(b"ab");
@@ -254,6 +313,54 @@ mod heal_control_tests {
         assert!(is_heal_control_capability_probe(&probe));
         assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
     }
+
+    #[test]
+    fn canonical_response_binds_request_and_result() {
+        let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
+        assert_ne!(baseline, canonical_heal_control_response_body(1, "abcdef", b"query", b"result").unwrap());
+        assert_ne!(baseline, canonical_heal_control_response_body(2, "bbcdef", b"query", b"result").unwrap());
+        assert_ne!(baseline, canonical_heal_control_response_body(2, "abcdef", b"cancel", b"result").unwrap());
+        assert_ne!(
+            baseline,
+            canonical_heal_control_response_body(2, "abcdef", b"query", b"tampered").unwrap()
+        );
+    }
+
+    #[test]
+    fn coordinator_epoch_is_stable_and_rejects_invalid_fingerprints() {
+        assert_eq!(heal_control_coordinator_epoch("0123456789abcdefextra"), Ok(0x0123_4567_89ab_cdef));
+        assert_eq!(
+            heal_control_coordinator_epoch("0000000000000000"),
+            Err("heal control topology epoch is zero")
+        );
+        assert_eq!(
+            heal_control_coordinator_epoch("short"),
+            Err("heal control topology fingerprint is too short")
+        );
+        assert_eq!(
+            heal_control_coordinator_epoch("not-hex-value!!!!"),
+            Err("heal control topology fingerprint is not hexadecimal")
+        );
+    }
+
+    #[test]
+    fn execution_budget_precedes_transport_timeout() {
+        let execution = heal_control_execution_timeout();
+        assert!(!execution.is_zero());
+        assert!(execution < internode_rpc_timeout());
+        assert!(execution <= std::time::Duration::from_millis(heal_control::MAX_LIFETIME_MS as u64));
+    }
+
+    #[test]
+    fn execution_budget_is_nonzero_for_zero_transport_configuration() {
+        let normalized_transport = Duration::from_secs(1);
+        assert_eq!(normalize_internode_rpc_timeout(Duration::ZERO), normalized_transport);
+        for configured_transport in [Duration::ZERO, normalized_transport] {
+            let execution = heal_control_execution_timeout_for(configured_transport);
+            assert!(execution > Duration::ZERO);
+            assert!(execution < normalized_transport);
+        }
+    }
 }
 
 /// Whether internode metadata RPCs should send only the msgpack `_bin` payloads and leave the JSON

crates/protos/src/node.proto

@@ -783,6 +783,7 @@ message HealControlResponse {
   bool success = 1;
   bytes result = 2;
   optional string error_info = 3;
+  bytes response_proof = 4;
 }
 
 message GetMetacacheListingRequest {