feat(heal): add authenticated control RPC contract (#4993)

* fix(rpc): bind internode auth to exact targets

* fix(heal): initialize the runtime atomically

* fix(heal): aggregate status across cluster nodes

* fix(heal): return canonical tokens for duplicate starts

* feat(heal): add authenticated control RPC contract

* fix(heal): return canonical tokens for duplicate starts (#4992)

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

crates/ecstore/src/cluster/rpc/client.rs

@@ -18,7 +18,8 @@ use crate::disk::error::{DiskError, Error as DiskErrorType};
 use crate::runtime::sources as runtime_sources;
 use http::Uri;
 use rustfs_protos::{
-    ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::node_service_client::NodeServiceClient,
+    ChannelClass, create_new_channel, get_channel_for_class,
+    proto_gen::node_service::{heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient},
 };
 use std::{error::Error, io::ErrorKind};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
@@ -37,6 +38,21 @@ pub async fn node_service_time_out_client(
     node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
 }
 
+pub async fn heal_control_time_out_client(
+    addr: &str,
+    interceptor: TonicInterceptor,
+) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
+    let interceptor = interceptor.with_rpc_audience(addr)?;
+    let channel = match runtime_sources::cached_node_channel(addr).await {
+        Some(channel) => channel,
+        None => create_new_channel(addr).await?,
+    };
+    let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
+    Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
+        .max_decoding_message_size(max_message_size)
+        .max_encoding_message_size(max_message_size))
+}
+
 /// Build a `NodeServiceClient` bound to the [`ChannelClass`]-appropriate channel for `addr`.
 ///
 /// Bulk `bytes`-carrying RPCs (ReadAll/WriteAll/ReadMultiple/BatchReadVersion) pass

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -1104,12 +1104,13 @@ mod tests {
             .metadata()
             .get(RPC_CONTENT_SHA256_HEADER)
             .and_then(|value| value.to_str().ok());
-        let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "HealControl", content_sha256)
-            .expect("body-bound auth headers should build");
+        let headers =
+            gen_tonic_signature_headers("node-a:9000", "node_service.HealControlService", "HealControl", content_sha256)
+                .expect("body-bound auth headers should build");
         request.metadata_mut().as_mut().extend(headers.clone());
 
-        assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers).is_ok());
-        let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers)
+        assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.HealControlService/HealControl", &headers).is_ok());
+        let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.HealControlService/HealControl", &headers)
             .expect_err("reusing a body-bound nonce must fail");
         assert_eq!(replay.to_string(), "RPC request replay detected");
         assert!(verify_tonic_canonical_body_digest(&request, body).is_ok());

crates/ecstore/src/cluster/rpc/peer_rest_client.rs

@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::cluster::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, heal_control_time_out_client, node_service_time_out_client,
+};
+use crate::cluster::rpc::set_tonic_canonical_body_digest;
 use crate::error::{Error, Result};
 use crate::{
     disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
@@ -32,11 +35,11 @@ use rustfs_protos::proto_gen::node_service::{
     BackgroundHealStatusRequest, CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest,
     DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest,
     GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest,
-    GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest,
-    LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
-    LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
-    ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
-    StopRebalanceRequest, node_service_client::NodeServiceClient,
+    GetSysConfigRequest, GetSysErrorsRequest, HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest,
+    LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
+    LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
+    ReloadSiteReplicationConfigRequest, ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest,
+    StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, node_service_client::NodeServiceClient,
 };
 use rustfs_utils::XHost;
 use serde::{Deserialize, Serialize as _};
@@ -61,6 +64,8 @@ pub const PEER_RESTDRY_RUN: &str = "dry-run";
 pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
 pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
 const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
+const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
+const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
 const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
 const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
 const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
@@ -167,6 +172,29 @@ impl PeerRestClient {
             })
     }
 
+    async fn get_heal_control_client(
+        &self,
+    ) -> Result<
+        rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
+            InterceptedService<Channel, TonicInterceptor>,
+        >,
+    > {
+        if self.offline.load(Ordering::Acquire) {
+            self.mark_offline_and_spawn_recovery();
+            return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
+        }
+
+        heal_control_time_out_client(&self.grid_host, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
+            .await
+            .map_err(|err| {
+                let storage_err = Error::other(format!("can not get heal control client, err: {err}"));
+                if Self::is_network_like_error(&storage_err) {
+                    self.mark_offline_and_spawn_recovery();
+                }
+                storage_err
+            })
+    }
+
     /// Evict the connection to this peer from the global cache.
     /// This should be called when communication with this peer fails.
     pub async fn evict_connection(&self) {
@@ -697,6 +725,43 @@ impl PeerRestClient {
         .await
     }
 
+    pub async fn heal_control(&self, version: u32, topology_fingerprint: String, command: Vec<u8>) -> Result<Vec<u8>> {
+        if topology_fingerprint.len() > HEAL_CONTROL_FINGERPRINT_MAX_SIZE {
+            return Err(Error::other("heal control topology fingerprint exceeds size limit"));
+        }
+        if command.len() > HEAL_CONTROL_PAYLOAD_MAX_SIZE {
+            return Err(Error::other("heal control command exceeds size limit"));
+        }
+        self.finalize_result(
+            async {
+                let mut client = self
+                    .get_heal_control_client()
+                    .await?
+                    .max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
+                    .max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
+                let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
+                    .map_err(|_| Error::other("heal control request length cannot be represented"))?;
+                let mut request = Request::new(HealControlRequest {
+                    version,
+                    topology_fingerprint,
+                    command: command.into(),
+                });
+                set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
+                let response = client.heal_control(request).await?.into_inner();
+                if !response.success {
+                    return Err(Error::other(
+                        response
+                            .error_info
+                            .unwrap_or_else(|| "peer heal control failed without an error".to_string()),
+                    ));
+                }
+                Ok(response.result.to_vec())
+            }
+            .await,
+        )
+        .await
+    }
+
     pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
         self.finalize_result(
             async {
@@ -1297,6 +1362,18 @@ mod tests {
         assert!(err.to_string().contains("temporarily offline"));
     }
 
+    #[tokio::test]
+    async fn peer_rest_client_rejects_oversized_heal_control_before_dialing() {
+        let client = test_peer_client();
+        let err = client
+            .heal_control(1, "fingerprint".to_string(), vec![0; HEAL_CONTROL_PAYLOAD_MAX_SIZE + 1])
+            .await
+            .expect_err("oversized heal control payload must fail locally");
+
+        assert!(err.to_string().contains("exceeds size limit"));
+        assert!(!client.offline.load(Ordering::Acquire));
+    }
+
     #[tokio::test]
     async fn peer_rest_client_prepare_retry_clears_offline_gate() {
         // finalize_result sets the offline gate on a network error; without

crates/protos/src/generated/proto_gen/node_service.rs

@@ -586,6 +586,8 @@ pub struct DeleteVersionRequest {
     pub force_del_marker: bool,
     #[prost(string, tag = "6")]
     pub opts: ::prost::alloc::string::String,
+    /// msgpack payloads mirroring the JSON fields above (grpc-optimization P2). Senders dual-write
+    /// both; receivers prefer the \*\_bin form and fall back to the JSON string when it is empty.
     #[prost(bytes = "bytes", tag = "7")]
     pub file_info_bin: ::prost::bytes::Bytes,
     #[prost(bytes = "bytes", tag = "8")]
@@ -610,6 +612,8 @@ pub struct DeleteVersionsRequest {
     pub versions: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
     #[prost(string, tag = "4")]
     pub opts: ::prost::alloc::string::String,
+    /// msgpack payloads mirroring the JSON fields above (grpc-optimization P2). Senders dual-write
+    /// both; receivers prefer the \*\_bin form and fall back to the JSON string when it is empty.
     #[prost(bytes = "bytes", repeated, tag = "5")]
     pub versions_bin: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
     #[prost(bytes = "bytes", tag = "6")]
@@ -650,6 +654,9 @@ pub struct DeleteVolumeRequest {
     pub disk: ::prost::alloc::string::String,
     #[prost(string, tag = "2")]
     pub volume: ::prost::alloc::string::String,
+    /// When false (default), a non-empty volume is refused (VolumeNotEmpty); when
+    /// true, the volume is deleted recursively. Old peers omit this field, which
+    /// decodes to false — the safe, non-recursive behavior (backlog#799 B1).
     #[prost(bool, tag = "3")]
     pub force: bool,
 }
@@ -1091,6 +1098,24 @@ pub struct BackgroundHealStatusResponse {
     pub error_info: ::core::option::Option<::prost::alloc::string::String>,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct HealControlRequest {
+    #[prost(uint32, tag = "1")]
+    pub version: u32,
+    #[prost(string, tag = "2")]
+    pub topology_fingerprint: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "3")]
+    pub command: ::prost::bytes::Bytes,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct HealControlResponse {
+    #[prost(bool, tag = "1")]
+    pub success: bool,
+    #[prost(bytes = "bytes", tag = "2")]
+    pub result: ::prost::bytes::Bytes,
+    #[prost(string, optional, tag = "3")]
+    pub error_info: ::core::option::Option<::prost::alloc::string::String>,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct GetMetacacheListingRequest {
     #[prost(bytes = "bytes", tag = "1")]
     pub opts: ::prost::bytes::Bytes,
@@ -5329,3 +5354,237 @@ pub mod node_service_server {
         const NAME: &'static str = SERVICE_NAME;
     }
 }
+/// Generated client implementations.
+pub mod heal_control_service_client {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
+    use tonic::codegen::http::Uri;
+    use tonic::codegen::*;
+    #[derive(Debug, Clone)]
+    pub struct HealControlServiceClient<T> {
+        inner: tonic::client::Grpc<T>,
+    }
+    impl HealControlServiceClient<tonic::transport::Channel> {
+        /// Attempt to create a new client by connecting to a given endpoint.
+        pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
+        where
+            D: TryInto<tonic::transport::Endpoint>,
+            D::Error: Into<StdError>,
+        {
+            let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
+            Ok(Self::new(conn))
+        }
+    }
+    impl<T> HealControlServiceClient<T>
+    where
+        T: tonic::client::GrpcService<tonic::body::Body>,
+        T::Error: Into<StdError>,
+        T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
+        <T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
+    {
+        pub fn new(inner: T) -> Self {
+            let inner = tonic::client::Grpc::new(inner);
+            Self { inner }
+        }
+        pub fn with_origin(inner: T, origin: Uri) -> Self {
+            let inner = tonic::client::Grpc::with_origin(inner, origin);
+            Self { inner }
+        }
+        pub fn with_interceptor<F>(inner: T, interceptor: F) -> HealControlServiceClient<InterceptedService<T, F>>
+        where
+            F: tonic::service::Interceptor,
+            T::ResponseBody: Default,
+            T: tonic::codegen::Service<
+                    http::Request<tonic::body::Body>,
+                    Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
+                >,
+            <T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
+                Into<StdError> + std::marker::Send + std::marker::Sync,
+        {
+            HealControlServiceClient::new(InterceptedService::new(inner, interceptor))
+        }
+        /// Compress requests with the given encoding.
+        ///
+        /// This requires the server to support it otherwise it might respond with an
+        /// error.
+        #[must_use]
+        pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.send_compressed(encoding);
+            self
+        }
+        /// Enable decompressing responses.
+        #[must_use]
+        pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.inner = self.inner.accept_compressed(encoding);
+            self
+        }
+        /// Limits the maximum size of a decoded message.
+        ///
+        /// Default: `4MB`
+        #[must_use]
+        pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_decoding_message_size(limit);
+            self
+        }
+        /// Limits the maximum size of an encoded message.
+        ///
+        /// Default: `usize::MAX`
+        #[must_use]
+        pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+            self.inner = self.inner.max_encoding_message_size(limit);
+            self
+        }
+        pub async fn heal_control(
+            &mut self,
+            request: impl tonic::IntoRequest<super::HealControlRequest>,
+        ) -> std::result::Result<tonic::Response<super::HealControlResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.HealControlService/HealControl");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.HealControlService", "HealControl"));
+            self.inner.unary(req, path, codec).await
+        }
+    }
+}
+/// Generated server implementations.
+pub mod heal_control_service_server {
+    #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
+    use tonic::codegen::*;
+    /// Generated trait containing gRPC methods that should be implemented for use with HealControlServiceServer.
+    #[async_trait]
+    pub trait HealControlService: std::marker::Send + std::marker::Sync + 'static {
+        async fn heal_control(
+            &self,
+            request: tonic::Request<super::HealControlRequest>,
+        ) -> std::result::Result<tonic::Response<super::HealControlResponse>, tonic::Status>;
+    }
+    #[derive(Debug)]
+    pub struct HealControlServiceServer<T> {
+        inner: Arc<T>,
+        accept_compression_encodings: EnabledCompressionEncodings,
+        send_compression_encodings: EnabledCompressionEncodings,
+        max_decoding_message_size: Option<usize>,
+        max_encoding_message_size: Option<usize>,
+    }
+    impl<T> HealControlServiceServer<T> {
+        pub fn new(inner: T) -> Self {
+            Self::from_arc(Arc::new(inner))
+        }
+        pub fn from_arc(inner: Arc<T>) -> Self {
+            Self {
+                inner,
+                accept_compression_encodings: Default::default(),
+                send_compression_encodings: Default::default(),
+                max_decoding_message_size: None,
+                max_encoding_message_size: None,
+            }
+        }
+        pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
+        where
+            F: tonic::service::Interceptor,
+        {
+            InterceptedService::new(Self::new(inner), interceptor)
+        }
+        /// Enable decompressing requests with the given encoding.
+        #[must_use]
+        pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.accept_compression_encodings.enable(encoding);
+            self
+        }
+        /// Compress responses with the given encoding, if the client supports it.
+        #[must_use]
+        pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
+            self.send_compression_encodings.enable(encoding);
+            self
+        }
+        /// Limits the maximum size of a decoded message.
+        ///
+        /// Default: `4MB`
+        #[must_use]
+        pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
+            self.max_decoding_message_size = Some(limit);
+            self
+        }
+        /// Limits the maximum size of an encoded message.
+        ///
+        /// Default: `usize::MAX`
+        #[must_use]
+        pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
+            self.max_encoding_message_size = Some(limit);
+            self
+        }
+    }
+    impl<T, B> tonic::codegen::Service<http::Request<B>> for HealControlServiceServer<T>
+    where
+        T: HealControlService,
+        B: Body + std::marker::Send + 'static,
+        B::Error: Into<StdError> + std::marker::Send + 'static,
+    {
+        type Response = http::Response<tonic::body::Body>;
+        type Error = std::convert::Infallible;
+        type Future = BoxFuture<Self::Response, Self::Error>;
+        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
+            Poll::Ready(Ok(()))
+        }
+        fn call(&mut self, req: http::Request<B>) -> Self::Future {
+            match req.uri().path() {
+                "/node_service.HealControlService/HealControl" => {
+                    #[allow(non_camel_case_types)]
+                    struct HealControlSvc<T: HealControlService>(pub Arc<T>);
+                    impl<T: HealControlService> tonic::server::UnaryService<super::HealControlRequest> for HealControlSvc<T> {
+                        type Response = super::HealControlResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::HealControlRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as HealControlService>::heal_control(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = HealControlSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
+                _ => Box::pin(async move {
+                    let mut response = http::Response::new(tonic::body::Body::default());
+                    let headers = response.headers_mut();
+                    headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
+                    headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
+                    Ok(response)
+                }),
+            }
+        }
+    }
+    impl<T> Clone for HealControlServiceServer<T> {
+        fn clone(&self) -> Self {
+            let inner = self.inner.clone();
+            Self {
+                inner,
+                accept_compression_encodings: self.accept_compression_encodings,
+                send_compression_encodings: self.send_compression_encodings,
+                max_decoding_message_size: self.max_decoding_message_size,
+                max_encoding_message_size: self.max_encoding_message_size,
+            }
+        }
+    }
+    /// Generated gRPC service name
+    pub const SERVICE_NAME: &str = "node_service.HealControlService";
+    impl<T> tonic::server::NamedService for HealControlServiceServer<T> {
+        const NAME: &'static str = SERVICE_NAME;
+    }
+}

crates/protos/src/lib.rs

@@ -140,6 +140,64 @@ pub fn internode_rpc_max_message_size() -> usize {
     rustfs_utils::get_env_usize(rustfs_config::ENV_INTERNODE_RPC_MAX_MESSAGE_SIZE, DEFAULT_GRPC_SERVER_MESSAGE_LEN)
 }
 
+pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = 65 * 1024;
+
+/// Builds the stable byte representation authenticated for a heal-control request.
+///
+/// This deliberately does not reuse protobuf encoding: mixed-version peers may
+/// retain unknown protobuf fields, which must not change the signed contract.
+pub fn canonical_heal_control_request_body(
+    version: u32,
+    topology_fingerprint: &str,
+    command: &[u8],
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    const DOMAIN: &[u8] = b"rustfs-heal-control-v1\0";
+
+    let fingerprint = topology_fingerprint.as_bytes();
+    let mut body = Vec::with_capacity(DOMAIN.len() + 4 + 8 + fingerprint.len() + 8 + command.len());
+    body.extend_from_slice(DOMAIN);
+    body.extend_from_slice(&version.to_be_bytes());
+    body.extend_from_slice(&u64::try_from(fingerprint.len())?.to_be_bytes());
+    body.extend_from_slice(fingerprint);
+    body.extend_from_slice(&u64::try_from(command.len())?.to_be_bytes());
+    body.extend_from_slice(command);
+    Ok(body)
+}
+
+#[cfg(test)]
+mod heal_control_tests {
+    use super::canonical_heal_control_request_body;
+
+    #[test]
+    fn canonical_heal_control_body_binds_every_field_and_boundary() {
+        let baseline = canonical_heal_control_request_body(1, "ab", b"c").expect("small request should encode");
+        let mut golden = b"rustfs-heal-control-v1\0".to_vec();
+        golden.extend_from_slice(&1_u32.to_be_bytes());
+        golden.extend_from_slice(&2_u64.to_be_bytes());
+        golden.extend_from_slice(b"ab");
+        golden.extend_from_slice(&1_u64.to_be_bytes());
+        golden.extend_from_slice(b"c");
+        assert_eq!(baseline, golden);
+
+        assert_ne!(
+            baseline,
+            canonical_heal_control_request_body(2, "ab", b"c").expect("small request should encode")
+        );
+        assert_ne!(
+            baseline,
+            canonical_heal_control_request_body(1, "ac", b"c").expect("small request should encode")
+        );
+        assert_ne!(
+            baseline,
+            canonical_heal_control_request_body(1, "ab", b"d").expect("small request should encode")
+        );
+        assert_ne!(
+            canonical_heal_control_request_body(1, "ab", b"c").expect("small request should encode"),
+            canonical_heal_control_request_body(1, "a", b"bc").expect("small request should encode")
+        );
+    }
+}
+
 /// Whether internode metadata RPCs should send only the msgpack `_bin` payloads and leave the JSON
 /// compatibility strings empty (grpc-optimization P2-1). Shared by the client (`remote_disk`) and
 /// server (`node_service`) send paths. Defaults to `false` (dual-write); see

crates/protos/src/node.proto

@@ -773,6 +773,18 @@ message BackgroundHealStatusResponse {
   optional string error_info = 3;
 }
 
+message HealControlRequest {
+  uint32 version = 1;
+  string topology_fingerprint = 2;
+  bytes command = 3;
+}
+
+message HealControlResponse {
+  bool success = 1;
+  bytes result = 2;
+  optional string error_info = 3;
+}
+
 message GetMetacacheListingRequest {
   bytes opts = 1;
 }
@@ -966,3 +978,7 @@ service NodeService {
   rpc LoadTransitionTierConfig(LoadTransitionTierConfigRequest) returns (LoadTransitionTierConfigResponse) {};
   rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {};
 }
+
+service HealControlService {
+  rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
+}