gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 03:46:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(heal): add authenticated control RPC contract (#4993)

Browse Source 
* fix(rpc): bind internode auth to exact targets

* fix(heal): initialize the runtime atomically

* fix(heal): aggregate status across cluster nodes

* fix(heal): return canonical tokens for duplicate starts

* feat(heal): add authenticated control RPC contract

* fix(heal): return canonical tokens for duplicate starts (#4992)

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
cxymds
2026-07-20 12:07:38 +08:00
committed by GitHub
parent 69f543568b
commit a774bc07da
12 changed files with 705 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/cluster/rpc/client.rs
+17 -1
View File
@@ -18,7 +18,8 @@ use crate::disk::error::{DiskError, Error as DiskErrorType};
use crate::runtime::sources as runtime_sources;
use http::Uri;
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class, proto_gen::node_service::node_service_client::NodeServiceClient,
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient},
};
use std::{error::Error, io::ErrorKind};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
@@ -37,6 +38,21 @@ pub async fn node_service_time_out_client(
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
}
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
}
/// Build a `NodeServiceClient` bound to the [`ChannelClass`]-appropriate channel for `addr`.
///
/// Bulk `bytes`-carrying RPCs (ReadAll/WriteAll/ReadMultiple/BatchReadVersion) pass
crates/ecstore/src/cluster/rpc/http_auth.rs
+5 -4
View File
@@ -1104,12 +1104,13 @@ mod tests {
.metadata()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok());
let headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "HealControl", content_sha256)
.expect("body-bound auth headers should build");
let headers =
gen_tonic_signature_headers("node-a:9000", "node_service.HealControlService", "HealControl", content_sha256)
.expect("body-bound auth headers should build");
request.metadata_mut().as_mut().extend(headers.clone());
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers).is_ok());
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.NodeService/HealControl", &headers)
assert!(verify_tonic_rpc_signature("node-a:9000", "/node_service.HealControlService/HealControl", &headers).is_ok());
let replay = verify_tonic_rpc_signature("node-a:9000", "/node_service.HealControlService/HealControl", &headers)
.expect_err("reusing a body-bound nonce must fail");
assert_eq!(replay.to_string(), "RPC request replay detected");
assert!(verify_tonic_canonical_body_digest(&request, body).is_ok());
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+83 -6
View File
@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::cluster::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, heal_control_time_out_client, node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_canonical_body_digest;
use crate::error::{Error, Result};
use crate::{
disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout},
@@ -32,11 +35,11 @@ use rustfs_protos::proto_gen::node_service::{
BackgroundHealStatusRequest, CancelDecommissionRequest, ClearDecommissionRequest, DeleteBucketMetadataRequest,
DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetLiveEventsRequest, GetMemInfoRequest,
GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest,
GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest,
LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
StopRebalanceRequest, node_service_client::NodeServiceClient,
GetSysConfigRequest, GetSysErrorsRequest, HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest,
LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest,
ReloadSiteReplicationConfigRequest, ScannerActivityRequest, ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest,
StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, node_service_client::NodeServiceClient,
};
use rustfs_utils::XHost;
use serde::{Deserialize, Serialize as _};
@@ -61,6 +64,8 @@ pub const PEER_RESTDRY_RUN: &str = "dry-run";
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
@@ -167,6 +172,29 @@ impl PeerRestClient {
})
}
async fn get_heal_control_client(
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
InterceptedService<Channel, TonicInterceptor>,
>,
> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
}
heal_control_time_out_client(&self.grid_host, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
.await
.map_err(|err| {
let storage_err = Error::other(format!("can not get heal control client, err: {err}"));
if Self::is_network_like_error(&storage_err) {
self.mark_offline_and_spawn_recovery();
}
storage_err
})
}
/// Evict the connection to this peer from the global cache.
/// This should be called when communication with this peer fails.
pub async fn evict_connection(&self) {
@@ -697,6 +725,43 @@ impl PeerRestClient {
.await
}
pub async fn heal_control(&self, version: u32, topology_fingerprint: String, command: Vec<u8>) -> Result<Vec<u8>> {
if topology_fingerprint.len() > HEAL_CONTROL_FINGERPRINT_MAX_SIZE {
return Err(Error::other("heal control topology fingerprint exceeds size limit"));
}
if command.len() > HEAL_CONTROL_PAYLOAD_MAX_SIZE {
return Err(Error::other("heal control command exceeds size limit"));
}
self.finalize_result(
async {
let mut client = self
.get_heal_control_client()
.await?
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
let mut request = Request::new(HealControlRequest {
version,
topology_fingerprint,
command: command.into(),
});
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
let response = client.heal_control(request).await?.into_inner();
if !response.success {
return Err(Error::other(
response
.error_info
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
));
}
Ok(response.result.to_vec())
}
.await,
)
.await
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result(
async {
@@ -1297,6 +1362,18 @@ mod tests {
assert!(err.to_string().contains("temporarily offline"));
}
#[tokio::test]
async fn peer_rest_client_rejects_oversized_heal_control_before_dialing() {
let client = test_peer_client();
let err = client
.heal_control(1, "fingerprint".to_string(), vec![0; HEAL_CONTROL_PAYLOAD_MAX_SIZE + 1])
.await
.expect_err("oversized heal control payload must fail locally");
assert!(err.to_string().contains("exceeds size limit"));
assert!(!client.offline.load(Ordering::Acquire));
}
#[tokio::test]
async fn peer_rest_client_prepare_retry_clears_offline_gate() {
// finalize_result sets the offline gate on a network error; without