// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use crate::error::{Error, Result}; use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::{ endpoints::EndpointServerPools, global::is_dist_erasure, metrics_realtime::{CollectMetricsOpts, MetricType}, }; use rmp_serde::{Deserializer, Serializer}; use rustfs_madmin::{ ServerProperties, health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService}, metrics::RealtimeMetrics, net::NetInfo, }; use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::{ DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, DeleteUserRequest, GetCpusRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StopRebalanceRequest, node_service_client::NodeServiceClient, }; use rustfs_utils::XHost; use serde::{Deserialize, Serialize as _}; use std::{collections::HashMap, io::Cursor, time::SystemTime}; use tonic::Request; use tonic::service::interceptor::InterceptedService; use tonic::transport::Channel; use tracing::warn; pub const PEER_RESTSIGNAL: &str = "signal"; pub const PEER_RESTSUB_SYS: &str = "sub-sys"; pub const PEER_RESTDRY_RUN: &str = "dry-run"; #[derive(Clone, Debug)] pub struct PeerRestClient { pub host: XHost, pub grid_host: String, } impl PeerRestClient { pub fn new(host: XHost, grid_host: String) -> Self { Self { host, grid_host } } pub async fn new_clients(eps: EndpointServerPools) -> (Vec>, Vec>) { if !is_dist_erasure().await { return (Vec::new(), Vec::new()); } let eps = eps.clone(); let hosts = eps.hosts_sorted(); let mut remote = Vec::with_capacity(hosts.len()); let mut all = vec![None; hosts.len()]; for (i, hs_host) in hosts.iter().enumerate() { if let Some(host) = hs_host && let Some(grid_host) = eps.find_grid_hosts_from_peer(host) { let client = PeerRestClient::new(host.clone(), grid_host); all[i] = Some(client.clone()); remote.push(Some(client)); } } if all.len() != remote.len() + 1 { warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len()); } (remote, all) } pub async fn get_client(&self) -> Result>> { node_service_time_out_client(&self.grid_host, TonicInterceptor::Signature(gen_tonic_signature_interceptor())) .await .map_err(|err| Error::other(format!("can not get client, err: {err}"))) } /// Evict the connection to this peer from the global cache. /// This should be called when communication with this peer fails. pub async fn evict_connection(&self) { evict_failed_connection(&self.grid_host).await; } } impl PeerRestClient { pub async fn local_storage_info(&self) -> Result { let result = self.local_storage_info_inner().await; if result.is_err() { // Evict stale connection on any error for cluster recovery self.evict_connection().await; } result } async fn local_storage_info_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(LocalStorageInfoRequest { metrics: true }); let response = client.local_storage_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.storage_info; let mut buf = Deserializer::new(Cursor::new(data)); let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf)?; Ok(storage_info) } pub async fn server_info(&self) -> Result { let result = self.server_info_inner().await; if result.is_err() { // Evict stale connection on any error for cluster recovery self.evict_connection().await; } result } async fn server_info_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(ServerInfoRequest { metrics: true }); let response = client.server_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.server_properties; let mut buf = Deserializer::new(Cursor::new(data)); let storage_properties: ServerProperties = Deserialize::deserialize(&mut buf)?; Ok(storage_properties) } pub async fn get_cpus(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetCpusRequest {}); let response = client.get_cpus(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.cpus; let mut buf = Deserializer::new(Cursor::new(data)); let cpus: Cpus = Deserialize::deserialize(&mut buf)?; Ok(cpus) } pub async fn get_net_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetNetInfoRequest {}); let response = client.get_net_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.net_info; let mut buf = Deserializer::new(Cursor::new(data)); let net_info: NetInfo = Deserialize::deserialize(&mut buf)?; Ok(net_info) } pub async fn get_partitions(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetPartitionsRequest {}); let response = client.get_partitions(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.partitions; let mut buf = Deserializer::new(Cursor::new(data)); let partitions: Partitions = Deserialize::deserialize(&mut buf)?; Ok(partitions) } pub async fn get_os_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetOsInfoRequest {}); let response = client.get_os_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.os_info; let mut buf = Deserializer::new(Cursor::new(data)); let os_info: OsInfo = Deserialize::deserialize(&mut buf)?; Ok(os_info) } pub async fn get_se_linux_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetSeLinuxInfoRequest {}); let response = client.get_se_linux_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.sys_services; let mut buf = Deserializer::new(Cursor::new(data)); let sys_services: SysService = Deserialize::deserialize(&mut buf)?; Ok(sys_services) } pub async fn get_sys_config(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetSysConfigRequest {}); let response = client.get_sys_config(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.sys_config; let mut buf = Deserializer::new(Cursor::new(data)); let sys_config: SysConfig = Deserialize::deserialize(&mut buf)?; Ok(sys_config) } pub async fn get_sys_errors(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetSysErrorsRequest {}); let response = client.get_sys_errors(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.sys_errors; let mut buf = Deserializer::new(Cursor::new(data)); let sys_errors: SysErrors = Deserialize::deserialize(&mut buf)?; Ok(sys_errors) } pub async fn get_mem_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetMemInfoRequest {}); let response = client.get_mem_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.mem_info; let mut buf = Deserializer::new(Cursor::new(data)); let mem_info: MemInfo = Deserialize::deserialize(&mut buf)?; Ok(mem_info) } pub async fn get_metrics(&self, t: MetricType, opts: &CollectMetricsOpts) -> Result { let mut client = self.get_client().await?; let mut buf_t = Vec::new(); t.serialize(&mut Serializer::new(&mut buf_t))?; let mut buf_o = Vec::new(); opts.serialize(&mut Serializer::new(&mut buf_o))?; let request = Request::new(GetMetricsRequest { metric_type: buf_t.into(), opts: buf_o.into(), }); let response = client.get_metrics(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.realtime_metrics; let mut buf = Deserializer::new(Cursor::new(data)); let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf)?; Ok(realtime_metrics) } pub async fn get_proc_info(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetProcInfoRequest {}); let response = client.get_proc_info(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } let data = response.proc_info; let mut buf = Deserializer::new(Cursor::new(data)); let proc_info: ProcInfo = Deserialize::deserialize(&mut buf)?; Ok(proc_info) } pub async fn start_profiling(&self, profiler: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(StartProfilingRequest { profiler: profiler.to_string(), }); let response = client.start_profiling(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn download_profile_data(&self) -> Result<()> { todo!() } pub async fn get_bucket_stats(&self) -> Result<()> { todo!() } pub async fn get_sr_metrics(&self) -> Result<()> { todo!() } pub async fn get_all_bucket_stats(&self) -> Result<()> { todo!() } pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadBucketMetadataRequest { bucket: bucket.to_string(), }); let response = client.load_bucket_metadata(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(DeleteBucketMetadataRequest { bucket: bucket.to_string(), }); let response = client.delete_bucket_metadata(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn delete_policy(&self, policy: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(DeletePolicyRequest { policy_name: policy.to_string(), }); let response = client.delete_policy(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_policy(&self, policy: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadPolicyRequest { policy_name: policy.to_string(), }); let response = client.load_policy(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_policy_mapping(&self, user_or_group: &str, user_type: u64, is_group: bool) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadPolicyMappingRequest { user_or_group: user_or_group.to_string(), user_type, is_group, }); let response = client.load_policy_mapping(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn delete_user(&self, access_key: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(DeleteUserRequest { access_key: access_key.to_string(), }); let result = client.delete_user(request).await; if result.is_err() { self.evict_connection().await; } let response = result?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn delete_service_account(&self, access_key: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(DeleteServiceAccountRequest { access_key: access_key.to_string(), }); let result = client.delete_service_account(request).await; if result.is_err() { self.evict_connection().await; } let response = result?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_user(&self, access_key: &str, temp: bool) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadUserRequest { access_key: access_key.to_string(), temp, }); let result = client.load_user(request).await; if result.is_err() { self.evict_connection().await; } let response = result?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_service_account(&self, access_key: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadServiceAccountRequest { access_key: access_key.to_string(), }); let result = client.load_service_account(request).await; if result.is_err() { self.evict_connection().await; } let response = result?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_group(&self, group: &str) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadGroupRequest { group: group.to_string(), }); let result = client.load_group(request).await; if result.is_err() { self.evict_connection().await; } let response = result?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn reload_site_replication_config(&self) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(ReloadSiteReplicationConfigRequest {}); let response = client.reload_site_replication_config(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> { let mut client = self.get_client().await?; let mut vars = HashMap::new(); vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string()); vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string()); vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()); let request = Request::new(SignalServiceRequest { vars: Some(Mss { value: vars }), }); let response = client.signal_service(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn get_metacache_listing(&self) -> Result<()> { let _client = self.get_client().await?; todo!() } pub async fn update_metacache_listing(&self) -> Result<()> { let _client = self.get_client().await?; todo!() } pub async fn reload_pool_meta(&self) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(ReloadPoolMetaRequest {}); let response = client.reload_pool_meta(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn stop_rebalance(&self) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(StopRebalanceRequest {}); let response = client.stop_rebalance(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_rebalance_meta(&self, start_rebalance: bool) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadRebalanceMetaRequest { start_rebalance }); let response = client.load_rebalance_meta(request).await?.into_inner(); warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } pub async fn load_transition_tier_config(&self) -> Result<()> { let mut client = self.get_client().await?; let request = Request::new(LoadTransitionTierConfigRequest {}); let response = client.load_transition_tier_config(request).await?.into_inner(); if !response.success { if let Some(msg) = response.error_info { return Err(Error::other(msg)); } return Err(Error::other("")); } Ok(()) } }