init NotificationSys

ecstore/src/endpoints.rs

@@ -2,6 +2,7 @@ use crate::{
     disk::endpoint::{Endpoint, EndpointType},
     disks_layout::DisksLayout,
     error::{Error, Result},
+    global::global_rustfs_port,
     utils::net,
 };
 use std::{
@@ -529,6 +530,46 @@ impl EndpointServerPools {
 
         nodes
     }
+    pub fn hosts_sorted(&self) -> Vec<()> {
+        let (mut peers, local) = self.peers();
+
+        peers.sort();
+
+        for peer in peers.iter() {
+            if &local == peer {
+                continue;
+            }
+
+            // FIXME:TODO
+        }
+
+        unimplemented!()
+    }
+    pub fn peers(&self) -> (Vec<String>, String) {
+        let mut local = None;
+        let mut set = HashSet::new();
+        for ep in self.0.iter() {
+            for endpoint in ep.endpoints.0.iter() {
+                if endpoint.get_type() != EndpointType::Url {
+                    continue;
+                }
+                let host = endpoint.host_port();
+                if endpoint.is_local {
+                    if endpoint.url.port() == Some(global_rustfs_port()) {
+                        if local.is_none() {
+                            local = Some(host.clone());
+                        }
+                    }
+                }
+
+                set.insert(host);
+            }
+        }
+
+        let hosts: Vec<String> = set.iter().cloned().collect();
+
+        (hosts, local.unwrap_or_default())
+    }
 }
 
 #[cfg(test)]

ecstore/src/global.rs

@@ -18,7 +18,10 @@ pub const DISK_MIN_INODES: u64 = 1000;
 pub const DISK_FILL_FRACTION: f64 = 0.99;
 pub const DISK_RESERVE_FRACTION: f64 = 0.15;
 
+pub const DEFAULT_PORT: u16 = 9000;
+
 lazy_static! {
+    static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
     pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
     pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
     pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
@@ -34,6 +37,18 @@ lazy_static! {
     static ref globalDeploymentIDPtr: RwLock<Uuid> = RwLock::new(Uuid::nil());
 }
 
+pub fn global_rustfs_port() -> u16 {
+    if let Some(p) = GLOBAL_RUSTFS_PORT.get() {
+        p.clone()
+    } else {
+        DEFAULT_PORT
+    }
+}
+
+pub fn set_global_rustfs_port(value: u16) {
+    GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
+}
+
 pub async fn set_global_deployment_id(id: Uuid) {
     let mut id_ptr = globalDeploymentIDPtr.write().await;
     *id_ptr = id

ecstore/src/heal/heal_commands.rs

@@ -1,4 +1,8 @@
-use std::{path::Path, time::SystemTime};
+use std::{
+    collections::{HashMap, HashSet},
+    path::Path,
+    time::SystemTime,
+};
 
 use chrono::{DateTime, Utc};
 use lazy_static::lazy_static;
@@ -7,15 +11,18 @@ use time::OffsetDateTime;
 use tokio::sync::RwLock;
 
 use crate::{
+    config::storageclass::{RRS, STANDARD},
     disk::{DeleteOptions, DiskAPI, DiskStore, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
     error::{Error, Result},
     global::GLOBAL_BackgroundHealState,
     heal::heal_ops::HEALING_TRACKER_FILENAME,
     new_object_layer_fn,
-    store_api::{BucketInfo, StorageAPI},
+    store_api::{BucketInfo, StorageAPI, StorageDisk},
     utils::fs::read_file,
 };
 
+use super::{background_heal_ops::get_local_disks_to_heal, heal_ops::BG_HEALING_UUID};
+
 pub type HealScanMode = usize;
 pub type HealItemType = String;
 
@@ -494,3 +501,99 @@ pub async fn healing(derive_path: &str) -> Result<Option<HealingTracker>> {
 
     Ok(Some(healing_tracker))
 }
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct MRFStatus {
+    bytes_healed: u64,
+    items_healed: u64,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct SetStatus {
+    pub id: String,
+    pub pool_index: i32,
+    pub set_index: i32,
+    pub heal_status: String,
+    pub heal_priority: String,
+    pub total_objects: usize,
+    pub disks: Vec<StorageDisk>,
+}
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct BgHealState {
+    offline_endpoints: Vec<String>,
+    scanned_items_count: u64,
+    heal_disks: Vec<String>,
+    sets: Vec<SetStatus>,
+    mrf: HashMap<String, MRFStatus>,
+    scparity: HashMap<String, usize>,
+}
+
+pub async fn get_local_background_heal_status() -> (BgHealState, bool) {
+    let (bg_seq, ok) = GLOBAL_BackgroundHealState.get_heal_sequence_by_token(BG_HEALING_UUID).await;
+    if !ok {
+        return (BgHealState::default(), false);
+    }
+    let bg_seq = bg_seq.unwrap();
+    let mut status = BgHealState {
+        scanned_items_count: bg_seq.get_scanned_items_count().await as u64,
+        ..Default::default()
+    };
+    let mut heal_disks_map = HashSet::new();
+    for ep in get_local_disks_to_heal().await.iter() {
+        heal_disks_map.insert(ep.to_string());
+    }
+
+    let Some(store) = new_object_layer_fn() else {
+        let healing = GLOBAL_BackgroundHealState.get_local_healing_disks().await;
+        for disk in healing.values() {
+            status.heal_disks.push(disk.endpoint.clone());
+        }
+        return (status, true);
+    };
+
+    let si = store.local_storage_info().await;
+    let mut indexed = HashMap::new();
+    for disk in si.disks.iter() {
+        let set_idx = format!("{}-{}", disk.pool_index, disk.set_index);
+        // indexed.insert(set_idx, disk);
+        indexed.entry(set_idx).or_insert(Vec::new()).push(disk);
+    }
+
+    for (id, disks) in indexed {
+        let mut ss = SetStatus {
+            id,
+            set_index: disks[0].set_index,
+            pool_index: disks[0].pool_index,
+            ..Default::default()
+        };
+        for disk in disks {
+            ss.disks.push(disk.clone());
+            if disk.healing {
+                ss.heal_status = "healing".to_string();
+                ss.heal_priority = "high".to_string();
+                status.heal_disks.push(disk.endpoint.clone());
+            }
+        }
+        ss.disks.sort_by(|a, b| {
+            if a.pool_index != b.pool_index {
+                return a.pool_index.cmp(&b.pool_index);
+            }
+            if a.set_index != b.set_index {
+                return a.set_index.cmp(&b.set_index);
+            }
+            a.disk_index.cmp(&b.disk_index)
+        });
+        status.sets.push(ss);
+    }
+    status.sets.sort_by(|a, b| a.id.cmp(&b.id));
+    let backend_info = store.backend_info().await;
+    status
+        .scparity
+        .insert(STANDARD.to_string(), backend_info.standard_sc_parity.unwrap_or_default());
+    status
+        .scparity
+        .insert(RRS.to_string(), backend_info.rr_sc_parity.unwrap_or_default());
+
+    (status, true)
+}

ecstore/src/lib.rs

@@ -11,7 +11,9 @@ pub mod error;
 mod file_meta;
 pub mod global;
 pub mod heal;
+pub mod notification_sys;
 pub mod peer;
+mod peer_rest_client;
 mod quorum;
 pub mod set_disk;
 mod sets;
@@ -22,7 +24,7 @@ pub mod utils;
 
 pub mod bucket;
 pub mod file_meta_inline;
-pub mod options;
+
 pub mod pools;
 pub mod store_err;
 pub mod xhttp;

ecstore/src/notification_sys.rs

@@ -0,0 +1,53 @@
+use std::sync::OnceLock;
+
+use crate::endpoints::EndpointServerPools;
+use crate::error::{Error, Result};
+use crate::peer_rest_client::PeerRestClient;
+use lazy_static::lazy_static;
+
+lazy_static! {
+    pub static ref GLOBAL_NotificationSys: OnceLock<NotificationSys> = OnceLock::new();
+}
+
+pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
+    let _ = GLOBAL_NotificationSys
+        .set(NotificationSys::new(eps).await)
+        .map_err(|_| Error::msg("init notification_sys fail"));
+    Ok(())
+}
+
+pub struct NotificationSys {
+    pub peer_clients: Vec<PeerRestClient>,
+    pub all_peer_clients: Vec<PeerRestClient>,
+}
+
+impl NotificationSys {
+    pub async fn new(eps: EndpointServerPools) -> Self {
+        let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await;
+        Self {
+            peer_clients,
+            all_peer_clients,
+        }
+    }
+}
+
+pub struct NotificationPeerErr {
+    pub host: String,
+    pub err: Option<Error>,
+}
+
+impl NotificationSys {
+    pub async fn delete_policy(&self) -> Vec<NotificationPeerErr> {
+        unimplemented!()
+    }
+    pub async fn load_policy(&self) -> Vec<NotificationPeerErr> {
+        unimplemented!()
+    }
+
+    pub async fn load_policy_mapping(&self) -> Vec<NotificationPeerErr> {
+        unimplemented!()
+    }
+    pub async fn delete_user(&self) -> Vec<NotificationPeerErr> {
+        unimplemented!()
+    }
+}

ecstore/src/options.rs

@@ -1,169 +0,0 @@
-use crate::bucket::versioning_sys::BucketVersioningSys;
-use crate::error::{Error, Result};
-use crate::store_api::ObjectOptions;
-use crate::store_err::StorageError;
-use crate::utils::path::is_dir_object;
-use http::{HeaderMap, HeaderValue};
-use lazy_static::lazy_static;
-use std::collections::HashMap;
-use uuid::Uuid;
-
-pub async fn del_opts(
-    bucket: &str,
-    object: &str,
-    vid: Option<String>,
-    headers: &HeaderMap<HeaderValue>,
-    metadata: Option<HashMap<String, String>>,
-) -> Result<ObjectOptions> {
-    let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
-    let version_suspended = BucketVersioningSys::suspended(bucket).await;
-
-    // TODO: delete_prefix
-
-    let vid = vid.map(|v| v.as_str().trim().to_owned());
-
-    if let Some(ref id) = vid {
-        if let Err(_err) = Uuid::parse_str(id.as_str()) {
-            return Err(Error::new(StorageError::InvalidVersionID(
-                bucket.to_owned(),
-                object.to_owned(),
-                id.clone(),
-            )));
-        }
-
-        if !versioned {
-            return Err(Error::new(StorageError::InvalidArgument(
-                bucket.to_owned(),
-                object.to_owned(),
-                id.clone(),
-            )));
-        }
-    }
-
-    let mut opts = put_opts_from_headers(headers, metadata)
-        .map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?;
-
-    opts.version_id = {
-        if is_dir_object(object) && vid.is_none() {
-            Some(Uuid::nil().to_string())
-        } else {
-            vid
-        }
-    };
-    opts.version_suspended = version_suspended;
-    opts.versioned = versioned;
-
-    Ok(opts)
-}
-
-pub async fn put_opts(
-    bucket: &str,
-    object: &str,
-    vid: Option<String>,
-    headers: &HeaderMap<HeaderValue>,
-    metadata: Option<HashMap<String, String>>,
-) -> Result<ObjectOptions> {
-    let versioned = BucketVersioningSys::prefix_enabled(bucket, object).await;
-    let version_suspended = BucketVersioningSys::prefix_suspended(bucket, object).await;
-
-    let vid = vid.map(|v| v.as_str().trim().to_owned());
-
-    if let Some(ref id) = vid {
-        if let Err(_err) = Uuid::parse_str(id.as_str()) {
-            return Err(Error::new(StorageError::InvalidVersionID(
-                bucket.to_owned(),
-                object.to_owned(),
-                id.clone(),
-            )));
-        }
-
-        if !versioned {
-            return Err(Error::new(StorageError::InvalidArgument(
-                bucket.to_owned(),
-                object.to_owned(),
-                id.clone(),
-            )));
-        }
-    }
-
-    let mut opts = put_opts_from_headers(headers, metadata)
-        .map_err(|err| Error::new(StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string())))?;
-
-    opts.version_id = {
-        if is_dir_object(object) && vid.is_none() {
-            Some(Uuid::nil().to_string())
-        } else {
-            vid
-        }
-    };
-    opts.version_suspended = version_suspended;
-    opts.versioned = versioned;
-
-    Ok(opts)
-}
-
-pub fn put_opts_from_headers(
-    headers: &HeaderMap<HeaderValue>,
-    metadata: Option<HashMap<String, String>>,
-) -> Result<ObjectOptions> {
-    let metadata = metadata.unwrap_or_default();
-
-    get_default_opts(headers, metadata, false)
-}
-
-fn get_default_opts(
-    _headers: &HeaderMap<HeaderValue>,
-    metadata: HashMap<String, String>,
-    _copy_source: bool,
-) -> Result<ObjectOptions> {
-    Ok(ObjectOptions {
-        user_defined: metadata.clone(),
-        ..Default::default()
-    })
-}
-
-pub fn extract_metadata(headers: &HeaderMap<HeaderValue>) -> HashMap<String, String> {
-    let mut metadata = HashMap::new();
-
-    extract_metadata_from_mime(headers, &mut metadata);
-
-    metadata
-}
-
-fn extract_metadata_from_mime(headers: &HeaderMap<HeaderValue>, metadata: &mut HashMap<String, String>) {
-    for (k, v) in headers.iter() {
-        if k.as_str().starts_with("x-amz-meta-") {
-            metadata.insert(k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string());
-            continue;
-        }
-
-        if k.as_str().starts_with("x-rustfs-meta-") {
-            metadata.insert(k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string());
-            continue;
-        }
-
-        for hd in SUPPORTED_HEADERS.iter() {
-            if k.as_str() == *hd {
-                metadata.insert(k.to_string(), String::from_utf8_lossy(v.as_bytes()).to_string());
-                continue;
-            }
-        }
-    }
-
-    if !metadata.contains_key("content-type") {
-        metadata.insert("content-type".to_owned(), "binary/octet-stream".to_owned());
-    }
-}
-lazy_static! {
-    static ref SUPPORTED_HEADERS: Vec<&'static str> = vec![
-        "content-type",
-        "cache-control",
-        "content-language",
-        "content-encoding",
-        "content-disposition",
-        "x-amz-storage-class",
-        "x-amz-tagging",
-        "expires",
-        "x-amz-replication-status"
-    ];
-}

ecstore/src/peer_rest_client.rs

@@ -0,0 +1,669 @@
+use std::{collections::HashMap, io::Cursor, time::SystemTime};
+
+use crate::{
+    admin_server_info::ServerProperties, endpoints::EndpointServerPools, global::is_dist_erasure,
+    heal::heal_commands::BgHealState, store_api::StorageInfo,
+};
+use common::error::{Error, Result};
+use madmin::{
+    health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService},
+    metrics::{CollectMetricsOpts, MetricType, RealtimeMetrics},
+    net::NetInfo,
+};
+use protos::{
+    node_service_time_out_client,
+    proto_gen::node_service::{
+        BackgroundHealStatusRequest, DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest,
+        DeleteUserRequest, GetCpusRequest, GetMemInfoRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest,
+        GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, GetSysConfigRequest, GetSysErrorsRequest,
+        LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
+        LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, Mss,
+        ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ServerInfoRequest, SignalServiceRequest,
+        StartProfilingRequest, StopRebalanceRequest,
+    },
+};
+use rmp_serde::{Deserializer, Serializer};
+use serde::{Deserialize, Serialize as _};
+use tonic::Request;
+
+pub const PEER_RESTSIGNAL: &str = "signal";
+pub const PEER_RESTSUB_SYS: &str = "sub-sys";
+pub const PEER_RESTDRY_RUN: &str = "dry-run";
+
+pub struct PeerRestClient {
+    addr: String,
+}
+
+impl PeerRestClient {
+    pub fn new(url: url::Url) -> Self {
+        Self {
+            addr: format!("{}://{}:{}", url.scheme(), url.host_str().unwrap(), url.port().unwrap()),
+        }
+    }
+    pub async fn new_clients(_eps: EndpointServerPools) -> (Vec<Self>, Vec<Self>) {
+        if !is_dist_erasure().await {
+            return (Vec::new(), Vec::new());
+        }
+
+        // FIXME:TODO
+
+        todo!()
+    }
+}
+
+impl PeerRestClient {
+    pub async fn local_storage_info(&self) -> Result<StorageInfo> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LocalStorageInfoRequest { metrics: true });
+
+        let response = client.local_storage_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.storage_info;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let storage_info: StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(storage_info)
+    }
+
+    pub async fn server_info(&self) -> Result<ServerProperties> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(ServerInfoRequest { metrics: true });
+
+        let response = client.server_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.server_properties;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let storage_properties: ServerProperties = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(storage_properties)
+    }
+
+    pub async fn get_cpus(&self) -> Result<Cpus> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetCpusRequest {});
+
+        let response = client.get_cpus(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.cpus;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let cpus: Cpus = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(cpus)
+    }
+
+    pub async fn get_net_info(&self) -> Result<NetInfo> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetNetInfoRequest {});
+
+        let response = client.get_net_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.net_info;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let net_info: NetInfo = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(net_info)
+    }
+
+    pub async fn get_partitions(&self) -> Result<Partitions> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetPartitionsRequest {});
+
+        let response = client.get_partitions(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.partitions;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let partitions: Partitions = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(partitions)
+    }
+
+    pub async fn get_os_info(&self) -> Result<OsInfo> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetOsInfoRequest {});
+
+        let response = client.get_os_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.os_info;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let os_info: OsInfo = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(os_info)
+    }
+
+    pub async fn get_se_linux_info(&self) -> Result<SysService> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetSeLinuxInfoRequest {});
+
+        let response = client.get_se_linux_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.sys_services;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let sys_services: SysService = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(sys_services)
+    }
+
+    pub async fn get_sys_config(&self) -> Result<SysConfig> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetSysConfigRequest {});
+
+        let response = client.get_sys_config(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.sys_config;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let sys_config: SysConfig = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(sys_config)
+    }
+
+    pub async fn get_sys_errors(&self) -> Result<SysErrors> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetSysErrorsRequest {});
+
+        let response = client.get_sys_errors(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.sys_errors;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let sys_errors: SysErrors = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(sys_errors)
+    }
+
+    pub async fn get_mem_info(&self) -> Result<MemInfo> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetMemInfoRequest {});
+
+        let response = client.get_mem_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.mem_info;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let mem_info: MemInfo = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(mem_info)
+    }
+
+    pub async fn get_metrics(&self, t: MetricType, opts: &CollectMetricsOpts) -> Result<RealtimeMetrics> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let mut buf = Vec::new();
+        opts.serialize(&mut Serializer::new(&mut buf))?;
+        let request = Request::new(GetMetricsRequest {
+            metric_type: t,
+            opts: buf,
+        });
+
+        let response = client.get_metrics(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.realtime_metrics;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(realtime_metrics)
+    }
+
+    pub async fn get_proc_info(&self) -> Result<ProcInfo> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(GetProcInfoRequest {});
+
+        let response = client.get_proc_info(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.proc_info;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let proc_info: ProcInfo = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(proc_info)
+    }
+
+    pub async fn start_profiling(&self, profiler: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(StartProfilingRequest {
+            profiler: profiler.to_string(),
+        });
+
+        let response = client.start_profiling(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn download_profile_data(&self) -> Result<()> {
+        todo!()
+    }
+
+    pub async fn get_bucket_stats(&self) -> Result<()> {
+        todo!()
+    }
+
+    pub async fn get_sr_metrics(&self) -> Result<()> {
+        todo!()
+    }
+
+    pub async fn get_all_bucket_stats(&self) -> Result<()> {
+        todo!()
+    }
+
+    pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadBucketMetadataRequest {
+            bucket: bucket.to_string(),
+        });
+
+        let response = client.load_bucket_metadata(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(DeleteBucketMetadataRequest {
+            bucket: bucket.to_string(),
+        });
+
+        let response = client.delete_bucket_metadata(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn delete_policy(&self, policy: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(DeletePolicyRequest {
+            policy_name: policy.to_string(),
+        });
+
+        let response = client.delete_policy(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn load_policy(&self, policy: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadPolicyRequest {
+            policy_name: policy.to_string(),
+        });
+
+        let response = client.load_policy(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn load_policy_mapping(&self, user_or_group: &str, user_type: u64, is_group: bool) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadPolicyMappingRequest {
+            user_or_group: user_or_group.to_string(),
+            user_type,
+            is_group,
+        });
+
+        let response = client.load_policy_mapping(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn delete_user(&self, access_key: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(DeleteUserRequest {
+            access_key: access_key.to_string(),
+        });
+
+        let response = client.delete_user(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn delete_service_account(&self, access_key: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(DeleteServiceAccountRequest {
+            access_key: access_key.to_string(),
+        });
+
+        let response = client.delete_service_account(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn load_user(&self, access_key: &str, temp: bool) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadUserRequest {
+            access_key: access_key.to_string(),
+            temp,
+        });
+
+        let response = client.load_user(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn load_service_account(&self, access_key: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadServiceAccountRequest {
+            access_key: access_key.to_string(),
+        });
+
+        let response = client.load_service_account(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn load_group(&self, group: &str) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadGroupRequest {
+            group: group.to_string(),
+        });
+
+        let response = client.load_group(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn reload_site_replication_config(&self) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(ReloadSiteReplicationConfigRequest {});
+
+        let response = client.reload_site_replication_config(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let mut vars = HashMap::new();
+        vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string());
+        vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string());
+        vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string());
+        let request = Request::new(SignalServiceRequest {
+            vars: Some(Mss { value: vars }),
+        });
+
+        let response = client.signal_service(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        Ok(())
+    }
+
+    pub async fn background_heal_status(&self) -> Result<BgHealState> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(BackgroundHealStatusRequest {});
+
+        let response = client.background_heal_status(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+        let data = response.bg_heal_state;
+
+        let mut buf = Deserializer::new(Cursor::new(data));
+        let bg_heal_state: BgHealState = Deserialize::deserialize(&mut buf).unwrap();
+
+        Ok(bg_heal_state)
+    }
+
+    pub async fn get_metacache_listing(&self) -> Result<()> {
+        let mut _client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        todo!()
+    }
+
+    pub async fn update_metacache_listing(&self) -> Result<()> {
+        let mut _client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        todo!()
+    }
+
+    pub async fn reload_pool_meta(&self) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(ReloadPoolMetaRequest {});
+
+        let response = client.reload_pool_meta(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+
+        Ok(())
+    }
+
+    pub async fn stop_rebalance(&self) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(StopRebalanceRequest {});
+
+        let response = client.stop_rebalance(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+
+        Ok(())
+    }
+
+    pub async fn load_rebalance_meta(&self, start_rebalance: bool) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadRebalanceMetaRequest { start_rebalance });
+
+        let response = client.load_rebalance_meta(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+
+        Ok(())
+    }
+
+    pub async fn load_transition_tier_config(&self) -> Result<()> {
+        let mut client = node_service_time_out_client(&self.addr)
+            .await
+            .map_err(|err| Error::msg(err.to_string()))?;
+        let request = Request::new(LoadTransitionTierConfigRequest {});
+
+        let response = client.load_transition_tier_config(request).await?.into_inner();
+        if !response.success {
+            if let Some(msg) = response.error_info {
+                return Err(Error::msg(msg));
+            }
+            return Err(Error::msg(""));
+        }
+
+        Ok(())
+    }
+}

ecstore/src/utils/net.rs

@@ -2,8 +2,9 @@ use crate::error::{Error, Result};
 use lazy_static::lazy_static;
 use std::{
     collections::HashSet,
-    net::{IpAddr, SocketAddr, ToSocketAddrs},
+    net::{IpAddr, SocketAddr, TcpListener, ToSocketAddrs},
 };
+
 use url::Host;
 
 lazy_static! {
@@ -92,6 +93,10 @@ pub fn get_host_ip(host: Host<&str>) -> Result<HashSet<IpAddr>> {
     }
 }
 
+pub fn get_available_port() -> u16 {
+    TcpListener::bind("0.0.0.0:0").unwrap().local_addr().unwrap().port()
+}
+
 /// returns IPs of local interface
 pub(crate) fn must_get_local_ips() -> Result<Vec<IpAddr>> {
     match netif::up() {