refactor(runtime): wrap internode connection cache (#4036)

crates/common/src/globals.rs

@@ -143,6 +143,16 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv
     }
 }
 
+/// Get a cached gRPC connection for the given address.
+pub async fn cached_connection(addr: &str) -> Option<Channel> {
+    GLOBAL_CONN_MAP.read().await.get(addr).cloned()
+}
+
+/// Cache a gRPC connection for the given address.
+pub async fn cache_connection(addr: String, channel: Channel) {
+    GLOBAL_CONN_MAP.write().await.insert(addr, channel);
+}
+
 /// Check if a connection exists in the cache for the given address.
 ///
 /// # Arguments

crates/ecstore/src/runtime/sources.rs

@@ -42,7 +42,7 @@ use crate::{
     services::tier::tier::TierConfigMgr,
     store::ECStore,
 };
-use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST};
+use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST};
 use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
 use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
@@ -212,17 +212,17 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
 }
 
 pub(crate) async fn cached_node_channel(addr: &str) -> Option<Channel> {
-    GLOBAL_CONN_MAP.read().await.get(addr).cloned()
+    rustfs_common::cached_connection(addr).await
 }
 
 #[cfg(test)]
 pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) {
-    GLOBAL_CONN_MAP.write().await.insert(addr, channel);
+    rustfs_common::cache_connection(addr, channel).await;
 }
 
 #[cfg(test)]
 pub(crate) async fn test_node_channel_is_cached(addr: &str) -> bool {
-    GLOBAL_CONN_MAP.read().await.contains_key(addr)
+    rustfs_common::has_cached_connection(addr).await
 }
 
 #[cfg(test)]

crates/protos/src/lib.rs

@@ -19,7 +19,7 @@ mod generated;
 mod runtime_sources;
 
 use proto_gen::node_service::node_service_client::NodeServiceClient;
-use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
+use rustfs_common::{cache_connection, evict_connection_with_log_level};
 use std::{
     collections::HashMap,
     error::Error,
@@ -188,10 +188,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
         }
     };
 
-    // Cache the new connection
-    {
-        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone());
-    }
+    cache_connection(addr.to_string(), channel.clone()).await;
     {
         let mut generation_cache = TLS_GENERATION_CACHE.lock().await;
         enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr);
@@ -261,10 +258,10 @@ mod tests {
     async fn evict_failed_connection_with_log_level_removes_cached_connection() {
         let addr = "http://evict-failed-connection-debug-test";
         let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
-        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
+        cache_connection(addr.to_string(), channel).await;
 
         evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
 
-        assert!(!GLOBAL_CONN_MAP.read().await.contains_key(addr));
+        assert!(!rustfs_common::has_cached_connection(addr).await);
     }
 }