diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index ae8e78ccc..9a2a498ab 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -143,6 +143,16 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv } } +/// Get a cached gRPC connection for the given address. +pub async fn cached_connection(addr: &str) -> Option { + GLOBAL_CONN_MAP.read().await.get(addr).cloned() +} + +/// Cache a gRPC connection for the given address. +pub async fn cache_connection(addr: String, channel: Channel) { + GLOBAL_CONN_MAP.write().await.insert(addr, channel); +} + /// Check if a connection exists in the cache for the given address. /// /// # Arguments diff --git a/crates/ecstore/src/runtime/sources.rs b/crates/ecstore/src/runtime/sources.rs index 05b8e84f3..411e63fa6 100644 --- a/crates/ecstore/src/runtime/sources.rs +++ b/crates/ecstore/src/runtime/sources.rs @@ -42,7 +42,7 @@ use crate::{ services::tier::tier::TierConfigMgr, store::ECStore, }; -use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST}; +use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST}; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; @@ -212,17 +212,17 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option { } pub(crate) async fn cached_node_channel(addr: &str) -> Option { - GLOBAL_CONN_MAP.read().await.get(addr).cloned() + rustfs_common::cached_connection(addr).await } #[cfg(test)] pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) { - GLOBAL_CONN_MAP.write().await.insert(addr, channel); + rustfs_common::cache_connection(addr, channel).await; } #[cfg(test)] pub(crate) async fn test_node_channel_is_cached(addr: &str) -> bool { - GLOBAL_CONN_MAP.read().await.contains_key(addr) + rustfs_common::has_cached_connection(addr).await } #[cfg(test)] diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 70747585d..eadd2a69b 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -19,7 +19,7 @@ mod generated; mod runtime_sources; use proto_gen::node_service::node_service_client::NodeServiceClient; -use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level}; +use rustfs_common::{cache_connection, evict_connection_with_log_level}; use std::{ collections::HashMap, error::Error, @@ -188,10 +188,7 @@ pub async fn create_new_channel(addr: &str) -> Result> { } }; - // Cache the new connection - { - GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone()); - } + cache_connection(addr.to_string(), channel.clone()).await; { let mut generation_cache = TLS_GENERATION_CACHE.lock().await; enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr); @@ -261,10 +258,10 @@ mod tests { async fn evict_failed_connection_with_log_level_removes_cached_connection() { let addr = "http://evict-failed-connection-debug-test"; let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); - GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel); + cache_connection(addr.to_string(), channel).await; evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await; - assert!(!GLOBAL_CONN_MAP.read().await.contains_key(addr)); + assert!(!rustfs_common::has_cached_connection(addr).await); } } diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 2a225571d..009cdfb36 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -48,7 +48,7 @@ migration PR removes or replaces each item. | `GLOBAL_EventNotifier`, `GLOBAL_NotificationSys` | `crates/ecstore/src/services/*` | Runtime migration target | Preserve notification and audit side effects; move only through notify/runtime-source boundaries. | | `GLOBAL_BOOT_TIME`, `globalDeploymentIDPtr`, `GLOBAL_REGION`, `GLOBAL_RUSTFS_PORT`, `GLOBAL_LocalNodeName` | `crates/ecstore/src/runtime/global.rs` | Runtime migration target | Scalar handles may migrate in small PRs after call sites use AppContext or owner-local runtime-source readers. | | `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS`, `GLOBAL_LOCK_MANAGER` | `crates/ecstore/src/runtime/global.rs`, `crates/lock` | Runtime migration target / process-global split | Preserve lock quorum and lock client selection; keep process-level lock manager separate from ECStore endpoint-specific clients. | -| `GLOBAL_CONN_MAP`, `GLOBAL_RUSTFS_HOST`, `GLOBAL_RUSTFS_ADDR`, `GLOBAL_ROOT_CERT`, `GLOBAL_MTLS_IDENTITY`, `GLOBAL_OUTBOUND_TLS_GENERATION` | `crates/common`, `crates/tls-runtime`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | Migrate only after internode transport and outbound TLS ownership are explicit; do not change cached channel reuse or TLS reload semantics. | +| `GLOBAL_CONN_MAP`, `GLOBAL_RUSTFS_HOST`, `GLOBAL_RUSTFS_ADDR`, `GLOBAL_ROOT_CERT`, `GLOBAL_MTLS_IDENTITY`, `GLOBAL_OUTBOUND_TLS_GENERATION` | `crates/common`, `crates/tls-runtime`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | `GLOBAL_CONN_MAP` is now owned behind `rustfs_common` cache helpers; migrate the remaining transport and TLS state only after internode transport and outbound TLS ownership are explicit, without changing cached channel reuse or TLS reload semantics. | | `GLOBAL_HEAL_MANAGER`, `GLOBAL_HEAL_CHANNEL_PROCESSOR`, `GLOBAL_AHM_SERVICES_CANCEL_TOKEN` | `crates/heal` and `crates/common` | Runtime migration target | Needs heal runtime owner handles and queue tests; do not combine with ECStore disk-state movement. | | `AUDIT_SYSTEM`, `GLOBAL_CAPACITY_MANAGER`, `GLOBAL_BUCKET_TARGET_SYS`, `GLOBAL_PROCESSORS`, `INTERNODE_DATA_TRANSPORT`, `GLOBAL_KMS_SERVICE_MANAGER` | Owner crates | Runtime migration target / process-global split | Track as owner-specific follow-ups; each owner must decide whether AppContext, a runtime-source facade, or process-global state is the right final shape. | diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 38a1357a1..5a4dc67f5 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -182,6 +182,7 @@ EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_ FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt" EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt" REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt" +GLOBAL_CONN_MAP_BYPASS_HITS_FILE="${TMP_DIR}/global_conn_map_bypass_hits.txt" LEGACY_ECSTORE_CONFIG_MODEL_HITS_FILE="${TMP_DIR}/legacy_ecstore_config_model_hits.txt" ECSTORE_ROOT_STORE_SET_DISK_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_store_set_disk_module_hits.txt" ECSTORE_ROOT_STORE_SUPPORT_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_store_support_module_hits.txt" @@ -2290,6 +2291,18 @@ if [[ -s "$REPLICATION_FACADE_BYPASS_HITS_FILE" ]]; then report_failure "replication facade imports must stay in local storage_api boundaries: $(paste -sd '; ' "$REPLICATION_FACADE_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename '\bGLOBAL_CONN_MAP\b' \ + crates rustfs fuzz \ + --glob '*.rs' | + rg -v '^crates/common/src/globals\.rs:' || true +) >"$GLOBAL_CONN_MAP_BYPASS_HITS_FILE" + +if [[ -s "$GLOBAL_CONN_MAP_BYPASS_HITS_FILE" ]]; then + report_failure "GLOBAL_CONN_MAP access must stay behind rustfs_common connection cache helpers: $(paste -sd '; ' "$GLOBAL_CONN_MAP_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" {