gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(runtime): wrap internode connection cache (#4036)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-29 10:43:13 +08:00
committed by GitHub
parent df82e3e645
commit 9165de3241
5 changed files with 32 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/globals.rs
+10
View File
@@ -143,6 +143,16 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv
}
}
/// Get a cached gRPC connection for the given address.
pub async fn cached_connection(addr: &str) -> Option<Channel> {
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
}
/// Cache a gRPC connection for the given address.
pub async fn cache_connection(addr: String, channel: Channel) {
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
}
/// Check if a connection exists in the cache for the given address.
///
/// # Arguments
crates/ecstore/src/runtime/sources.rs
+4 -4
View File
@@ -42,7 +42,7 @@ use crate::{
services::tier::tier::TierConfigMgr,
store::ECStore,
};
use rustfs_common::{GLOBAL_CONN_MAP, GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST};
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST};
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
@@ -212,17 +212,17 @@ pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
}
pub(crate) async fn cached_node_channel(addr: &str) -> Option<Channel> {
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
rustfs_common::cached_connection(addr).await
}
#[cfg(test)]
pub(crate) async fn cache_test_node_channel(addr: String, channel: Channel) {
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
rustfs_common::cache_connection(addr, channel).await;
}
#[cfg(test)]
pub(crate) async fn test_node_channel_is_cached(addr: &str) -> bool {
GLOBAL_CONN_MAP.read().await.contains_key(addr)
rustfs_common::has_cached_connection(addr).await
}
#[cfg(test)]
crates/protos/src/lib.rs
+4 -7
View File
@@ -19,7 +19,7 @@ mod generated;
mod runtime_sources;
use proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
use rustfs_common::{cache_connection, evict_connection_with_log_level};
use std::{
collections::HashMap,
error::Error,
@@ -188,10 +188,7 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
}
};
// Cache the new connection
{
GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel.clone());
}
cache_connection(addr.to_string(), channel.clone()).await;
{
let mut generation_cache = TLS_GENERATION_CACHE.lock().await;
enforce_tls_generation_cache_bound(&mut generation_cache, generation, addr);
@@ -261,10 +258,10 @@ mod tests {
async fn evict_failed_connection_with_log_level_removes_cached_connection() {
let addr = "http://evict-failed-connection-debug-test";
let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
cache_connection(addr.to_string(), channel).await;
evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
assert!(!GLOBAL_CONN_MAP.read().await.contains_key(addr));
assert!(!rustfs_common::has_cached_connection(addr).await);
}
}
docs/architecture/global-state-inventory.md
+1 -1
View File
@@ -48,7 +48,7 @@ migration PR removes or replaces each item.
| `GLOBAL_EventNotifier`, `GLOBAL_NotificationSys` | `crates/ecstore/src/services/*` | Runtime migration target | Preserve notification and audit side effects; move only through notify/runtime-source boundaries. |
| `GLOBAL_BOOT_TIME`, `globalDeploymentIDPtr`, `GLOBAL_REGION`, `GLOBAL_RUSTFS_PORT`, `GLOBAL_LocalNodeName` | `crates/ecstore/src/runtime/global.rs` | Runtime migration target | Scalar handles may migrate in small PRs after call sites use AppContext or owner-local runtime-source readers. |
| `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS`, `GLOBAL_LOCK_MANAGER` | `crates/ecstore/src/runtime/global.rs`, `crates/lock` | Runtime migration target / process-global split | Preserve lock quorum and lock client selection; keep process-level lock manager separate from ECStore endpoint-specific clients. |
| `GLOBAL_CONN_MAP`, `GLOBAL_RUSTFS_HOST`, `GLOBAL_RUSTFS_ADDR`, `GLOBAL_ROOT_CERT`, `GLOBAL_MTLS_IDENTITY`, `GLOBAL_OUTBOUND_TLS_GENERATION` | `crates/common`, `crates/tls-runtime`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | Migrate only after internode transport and outbound TLS ownership are explicit; do not change cached channel reuse or TLS reload semantics. |
| `GLOBAL_CONN_MAP`, `GLOBAL_RUSTFS_HOST`, `GLOBAL_RUSTFS_ADDR`, `GLOBAL_ROOT_CERT`, `GLOBAL_MTLS_IDENTITY`, `GLOBAL_OUTBOUND_TLS_GENERATION` | `crates/common`, `crates/tls-runtime`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | `GLOBAL_CONN_MAP` is now owned behind `rustfs_common` cache helpers; migrate the remaining transport and TLS state only after internode transport and outbound TLS ownership are explicit, without changing cached channel reuse or TLS reload semantics. |
| `GLOBAL_HEAL_MANAGER`, `GLOBAL_HEAL_CHANNEL_PROCESSOR`, `GLOBAL_AHM_SERVICES_CANCEL_TOKEN` | `crates/heal` and `crates/common` | Runtime migration target | Needs heal runtime owner handles and queue tests; do not combine with ECStore disk-state movement. |
| `AUDIT_SYSTEM`, `GLOBAL_CAPACITY_MANAGER`, `GLOBAL_BUCKET_TARGET_SYS`, `GLOBAL_PROCESSORS`, `INTERNODE_DATA_TRANSPORT`, `GLOBAL_KMS_SERVICE_MANAGER` | Owner crates | Runtime migration target / process-global split | Track as owner-specific follow-ups; each owner must decide whether AppContext, a runtime-source facade, or process-global state is the right final shape. |
scripts/check_architecture_migration_rules.sh
+13
View File
@@ -182,6 +182,7 @@ EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_
FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt"
EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt"
REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt"
GLOBAL_CONN_MAP_BYPASS_HITS_FILE="${TMP_DIR}/global_conn_map_bypass_hits.txt"
LEGACY_ECSTORE_CONFIG_MODEL_HITS_FILE="${TMP_DIR}/legacy_ecstore_config_model_hits.txt"
ECSTORE_ROOT_STORE_SET_DISK_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_store_set_disk_module_hits.txt"
ECSTORE_ROOT_STORE_SUPPORT_MODULE_HITS_FILE="${TMP_DIR}/ecstore_root_store_support_module_hits.txt"
@@ -2290,6 +2291,18 @@ if [[ -s "$REPLICATION_FACADE_BYPASS_HITS_FILE" ]]; then
report_failure "replication facade imports must stay in local storage_api boundaries: $(paste -sd '; ' "$REPLICATION_FACADE_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --with-filename '\bGLOBAL_CONN_MAP\b' \
crates rustfs fuzz \
--glob '*.rs' |
rg -v '^crates/common/src/globals\.rs:' || true
) >"$GLOBAL_CONN_MAP_BYPASS_HITS_FILE"
if [[ -s "$GLOBAL_CONN_MAP_BYPASS_HITS_FILE" ]]; then
report_failure "GLOBAL_CONN_MAP access must stay behind rustfs_common connection cache helpers: $(paste -sd '; ' "$GLOBAL_CONN_MAP_BYPASS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{