diff --git a/crates/common/src/globals.rs b/crates/common/src/globals.rs index cc0bacd12..ae8e78ccc 100644 --- a/crates/common/src/globals.rs +++ b/crates/common/src/globals.rs @@ -30,6 +30,14 @@ pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock = LazyLock::new(| /// Global initialization time of the RustFS node. pub static GLOBAL_INIT_TIME: LazyLock>>> = LazyLock::new(|| RwLock::new(None)); +/// Log level to use when reporting cached gRPC connection eviction. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConnectionEvictionLogLevel { + Warn, + Info, + Debug, +} + /// Set the global local node name. /// /// # Arguments @@ -105,12 +113,33 @@ pub fn get_global_outbound_tls_generation() -> u64 { /// # Arguments /// * `addr` - The address of the connection to evict. pub async fn evict_connection(addr: &str) { + evict_connection_with_log_level(addr, ConnectionEvictionLogLevel::Info).await; +} + +/// Evict a stale/dead connection from the global connection cache with an explicit log level. +pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) { let removed = GLOBAL_CONN_MAP.write().await.remove(addr); if removed.is_some() { - tracing::info!( - addr = %addr, - "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel" - ); + match log_level { + ConnectionEvictionLogLevel::Warn => { + tracing::warn!( + addr = %addr, + "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel" + ); + } + ConnectionEvictionLogLevel::Info => { + tracing::info!( + addr = %addr, + "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel" + ); + } + ConnectionEvictionLogLevel::Debug => { + tracing::debug!( + addr = %addr, + "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel" + ); + } + } } } diff --git a/crates/ecstore/src/rpc/remote_locker.rs b/crates/ecstore/src/rpc/remote_locker.rs index a89854caf..3138920d4 100644 --- a/crates/ecstore/src/rpc/remote_locker.rs +++ b/crates/ecstore/src/rpc/remote_locker.rs @@ -21,7 +21,8 @@ use rustfs_lock::{ }; use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest}; use rustfs_protos::{ - evict_failed_connection, models::PingBodyBuilder, proto_gen::node_service::node_service_client::NodeServiceClient, + ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder, + proto_gen::node_service::node_service_client::NodeServiceClient, }; use std::time::Duration; use tokio::time::timeout; @@ -86,7 +87,7 @@ impl RemoteClient { } async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) { - if Self::is_scanner_leader_lock(resource_summary) { + let log_level = if Self::is_scanner_leader_lock(resource_summary) { debug!( addr = %self.addr, op, @@ -94,6 +95,7 @@ impl RemoteClient { resource_summary, "Evicting cached remote lock connection for scanner leader-lock RPC failure" ); + ConnectionEvictionLogLevel::Debug } else { warn!( addr = %self.addr, @@ -102,8 +104,9 @@ impl RemoteClient { resource_summary, "Evicting cached remote lock connection after RPC failure" ); - } - evict_failed_connection(&self.addr).await; + ConnectionEvictionLogLevel::Warn + }; + evict_failed_connection_with_log_level(&self.addr, log_level).await; } fn summarize_resources(requests: &[LockRequest]) -> String { diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 735ec910b..c3190b893 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -18,7 +18,7 @@ mod generated; use proto_gen::node_service::node_service_client::NodeServiceClient; -use rustfs_common::{GLOBAL_CONN_MAP, evict_connection}; +use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level}; use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation}; use std::{ @@ -33,7 +33,7 @@ use tonic::{ service::interceptor::InterceptedService, transport::{Certificate, Channel, ClientTlsConfig, Endpoint}, }; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; // Type alias for the complex client type pub type NodeServiceClientType = NodeServiceClient< @@ -41,6 +41,7 @@ pub type NodeServiceClientType = NodeServiceClient< >; pub use generated::*; +pub use rustfs_common::ConnectionEvictionLogLevel; // Default 100 MB pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024; @@ -208,11 +209,37 @@ pub async fn create_new_channel(addr: &str) -> Result> { /// Evict a connection from the cache after a failure. /// This should be called when an RPC fails to ensure fresh connections are tried. pub async fn evict_failed_connection(addr: &str) { - warn!( - addr = %addr, - "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically" - ); - evict_connection(addr).await; + evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Warn).await; +} + +/// Evict a connection from the cache after a failure with an explicit log level. +pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) { + match log_level { + ConnectionEvictionLogLevel::Warn => { + warn!( + addr = %addr, + "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically" + ); + } + ConnectionEvictionLogLevel::Info => { + info!( + addr = %addr, + "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically" + ); + } + ConnectionEvictionLogLevel::Debug => { + debug!( + addr = %addr, + "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically" + ); + } + } + + let cache_log_level = match log_level { + ConnectionEvictionLogLevel::Warn => ConnectionEvictionLogLevel::Info, + level => level, + }; + evict_connection_with_log_level(addr, cache_log_level).await; TLS_GENERATION_CACHE.lock().await.remove(addr); } @@ -230,4 +257,15 @@ mod tests { enforce_tls_generation_cache_bound(&mut cache, 42, "new-node"); assert_eq!(cache.len(), TLS_GENERATION_CACHE_MAX_SIZE - 1); } + + #[tokio::test] + async fn evict_failed_connection_with_log_level_removes_cached_connection() { + let addr = "http://evict-failed-connection-debug-test"; + let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy(); + GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel); + + evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await; + + assert!(!GLOBAL_CONN_MAP.read().await.contains_key(addr)); + } }