fix(logging): suppress scanner leader eviction noise (#3653)

crates/common/src/globals.rs

@@ -30,6 +30,14 @@ pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|
 /// Global initialization time of the RustFS node.
 pub static GLOBAL_INIT_TIME: LazyLock<RwLock<Option<DateTime<Utc>>>> = LazyLock::new(|| RwLock::new(None));
 
+/// Log level to use when reporting cached gRPC connection eviction.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ConnectionEvictionLogLevel {
+    Warn,
+    Info,
+    Debug,
+}
+
 /// Set the global local node name.
 ///
 /// # Arguments
@@ -105,12 +113,33 @@ pub fn get_global_outbound_tls_generation() -> u64 {
 /// # Arguments
 /// * `addr` - The address of the connection to evict.
 pub async fn evict_connection(addr: &str) {
+    evict_connection_with_log_level(addr, ConnectionEvictionLogLevel::Info).await;
+}
+
+/// Evict a stale/dead connection from the global connection cache with an explicit log level.
+pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
     let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
     if removed.is_some() {
-        tracing::info!(
-            addr = %addr,
-            "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel"
-        );
+        match log_level {
+            ConnectionEvictionLogLevel::Warn => {
+                tracing::warn!(
+                    addr = %addr,
+                    "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel"
+                );
+            }
+            ConnectionEvictionLogLevel::Info => {
+                tracing::info!(
+                    addr = %addr,
+                    "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel"
+                );
+            }
+            ConnectionEvictionLogLevel::Debug => {
+                tracing::debug!(
+                    addr = %addr,
+                    "Removed cached gRPC connection so future RPCs will attempt to establish a fresh channel"
+                );
+            }
+        }
     }
 }
 

crates/ecstore/src/rpc/remote_locker.rs

@@ -21,7 +21,8 @@ use rustfs_lock::{
 };
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
 use rustfs_protos::{
-    evict_failed_connection, models::PingBodyBuilder, proto_gen::node_service::node_service_client::NodeServiceClient,
+    ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
+    proto_gen::node_service::node_service_client::NodeServiceClient,
 };
 use std::time::Duration;
 use tokio::time::timeout;
@@ -86,7 +87,7 @@ impl RemoteClient {
     }
 
     async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
-        if Self::is_scanner_leader_lock(resource_summary) {
+        let log_level = if Self::is_scanner_leader_lock(resource_summary) {
             debug!(
                 addr = %self.addr,
                 op,
@@ -94,6 +95,7 @@ impl RemoteClient {
                 resource_summary,
                 "Evicting cached remote lock connection for scanner leader-lock RPC failure"
             );
+            ConnectionEvictionLogLevel::Debug
         } else {
             warn!(
                 addr = %self.addr,
@@ -102,8 +104,9 @@ impl RemoteClient {
                 resource_summary,
                 "Evicting cached remote lock connection after RPC failure"
             );
-        }
-        evict_failed_connection(&self.addr).await;
+            ConnectionEvictionLogLevel::Warn
+        };
+        evict_failed_connection_with_log_level(&self.addr, log_level).await;
     }
 
     fn summarize_resources(requests: &[LockRequest]) -> String {

crates/protos/src/lib.rs

@@ -18,7 +18,7 @@
 mod generated;
 
 use proto_gen::node_service::node_service_client::NodeServiceClient;
-use rustfs_common::{GLOBAL_CONN_MAP, evict_connection};
+use rustfs_common::{GLOBAL_CONN_MAP, evict_connection_with_log_level};
 use rustfs_io_metrics::internode_metrics::global_internode_metrics;
 use rustfs_tls_runtime::{load_global_outbound_tls_state, record_tls_consumer_stale_generation};
 use std::{
@@ -33,7 +33,7 @@ use tonic::{
     service::interceptor::InterceptedService,
     transport::{Certificate, Channel, ClientTlsConfig, Endpoint},
 };
-use tracing::{debug, warn};
+use tracing::{debug, info, warn};
 
 // Type alias for the complex client type
 pub type NodeServiceClientType = NodeServiceClient<
@@ -41,6 +41,7 @@ pub type NodeServiceClientType = NodeServiceClient<
 >;
 
 pub use generated::*;
+pub use rustfs_common::ConnectionEvictionLogLevel;
 
 // Default 100 MB
 pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
@@ -208,11 +209,37 @@ pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
 /// Evict a connection from the cache after a failure.
 /// This should be called when an RPC fails to ensure fresh connections are tried.
 pub async fn evict_failed_connection(addr: &str) {
-    warn!(
-        addr = %addr,
-        "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically"
-    );
-    evict_connection(addr).await;
+    evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Warn).await;
+}
+
+/// Evict a connection from the cache after a failure with an explicit log level.
+pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
+    match log_level {
+        ConnectionEvictionLogLevel::Warn => {
+            warn!(
+                addr = %addr,
+                "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically"
+            );
+        }
+        ConnectionEvictionLogLevel::Info => {
+            info!(
+                addr = %addr,
+                "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically"
+            );
+        }
+        ConnectionEvictionLogLevel::Debug => {
+            debug!(
+                addr = %addr,
+                "Evicting cached gRPC connection after RPC failure; the next request will attempt to reconnect automatically"
+            );
+        }
+    }
+
+    let cache_log_level = match log_level {
+        ConnectionEvictionLogLevel::Warn => ConnectionEvictionLogLevel::Info,
+        level => level,
+    };
+    evict_connection_with_log_level(addr, cache_log_level).await;
     TLS_GENERATION_CACHE.lock().await.remove(addr);
 }
 
@@ -230,4 +257,15 @@ mod tests {
         enforce_tls_generation_cache_bound(&mut cache, 42, "new-node");
         assert_eq!(cache.len(), TLS_GENERATION_CACHE_MAX_SIZE - 1);
     }
+
+    #[tokio::test]
+    async fn evict_failed_connection_with_log_level_removes_cached_connection() {
+        let addr = "http://evict-failed-connection-debug-test";
+        let channel = Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
+
+        evict_failed_connection_with_log_level(addr, ConnectionEvictionLogLevel::Debug).await;
+
+        assert!(!GLOBAL_CONN_MAP.read().await.contains_key(addr));
+    }
 }