fix(storage): harden offline drive fail-fast paths (#2564)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/ecstore/src/rpc/remote_locker.rs

@@ -18,8 +18,10 @@ use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
 };
-use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
+use rustfs_protos::{evict_failed_connection, proto_gen::node_service::node_service_client::NodeServiceClient};
+use std::time::Duration;
+use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
@@ -62,6 +64,77 @@ impl RemoteClient {
             .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))
     }
 
+    async fn evict_connection(&self, op: &'static str, reason: &str) {
+        warn!(
+            addr = %self.addr,
+            op,
+            reason,
+            "Evicting cached remote lock connection after RPC failure"
+        );
+        evict_failed_connection(&self.addr).await;
+    }
+
+    fn rpc_timeout(timeout_duration: Duration) -> Duration {
+        if timeout_duration.is_zero() {
+            Duration::from_millis(1)
+        } else {
+            timeout_duration
+        }
+    }
+
+    async fn execute_rpc<T, F>(
+        &self,
+        op: &'static str,
+        timeout_duration: Duration,
+        future: F,
+    ) -> std::result::Result<T, LockError>
+    where
+        F: std::future::Future<Output = std::result::Result<T, tonic::Status>>,
+    {
+        let timeout_duration = Self::rpc_timeout(timeout_duration);
+        match timeout(timeout_duration, future).await {
+            Ok(Ok(response)) => Ok(response),
+            Ok(Err(err)) => {
+                let reason = err.to_string();
+                self.evict_connection(op, &reason).await;
+                Err(LockError::internal(format!("{op} RPC failed: {reason}")))
+            }
+            Err(_) => {
+                let reason = format!("RPC timed out after {:?}", timeout_duration);
+                self.evict_connection(op, &reason).await;
+                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), timeout_duration))
+            }
+        }
+    }
+
+    fn timeout_failure_response(request: &LockRequest) -> LockResponse {
+        LockResponse::failure("Lock acquisition timeout", request.acquire_timeout)
+    }
+
+    fn rpc_failure_response(_request: &LockRequest, err: &LockError) -> LockResponse {
+        LockResponse::failure(format!("Remote lock RPC failed: {err}"), Duration::ZERO)
+    }
+
+    fn timeout_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
+        requests.iter().map(Self::timeout_failure_response).collect()
+    }
+
+    fn rpc_failure_batch(requests: &[LockRequest], err: &LockError) -> Vec<LockResponse> {
+        requests
+            .iter()
+            .map(|request| Self::rpc_failure_response(request, err))
+            .collect()
+    }
+
+    fn batch_rpc_timeout(requests: &[LockRequest]) -> Duration {
+        requests
+            .iter()
+            .map(|request| request.acquire_timeout)
+            .max()
+            .map(Self::rpc_timeout)
+            .unwrap_or_else(|| Duration::from_millis(1))
+    }
+
     fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
         if let Some(lock_info_json) = lock_info_json {
             match serde_json::from_str::<LockInfo>(&lock_info_json) {
@@ -111,11 +184,11 @@ impl LockClient for RemoteClient {
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
 
-        let resp = client
-            .lock(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        let resp = match self.execute_rpc("lock", request.acquire_timeout, client.lock(req)).await {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_response(request)),
+            Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
+        };
 
         // Check for explicit error first
         if let Some(error_info) = resp.error_info {
@@ -138,6 +211,10 @@ impl LockClient for RemoteClient {
     }
 
     async fn acquire_locks_batch(&self, requests: &[LockRequest]) -> Result<Vec<LockResponse>> {
+        if requests.is_empty() {
+            return Ok(Vec::new());
+        }
+
         let mut client = self.get_client().await?;
         let req = Request::new(BatchGenerallyLockRequest {
             args: requests
@@ -148,11 +225,14 @@ impl LockClient for RemoteClient {
                 .collect::<Result<Vec<_>>>()?,
         });
 
-        let resp = client
-            .lock_batch(req)
+        let resp = match self
+            .execute_rpc("lock_batch", Self::batch_rpc_timeout(requests), client.lock_batch(req))
             .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_batch(requests)),
+            Err(err) => return Ok(Self::rpc_failure_batch(requests, &err)),
+        };
 
         Ok(requests
             .iter()
@@ -380,3 +460,95 @@ impl LockClient for RemoteClient {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_common::GLOBAL_CONN_MAP;
+    use rustfs_lock::{ObjectKey, types::LockPriority};
+    use tokio::net::TcpListener;
+    use tokio::task::JoinHandle;
+    use tonic::transport::Endpoint as TonicEndpoint;
+
+    async fn spawn_hanging_listener() -> (String, JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = format!("http://{}", listener.local_addr().unwrap());
+        let task = tokio::spawn(async move {
+            if let Ok((stream, _)) = listener.accept().await {
+                let _stream = stream;
+                tokio::time::sleep(Duration::from_secs(2)).await;
+            }
+        });
+        (addr, task)
+    }
+
+    async fn cache_lazy_channel(addr: &str) {
+        let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
+    }
+
+    fn test_lock_request(timeout_duration: Duration) -> LockRequest {
+        LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner-a")
+            .with_acquire_timeout(timeout_duration)
+            .with_priority(LockPriority::Normal)
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_lock_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let request = test_lock_request(Duration::from_millis(50));
+        let started_at = tokio::time::Instant::now();
+
+        let response = client.acquire_lock(&request).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote lock RPC should honor request timeout"
+        );
+        assert!(!response.success, "timed out lock acquisition should fail");
+        assert_eq!(response.error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_locks_batch_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let requests = vec![test_lock_request(Duration::from_millis(50))];
+        let started_at = tokio::time::Instant::now();
+
+        let responses = client.acquire_locks_batch(&requests).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote batch lock RPC should honor request timeout"
+        );
+        assert_eq!(responses.len(), 1);
+        assert!(!responses[0].success, "timed out batch lock acquisition should fail");
+        assert_eq!(responses[0].error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "batch timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[test]
+    fn test_remote_client_zero_timeout_is_clamped() {
+        assert_eq!(RemoteClient::rpc_timeout(Duration::ZERO), Duration::from_millis(1));
+        assert_eq!(RemoteClient::rpc_timeout(Duration::from_millis(25)), Duration::from_millis(25));
+    }
+}