gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 13:16:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(lock): retry transient distributed lock timeouts (#3101)

Browse Source 
* fix(lock): retry transient distributed lock timeouts

* fix(lock): avoid retry during pending lock cleanup

* fix(lock): preserve acquire timeout budget

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-05-28 01:55:46 +08:00
committed by GitHub
parent f8e6fc1f10
commit 0c52334480
3 changed files with 269 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/rpc/remote_locker.rs
+32 -7
View File
@@ -101,7 +101,12 @@ impl RemoteClient {
}
Err(_) => {
let reason = format!("RPC timed out after {:?}", timeout_duration);
self.evict_connection(op, &reason).await;
warn!(
addr = %self.addr,
op,
reason,
"Remote lock RPC timed out without evicting cached connection"
);
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), timeout_duration))
}
}
@@ -494,7 +499,7 @@ mod tests {
}
#[tokio::test]
async fn test_remote_client_acquire_lock_respects_request_timeout_and_evicts_connection() {
async fn test_remote_client_acquire_lock_respects_request_timeout_without_evicting_connection() {
ensure_test_rpc_secret();
let (addr, accept_task) = spawn_hanging_listener().await;
cache_lazy_channel(&addr).await;
@@ -513,15 +518,15 @@ mod tests {
assert!(!response.success, "timed out lock acquisition should fail");
assert_eq!(response.error.as_deref(), Some("Lock acquisition timeout"));
assert!(
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"timeout should evict cached connection"
GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"local timeout should keep cached connection"
);
accept_task.abort();
}
#[tokio::test]
async fn test_remote_client_acquire_locks_batch_respects_request_timeout_and_evicts_connection() {
async fn test_remote_client_acquire_locks_batch_respects_request_timeout_without_evicting_connection() {
ensure_test_rpc_secret();
let (addr, accept_task) = spawn_hanging_listener().await;
cache_lazy_channel(&addr).await;
@@ -541,13 +546,33 @@ mod tests {
assert!(!responses[0].success, "timed out batch lock acquisition should fail");
assert_eq!(responses[0].error.as_deref(), Some("Lock acquisition timeout"));
assert!(
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"batch timeout should evict cached connection"
GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"local batch timeout should keep cached connection"
);
accept_task.abort();
}
#[tokio::test]
async fn test_remote_client_rpc_status_error_evicts_connection() {
let addr = "http://127.0.0.1:1".to_string();
cache_lazy_channel(&addr).await;
assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
let client = RemoteClient::new(addr.clone());
let result = client
.execute_rpc("lock", Duration::from_millis(50), async {
Err::<(), _>(tonic::Status::unavailable("connection unavailable"))
})
.await;
assert!(result.is_err(), "RPC status errors should be returned");
assert!(
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
"RPC status errors should evict cached connection"
);
}
#[test]
fn test_remote_client_zero_timeout_is_clamped() {
assert_eq!(RemoteClient::rpc_timeout(Duration::ZERO), Duration::from_millis(1));