gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(lock): retry transient distributed lock timeouts (#3101)

Browse Source 
* fix(lock): retry transient distributed lock timeouts

* fix(lock): avoid retry during pending lock cleanup

* fix(lock): preserve acquire timeout budget

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-05-28 01:55:46 +08:00
committed by GitHub
parent f8e6fc1f10
commit 0c52334480
3 changed files with 269 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/lock/src/distributed_lock.rs
+105 -15
View File
@@ -30,6 +30,8 @@ use uuid::Uuid;
const UNLOCK_RETRY_ATTEMPTS: usize = 3;
const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
const LOCK_ACQUIRE_RETRY_ATTEMPTS: usize = 3;
const LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
/// Generate a new aggregate lock ID for multiple client locks
fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
@@ -131,6 +133,12 @@ pub struct DistributedLock {
type LockAcquireTaskResult = (usize, Result<LockResponse>);
struct LockAcquireQuorumResult {
response: LockResponse,
individual_locks: Vec<(LockId, Arc<dyn LockClient>)>,
pending_cleanup_spawned: bool,
}
impl DistributedLock {
/// Create new distributed lock
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
@@ -185,7 +193,11 @@ impl DistributedLock {
}
let required_quorum = self.required_quorum(request.lock_type);
let (resp, individual_locks) = self.acquire_lock_quorum(request).await?;
let LockAcquireQuorumResult {
response: resp,
individual_locks,
..
} = self.acquire_lock_quorum_with_retry(request).await?;
if resp.success {
// Use aggregate lock_id from LockResponse's LockInfo
// The aggregate id is what we expose to callers; individual_locks carries
@@ -287,6 +299,20 @@ impl DistributedLock {
pending
}
fn lock_acquire_retry_backoff(attempt: usize) -> Duration {
LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF * attempt as u32
}
fn is_retryable_lock_failure(resp: &LockResponse) -> bool {
resp.error
.as_deref()
.map(|error| {
let error = error.to_ascii_lowercase();
error.contains("timeout") || error.contains("rpc failed")
})
.unwrap_or(false)
}
async fn release_entries(entries: Vec<(LockId, Arc<dyn LockClient>)>, context: &'static str) {
let mut pending = entries;
@@ -408,14 +434,57 @@ impl DistributedLock {
}
}
async fn acquire_lock_quorum_with_retry(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
let start = std::time::Instant::now();
let mut attempt = 1;
let mut last_result = None;
loop {
let elapsed = start.elapsed();
if elapsed >= request.acquire_timeout {
break;
}
let remaining = request.acquire_timeout - elapsed;
let mut attempt_request = request.clone();
attempt_request.acquire_timeout = remaining;
let result = self.acquire_lock_quorum_once(&attempt_request).await?;
if result.response.success
|| !result.individual_locks.is_empty()
|| result.pending_cleanup_spawned
|| !Self::is_retryable_lock_failure(&result.response)
|| attempt >= LOCK_ACQUIRE_RETRY_ATTEMPTS
{
return Ok(result);
}
last_result = Some(result);
let backoff = Self::lock_acquire_retry_backoff(attempt);
if start.elapsed().saturating_add(backoff) >= request.acquire_timeout {
break;
}
tokio::time::sleep(backoff).await;
attempt += 1;
}
Ok(last_result.unwrap_or_else(|| LockAcquireQuorumResult {
response: LockResponse::failure("Lock acquisition timeout", request.acquire_timeout),
individual_locks: Vec::new(),
pending_cleanup_spawned: false,
}))
}
/// Quorum-based lock acquisition: success if at least the required quorum succeeds.
/// Collects all individual lock_ids from successful clients and creates an aggregate lock_id.
/// Returns the LockResponse with aggregate lock_id and individual lock mappings.
async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<(LockId, Arc<dyn LockClient>)>)> {
async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
let required_quorum = self.required_quorum(request.lock_type);
let mut pending = self.spawn_lock_requests(request);
let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
let fallback_lock_id = request.lock_id.clone();
let mut last_failure = None;
while let Some(join_result) = pending.join_next().await {
match join_result {
@@ -434,18 +503,22 @@ impl DistributedLock {
}
} else {
let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
self.log_failed_lock_response(request, idx, error);
self.log_failed_lock_response(request, idx, error.clone());
last_failure = Some(error);
}
}
Ok((idx, Err(err))) => {
tracing::warn!("Failed to acquire lock on client {}: {}", idx, err);
last_failure = Some(err.to_string());
}
Err(err) => {
tracing::warn!("Lock acquisition task join failed: {}", err);
last_failure = Some(err.to_string());
}
}
if individual_locks.len() >= required_quorum {
let pending_cleanup_spawned = !pending.is_empty();
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -479,12 +552,17 @@ impl DistributedLock {
},
Duration::ZERO,
);
return Ok((resp, individual_locks));
return Ok(LockAcquireQuorumResult {
response: resp,
individual_locks,
pending_cleanup_spawned,
});
}
if individual_locks.len() + pending.len() < required_quorum {
if !individual_locks.is_empty() && individual_locks.len() + pending.len() < required_quorum {
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
let pending_cleanup_spawned = !pending.is_empty();
if !pending.is_empty() {
Self::spawn_pending_cleanup(
pending,
@@ -494,21 +572,33 @@ impl DistributedLock {
);
}
let resp = LockResponse::failure(
format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
Duration::ZERO,
);
return Ok((resp, individual_locks));
let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
if let Some(last_failure) = last_failure {
error.push_str("; last failure: ");
error.push_str(&last_failure);
}
let resp = LockResponse::failure(error, Duration::ZERO);
return Ok(LockAcquireQuorumResult {
response: resp,
individual_locks,
pending_cleanup_spawned,
});
}
}
let rollback_count = individual_locks.len();
Self::spawn_release_cleanup(individual_locks.clone(), "distributed_lock_quorum_rollback");
let resp = LockResponse::failure(
format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
Duration::ZERO,
);
Ok((resp, individual_locks))
let mut error = format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required");
if let Some(last_failure) = last_failure {
error.push_str("; last failure: ");
error.push_str(&last_failure);
}
let resp = LockResponse::failure(error, Duration::ZERO);
Ok(LockAcquireQuorumResult {
response: resp,
individual_locks,
pending_cleanup_spawned: false,
})
}
}