mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 08:36:54 +00:00
fix(storage): harden offline drive fail-fast paths (#2564)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
This commit is contained in:
houseme
@@ -108,7 +108,7 @@ impl LockClient for LocalClient {
|
||||
|
||||
match lock_manager.acquire_lock(lock_request).await {
|
||||
Ok(guard) => {
|
||||
let lock_id = LockId::new_unique(&request.resource);
|
||||
let lock_id = request.lock_id.clone();
|
||||
|
||||
{
|
||||
let shard = self.get_shard(&lock_id);
|
||||
|
||||
@@ -22,6 +22,7 @@ use futures::future::join_all;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -180,6 +181,8 @@ pub struct DistributedLock {
|
||||
quorum: usize,
|
||||
}
|
||||
|
||||
type LockAcquireTaskResult = (usize, Result<LockResponse>);
|
||||
|
||||
impl DistributedLock {
|
||||
/// Create new distributed lock
|
||||
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
@@ -327,108 +330,196 @@ impl DistributedLock {
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
fn spawn_lock_requests(&self, request: &LockRequest) -> JoinSet<LockAcquireTaskResult> {
|
||||
let mut pending = JoinSet::new();
|
||||
for (idx, client) in self.clients.iter().cloned().enumerate() {
|
||||
let request = request.clone();
|
||||
pending.spawn(async move { (idx, client.acquire_lock(&request).await) });
|
||||
}
|
||||
pending
|
||||
}
|
||||
|
||||
async fn release_entries(entries: &[(LockId, Arc<dyn LockClient>)], context: &'static str) {
|
||||
let release_results = join_all(
|
||||
entries
|
||||
.iter()
|
||||
.map(|(lock_id, client)| async move { (lock_id, client.release(lock_id).await) }),
|
||||
)
|
||||
.await;
|
||||
|
||||
for (lock_id, result) in release_results {
|
||||
match result {
|
||||
Ok(true) | Ok(false) => {}
|
||||
Err(err) => {
|
||||
tracing::warn!("{context}: failed to release lock {} on client: {}", lock_id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_pending_cleanup(
|
||||
mut pending: JoinSet<LockAcquireTaskResult>,
|
||||
clients: Vec<Arc<dyn LockClient>>,
|
||||
fallback_lock_id: LockId,
|
||||
context: &'static str,
|
||||
) {
|
||||
let handle = tokio::spawn(async move {
|
||||
while let Some(join_result) = pending.join_next().await {
|
||||
match join_result {
|
||||
Ok((idx, Ok(resp))) if resp.success => {
|
||||
let lock_id = resp
|
||||
.lock_info
|
||||
.as_ref()
|
||||
.map(|info| info.id.clone())
|
||||
.unwrap_or_else(|| fallback_lock_id.clone());
|
||||
let Some(client) = clients.get(idx) else {
|
||||
tracing::warn!("{context}: missing client for pending lock cleanup at index {}", idx);
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Err(err) = client.release(&lock_id).await {
|
||||
tracing::warn!("{context}: failed to cleanup late lock {} on client {}: {}", lock_id, idx, err);
|
||||
}
|
||||
}
|
||||
Ok((idx, Ok(resp))) => {
|
||||
tracing::debug!(
|
||||
"{context}: pending lock request on client {} completed without success: {:?}",
|
||||
idx,
|
||||
resp.error
|
||||
);
|
||||
}
|
||||
Ok((idx, Err(err))) => {
|
||||
tracing::warn!("{context}: pending lock request on client {} failed: {}", idx, err);
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!("{context}: pending lock cleanup task join failed: {}", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
drop(handle);
|
||||
}
|
||||
|
||||
fn log_failed_lock_response(&self, request: &LockRequest, idx: usize, error: String) {
|
||||
if request.suppress_contention_logs {
|
||||
tracing::debug!(
|
||||
resource = %request.resource,
|
||||
owner = %request.owner,
|
||||
"Failed to acquire lock on client from response: {}, error: {}",
|
||||
idx,
|
||||
error
|
||||
);
|
||||
} else {
|
||||
tracing::warn!(
|
||||
resource = %request.resource,
|
||||
owner = %request.owner,
|
||||
"Failed to acquire lock on client from response: {}, error: {}",
|
||||
idx,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Quorum-based lock acquisition: success if at least the required quorum succeeds.
|
||||
/// Collects all individual lock_ids from successful clients and creates an aggregate lock_id.
|
||||
/// Returns the LockResponse with aggregate lock_id and individual lock mappings.
|
||||
async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<(LockId, Arc<dyn LockClient>)>)> {
|
||||
let required_quorum = self.required_quorum(request.lock_type);
|
||||
let futs: Vec<_> = self
|
||||
.clients
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, client)| async move { (idx, client.acquire_lock(request).await) })
|
||||
.collect();
|
||||
|
||||
let results = futures::future::join_all(futs).await;
|
||||
|
||||
// Store all individual lock_ids and their corresponding clients
|
||||
let mut pending = self.spawn_lock_requests(request);
|
||||
let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
|
||||
let fallback_lock_id = request.lock_id.clone();
|
||||
|
||||
for (idx, result) in results {
|
||||
match result {
|
||||
Ok(resp) => {
|
||||
while let Some(join_result) = pending.join_next().await {
|
||||
match join_result {
|
||||
Ok((idx, Ok(resp))) => {
|
||||
if resp.success {
|
||||
// Collect individual lock_id and client for each successful acquisition
|
||||
if let Some(lock_info) = &resp.lock_info
|
||||
&& idx < self.clients.len()
|
||||
{
|
||||
// Save the individual lock_id returned by each client
|
||||
individual_locks.push((lock_info.id.clone(), self.clients[idx].clone()));
|
||||
let lock_id = resp
|
||||
.lock_info
|
||||
.as_ref()
|
||||
.map(|info| info.id.clone())
|
||||
.unwrap_or_else(|| fallback_lock_id.clone());
|
||||
|
||||
if let Some(client) = self.clients.get(idx) {
|
||||
individual_locks.push((lock_id, client.clone()));
|
||||
} else {
|
||||
tracing::warn!("Missing lock client at index {} while recording success", idx);
|
||||
}
|
||||
} else {
|
||||
let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
|
||||
if request.suppress_contention_logs {
|
||||
tracing::debug!(
|
||||
resource = %request.resource,
|
||||
owner = %request.owner,
|
||||
"Failed to acquire lock on client from response: {}, error: {}",
|
||||
idx,
|
||||
error
|
||||
);
|
||||
} else {
|
||||
tracing::warn!(
|
||||
resource = %request.resource,
|
||||
owner = %request.owner,
|
||||
"Failed to acquire lock on client from response: {}, error: {}",
|
||||
idx,
|
||||
error
|
||||
);
|
||||
}
|
||||
self.log_failed_lock_response(request, idx, error);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to acquire lock on client {}: {}", idx, e);
|
||||
Ok((idx, Err(err))) => {
|
||||
tracing::warn!("Failed to acquire lock on client {}: {}", idx, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if individual_locks.len() >= required_quorum {
|
||||
// Generate a new aggregate lock_id for multiple client locks
|
||||
let aggregate_lock_id = generate_aggregate_lock_id(&request.resource);
|
||||
|
||||
tracing::debug!(
|
||||
"Generated aggregate lock_id {} for {} individual locks on resource {}",
|
||||
aggregate_lock_id,
|
||||
individual_locks.len(),
|
||||
request.resource
|
||||
);
|
||||
|
||||
let resp = LockResponse::success(
|
||||
LockInfo {
|
||||
id: aggregate_lock_id,
|
||||
resource: request.resource.clone(),
|
||||
lock_type: request.lock_type,
|
||||
status: LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
Duration::ZERO,
|
||||
);
|
||||
Ok((resp, individual_locks))
|
||||
} else {
|
||||
// Rollback: release all locks that were successfully acquired
|
||||
let rollback_count = individual_locks.len();
|
||||
let rollback_results = join_all(individual_locks.iter().map(|(individual_lock_id, client)| async move {
|
||||
(individual_lock_id, client.release(individual_lock_id).await)
|
||||
}))
|
||||
.await;
|
||||
|
||||
for (individual_lock_id, result) in rollback_results {
|
||||
if let Err(e) = result {
|
||||
tracing::warn!("Failed to rollback lock {} on client: {}", individual_lock_id, e);
|
||||
Err(err) => {
|
||||
tracing::warn!("Lock acquisition task join failed: {}", err);
|
||||
}
|
||||
}
|
||||
|
||||
let resp = LockResponse::failure(
|
||||
format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
|
||||
Duration::ZERO,
|
||||
);
|
||||
Ok((resp, individual_locks))
|
||||
if individual_locks.len() >= required_quorum {
|
||||
if !pending.is_empty() {
|
||||
Self::spawn_pending_cleanup(
|
||||
pending,
|
||||
self.clients.clone(),
|
||||
fallback_lock_id.clone(),
|
||||
"distributed_lock_success_cleanup",
|
||||
);
|
||||
}
|
||||
|
||||
let aggregate_lock_id = generate_aggregate_lock_id(&request.resource);
|
||||
tracing::debug!(
|
||||
"Generated aggregate lock_id {} for {} individual locks on resource {}",
|
||||
aggregate_lock_id,
|
||||
individual_locks.len(),
|
||||
request.resource
|
||||
);
|
||||
|
||||
let resp = LockResponse::success(
|
||||
LockInfo {
|
||||
id: aggregate_lock_id,
|
||||
resource: request.resource.clone(),
|
||||
lock_type: request.lock_type,
|
||||
status: LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
Duration::ZERO,
|
||||
);
|
||||
return Ok((resp, individual_locks));
|
||||
}
|
||||
|
||||
if individual_locks.len() + pending.len() < required_quorum {
|
||||
let rollback_count = individual_locks.len();
|
||||
Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
|
||||
if !pending.is_empty() {
|
||||
Self::spawn_pending_cleanup(
|
||||
pending,
|
||||
self.clients.clone(),
|
||||
fallback_lock_id.clone(),
|
||||
"distributed_lock_failure_cleanup",
|
||||
);
|
||||
}
|
||||
|
||||
let resp = LockResponse::failure(
|
||||
format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
|
||||
Duration::ZERO,
|
||||
);
|
||||
return Ok((resp, individual_locks));
|
||||
}
|
||||
}
|
||||
|
||||
let rollback_count = individual_locks.len();
|
||||
Self::release_entries(&individual_locks, "distributed_lock_quorum_rollback").await;
|
||||
let resp = LockResponse::failure(
|
||||
format!("Failed to acquire quorum: {rollback_count}/{required_quorum} required"),
|
||||
Duration::ZERO,
|
||||
);
|
||||
Ok((resp, individual_locks))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,52 @@ impl crate::client::LockClient for FailingClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DelayedClient {
|
||||
inner: Arc<dyn crate::client::LockClient>,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::client::LockClient for DelayedClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
tokio::time::sleep(self.delay).await;
|
||||
self.inner.acquire_lock(request).await
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.inner.release(lock_id).await
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.inner.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.inner.force_release(lock_id).await
|
||||
}
|
||||
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.inner.check_status(lock_id).await
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.inner.get_stats().await
|
||||
}
|
||||
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.inner.close().await
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
self.inner.is_online().await
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
self.inner.is_local().await
|
||||
}
|
||||
}
|
||||
|
||||
fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
|
||||
ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
@@ -557,3 +603,103 @@ async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() {
|
||||
"expected quorum error, got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_read_lock_returns_after_quorum_without_waiting_for_slow_clients() {
|
||||
let manager_fast_1 = Arc::new(GlobalLockManager::new());
|
||||
let manager_fast_2 = Arc::new(GlobalLockManager::new());
|
||||
let manager_slow_1 = Arc::new(GlobalLockManager::new());
|
||||
let manager_slow_2 = Arc::new(GlobalLockManager::new());
|
||||
|
||||
let client_fast_1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast_1));
|
||||
let client_fast_2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast_2));
|
||||
let client_slow_1: Arc<dyn LockClient> = Arc::new(DelayedClient {
|
||||
inner: Arc::new(LocalClient::with_manager(manager_slow_1.clone())),
|
||||
delay: Duration::from_millis(250),
|
||||
});
|
||||
let client_slow_2: Arc<dyn LockClient> = Arc::new(DelayedClient {
|
||||
inner: Arc::new(LocalClient::with_manager(manager_slow_2.clone())),
|
||||
delay: Duration::from_millis(250),
|
||||
});
|
||||
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"four-node-read".to_string(),
|
||||
vec![client_fast_1, client_fast_2, client_slow_1, client_slow_2],
|
||||
);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let started = tokio::time::Instant::now();
|
||||
let mut guard = lock
|
||||
.get_read_lock(resource.clone(), "owner-a", Duration::from_secs(1))
|
||||
.await
|
||||
.expect("read lock should succeed after reaching quorum");
|
||||
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(150),
|
||||
"read lock should return once quorum is satisfied instead of waiting for slow clients"
|
||||
);
|
||||
assert!(guard.release(), "distributed read guard should release successfully");
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(350)).await;
|
||||
|
||||
let slow_lock_1 = NamespaceLock::with_local_manager("slow-node-1".to_string(), manager_slow_1);
|
||||
let slow_lock_2 = NamespaceLock::with_local_manager("slow-node-2".to_string(), manager_slow_2);
|
||||
|
||||
let write_guard_1 = slow_lock_1
|
||||
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
|
||||
.await
|
||||
.expect("late successful read lock should be cleaned up on slow node 1");
|
||||
let write_guard_2 = slow_lock_2
|
||||
.get_write_lock(resource, "owner-b", Duration::from_millis(100))
|
||||
.await
|
||||
.expect("late successful read lock should be cleaned up on slow node 2");
|
||||
|
||||
drop(write_guard_1);
|
||||
drop(write_guard_2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_failure_returns_early_and_cleans_up_late_successes() {
|
||||
let manager_fast = Arc::new(GlobalLockManager::new());
|
||||
let manager_slow = Arc::new(GlobalLockManager::new());
|
||||
|
||||
let client_fast: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast));
|
||||
let client_fail_1: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let client_fail_2: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let client_slow: Arc<dyn LockClient> = Arc::new(DelayedClient {
|
||||
inner: Arc::new(LocalClient::with_manager(manager_slow.clone())),
|
||||
delay: Duration::from_millis(250),
|
||||
});
|
||||
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"four-node-write".to_string(),
|
||||
vec![client_fast, client_fail_1, client_fail_2, client_slow],
|
||||
);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = lock
|
||||
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("write lock should fail when quorum becomes impossible");
|
||||
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(150),
|
||||
"write lock should fail as soon as quorum becomes impossible"
|
||||
);
|
||||
let err_str = err.to_string().to_lowercase();
|
||||
assert!(
|
||||
err_str.contains("quorum") || err_str.contains("not reached"),
|
||||
"expected quorum failure, got: {err}"
|
||||
);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(350)).await;
|
||||
|
||||
let slow_lock = NamespaceLock::with_local_manager("slow-node".to_string(), manager_slow);
|
||||
let write_guard = slow_lock
|
||||
.get_write_lock(resource, "owner-b", Duration::from_millis(100))
|
||||
.await
|
||||
.expect("late successful write lock should be cleaned up after early quorum failure");
|
||||
|
||||
drop(write_guard);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user