perf(ecstore): batch delete object lock acquisition (#2374)

Co-authored-by: 安正超 <anzhengchao@gmail.com>

crates/lock/src/client/mod.rs

@@ -17,6 +17,7 @@ pub mod local;
 
 use crate::{LockId, LockInfo, LockRequest, LockResponse, LockStats, Result};
 use async_trait::async_trait;
+use futures::future::join_all;
 use std::sync::Arc;
 
 /// Lock client trait
@@ -25,9 +26,25 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
     /// Acquire lock (generic method)
     async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse>;
 
+    /// Acquire multiple locks. Default implementation fans out to single-lock requests.
+    async fn acquire_locks_batch(&self, requests: &[LockRequest]) -> Result<Vec<LockResponse>> {
+        Ok(join_all(requests.iter().map(|request| self.acquire_lock(request)))
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?)
+    }
+
     /// Release lock
     async fn release(&self, lock_id: &LockId) -> Result<bool>;
 
+    /// Release multiple locks. Default implementation fans out to single-lock releases.
+    async fn release_locks_batch(&self, lock_ids: &[LockId]) -> Result<Vec<bool>> {
+        Ok(join_all(lock_ids.iter().map(|lock_id| self.release(lock_id)))
+            .await
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?)
+    }
+
     /// Refresh lock
     async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
 

crates/lock/src/distributed_lock.rs

@@ -18,6 +18,7 @@ use crate::{
     error::{LockError, Result},
     types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
 };
+use futures::future::join_all;
 use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 use tokio::sync::mpsc;
@@ -52,12 +53,13 @@ static UNLOCK_RUNTIME: LazyLock<UnlockRuntime> = LazyLock::new(|| {
     tokio::spawn(async move {
         while let Some(job) = rx.recv().await {
             // Best-effort release across all (LockId, client) entries.
-            let mut any_ok = false;
-            for (lock_id, client) in job.entries.into_iter() {
-                if client.release(&lock_id).await.unwrap_or(false) {
-                    any_ok = true;
-                }
-            }
+            let results = join_all(
+                job.entries
+                    .into_iter()
+                    .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) }),
+            )
+            .await;
+            let any_ok = results.into_iter().any(|released| released);
 
             if !any_ok {
                 tracing::warn!("DistributedLockGuard background release failed for one or more entries");
@@ -142,7 +144,7 @@ impl DistributedLockGuard {
                 let futures_iter = entries
                     .into_iter()
                     .map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) });
-                let _ = futures::future::join_all(futures_iter).await;
+                let _ = join_all(futures_iter).await;
             });
             // Explicitly drop the JoinHandle to acknowledge detaching the task.
             drop(handle);
@@ -411,8 +413,13 @@ impl DistributedLock {
         } else {
             // Rollback: release all locks that were successfully acquired
             let rollback_count = individual_locks.len();
-            for (individual_lock_id, client) in &individual_locks {
-                if let Err(e) = client.release(individual_lock_id).await {
+            let rollback_results = join_all(individual_locks.iter().map(|(individual_lock_id, client)| async move {
+                (individual_lock_id, client.release(individual_lock_id).await)
+            }))
+            .await;
+
+            for (individual_lock_id, result) in rollback_results {
+                if let Err(e) = result {
                     tracing::warn!("Failed to rollback lock {} on client: {}", individual_lock_id, e);
                 }
             }

crates/lock/src/fast_lock/manager.rs

@@ -138,7 +138,7 @@ impl FastObjectLockManager {
             shard_a.cmp(&shard_b).then_with(|| a.key.cmp(&b.key))
         });
 
-        // Try to use stack-allocated vectors for small batches, fallback to heap if needed
+        // Preserve shard order so every concurrent batch acquires locks in the same global order.
         let shard_groups = self.group_requests_by_shard(sorted_requests);
 
         // Choose strategy based on request type
@@ -150,31 +150,28 @@ impl FastObjectLockManager {
     }
 
     /// Group requests by shard with proper fallback handling
-    fn group_requests_by_shard(
-        &self,
-        requests: Vec<ObjectLockRequest>,
-    ) -> std::collections::HashMap<usize, Vec<ObjectLockRequest>> {
-        let mut shard_groups = std::collections::HashMap::new();
+    fn group_requests_by_shard(&self, requests: Vec<ObjectLockRequest>) -> Vec<(usize, Vec<ObjectLockRequest>)> {
+        let mut shard_groups: Vec<(usize, Vec<ObjectLockRequest>)> = Vec::new();
 
         for request in requests {
             let shard_id = request.key.shard_index(self.shard_mask);
-            shard_groups.entry(shard_id).or_insert_with(Vec::new).push(request);
+            match shard_groups.last_mut() {
+                Some((last_shard_id, grouped_requests)) if *last_shard_id == shard_id => grouped_requests.push(request),
+                _ => shard_groups.push((shard_id, vec![request])),
+            }
         }
 
         shard_groups
     }
 
     /// Best effort acquisition (allows partial success)
-    async fn acquire_locks_best_effort(
-        &self,
-        shard_groups: &std::collections::HashMap<usize, Vec<ObjectLockRequest>>,
-    ) -> BatchLockResult {
+    async fn acquire_locks_best_effort(&self, shard_groups: &[(usize, Vec<ObjectLockRequest>)]) -> BatchLockResult {
         let mut all_successful = Vec::new();
         let mut all_failed = Vec::new();
         let mut guards = Vec::new();
 
-        for (&shard_id, requests) in shard_groups {
-            let shard = self.shards[shard_id].clone();
+        for (shard_id, requests) in shard_groups {
+            let shard = self.shards[*shard_id].clone();
 
             for request in requests {
                 let key = request.key.clone();
@@ -212,16 +209,13 @@ impl FastObjectLockManager {
     }
 
     /// Two-phase commit for atomic acquisition
-    async fn acquire_locks_two_phase_commit(
-        &self,
-        shard_groups: &std::collections::HashMap<usize, Vec<ObjectLockRequest>>,
-    ) -> BatchLockResult {
+    async fn acquire_locks_two_phase_commit(&self, shard_groups: &[(usize, Vec<ObjectLockRequest>)]) -> BatchLockResult {
         // Phase 1: Try to acquire all locks
         let mut acquired_guards = Vec::new();
         let mut failed_locks = Vec::new();
 
-        'outer: for (&shard_id, requests) in shard_groups {
-            let shard = self.shards[shard_id].clone();
+        'outer: for (shard_id, requests) in shard_groups {
+            let shard = self.shards[*shard_id].clone();
 
             for request in requests {
                 match shard.acquire_lock(request).await {
@@ -438,3 +432,48 @@ impl LockManager for FastObjectLockManager {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn make_request(manager: &FastObjectLockManager, shard_id: usize, suffix: usize) -> ObjectLockRequest {
+        let mut candidate = 0usize;
+        loop {
+            let object = format!("object-{shard_id}-{suffix}-{candidate}");
+            let key = ObjectKey::new("bucket", object);
+            if key.shard_index(manager.shard_mask) == shard_id {
+                return ObjectLockRequest::new_write(key, "owner");
+            }
+            candidate += 1;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_group_requests_by_shard_preserves_sorted_shard_order() {
+        let manager = FastObjectLockManager::new();
+        let mut requests = vec![
+            make_request(&manager, 3, 0),
+            make_request(&manager, 1, 0),
+            make_request(&manager, 2, 0),
+            make_request(&manager, 1, 1),
+            make_request(&manager, 3, 1),
+        ];
+
+        requests.sort_unstable_by(|a, b| {
+            let shard_a = a.key.shard_index(manager.shard_mask);
+            let shard_b = b.key.shard_index(manager.shard_mask);
+            shard_a.cmp(&shard_b).then_with(|| a.key.cmp(&b.key))
+        });
+
+        let shard_groups = manager.group_requests_by_shard(requests);
+        let shard_ids: Vec<_> = shard_groups.iter().map(|(shard_id, _)| *shard_id).collect();
+
+        assert_eq!(shard_ids, vec![1, 2, 3]);
+        assert_eq!(shard_groups[0].1.len(), 2);
+        assert_eq!(shard_groups[1].1.len(), 1);
+        assert_eq!(shard_groups[2].1.len(), 2);
+
+        manager.shutdown().await;
+    }
+}

crates/lock/src/namespace/tests.rs

@@ -97,6 +97,37 @@ async fn test_namespace_lock_with_clients() {
     assert_eq!(lock.namespace(), "multi-client");
 }
 
+#[tokio::test]
+async fn test_lock_client_default_batch_acquire_and_release() {
+    let manager = Arc::new(GlobalLockManager::new());
+    let client = LocalClient::with_manager(manager);
+    let requests = vec![
+        LockRequest::new(create_test_object_key("bucket", "object-a"), LockType::Exclusive, "owner-a")
+            .with_acquire_timeout(Duration::from_secs(1)),
+        LockRequest::new(create_test_object_key("bucket", "object-b"), LockType::Exclusive, "owner-a")
+            .with_acquire_timeout(Duration::from_secs(1)),
+    ];
+
+    let responses = client.acquire_locks_batch(&requests).await.unwrap();
+    assert_eq!(responses.len(), requests.len());
+    assert!(responses.iter().all(|response| response.success));
+
+    let lock_ids = responses
+        .iter()
+        .map(|response| {
+            response
+                .lock_info
+                .as_ref()
+                .expect("successful batch acquire should return lock info")
+                .id
+                .clone()
+        })
+        .collect::<Vec<_>>();
+    let released = client.release_locks_batch(&lock_ids).await.unwrap();
+
+    assert_eq!(released, vec![true, true]);
+}
+
 #[tokio::test]
 async fn test_namespace_lock_get_resource_key() {
     let client = ClientFactory::create_local();
@@ -452,6 +483,45 @@ async fn test_namespace_lock_distributed_write_lock_fails_with_two_nodes_one_off
     );
 }
 
+#[tokio::test]
+async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_nodes() {
+    let manager1 = Arc::new(GlobalLockManager::new());
+    let manager2 = Arc::new(GlobalLockManager::new());
+
+    let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1.clone()));
+    let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2.clone()));
+    let client3: Arc<dyn LockClient> = Arc::new(FailingClient);
+
+    let resource = create_test_object_key("bucket", "object");
+
+    let distributed_lock = NamespaceLock::with_clients_and_quorum("three-node".to_string(), vec![client1, client2, client3], 3);
+    let err = distributed_lock
+        .get_write_lock(resource.clone(), "owner-a", Duration::from_millis(100))
+        .await
+        .expect_err("write lock should fail when quorum requires all three nodes");
+
+    let err_str = err.to_string().to_lowercase();
+    assert!(
+        err_str.contains("quorum") || err_str.contains("not reached"),
+        "expected quorum error, got: {err}"
+    );
+
+    let local_lock_1 = NamespaceLock::with_local_manager("node-1".to_string(), manager1);
+    let local_lock_2 = NamespaceLock::with_local_manager("node-2".to_string(), manager2);
+
+    let guard1 = local_lock_1
+        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
+        .await
+        .expect("quorum rollback should release node 1");
+    let guard2 = local_lock_2
+        .get_write_lock(resource, "owner-b", Duration::from_millis(100))
+        .await
+        .expect("quorum rollback should release node 2");
+
+    drop(guard1);
+    drop(guard2);
+}
+
 #[tokio::test]
 async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() {
     let manager1 = Arc::new(GlobalLockManager::new());