perf(storage): optimize internode RPC transfer path (#2262)

Co-authored-by: momoda693 <momoda693@gmail.com>

crates/lock/src/distributed_lock.rs

@@ -230,7 +230,21 @@ impl DistributedLock {
         } else {
             // Check if it's a timeout or quorum failure
             if let Some(error_msg) = &resp.error {
-                warn!("acquire_lock_quorum error: {}", error_msg);
+                if request.suppress_contention_logs {
+                    tracing::debug!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum contention: {}",
+                        error_msg
+                    );
+                } else {
+                    warn!(
+                        resource = %request.resource,
+                        owner = %request.owner,
+                        "acquire_lock_quorum error: {}",
+                        error_msg
+                    );
+                }
                 if error_msg.contains("quorum") {
                     // This is a quorum failure - return appropriate error
                     // Extract achieved count from error message or use individual_locks.len()
@@ -266,6 +280,21 @@ impl DistributedLock {
         self.acquire_guard(&req).await
     }
 
+    /// Convenience: acquire exclusive lock with expected contention logs suppressed
+    pub async fn lock_guard_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+        ttl: Duration,
+    ) -> Result<Option<DistributedLockGuard>> {
+        let req = LockRequest::new(resource, LockType::Exclusive, owner)
+            .with_acquire_timeout(timeout)
+            .with_ttl(ttl)
+            .with_suppress_contention_logs(true);
+        self.acquire_guard(&req).await
+    }
+
     /// Convenience: acquire shared lock as a guard
     pub async fn rlock_guard(
         &self,
@@ -307,11 +336,24 @@ impl DistributedLock {
                             individual_locks.push((lock_info.id.clone(), self.clients[idx].clone()));
                         }
                     } else {
-                        tracing::warn!(
-                            "Failed to acquire lock on client from response: {}, error: {}",
-                            idx,
-                            resp.error.unwrap_or_else(|| "unknown error".to_string())
-                        );
+                        let error = resp.error.unwrap_or_else(|| "unknown error".to_string());
+                        if request.suppress_contention_logs {
+                            tracing::debug!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        } else {
+                            tracing::warn!(
+                                resource = %request.resource,
+                                owner = %request.owner,
+                                "Failed to acquire lock on client from response: {}, error: {}",
+                                idx,
+                                error
+                            );
+                        }
                     }
                 }
                 Err(e) => {

crates/lock/src/namespace/mod.rs

@@ -58,6 +58,16 @@ impl NamespaceLockWrapper {
         self.lock.get_write_lock(self.resource.clone(), &self.owner, timeout).await
     }
 
+    /// Acquire write lock with expected contention logs suppressed
+    pub async fn get_write_lock_quiet(
+        &self,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        self.lock
+            .get_write_lock_quiet(self.resource.clone(), &self.owner, timeout)
+            .await
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
@@ -237,6 +247,29 @@ impl NamespaceLock {
         }
     }
 
+    /// Acquire write lock while suppressing expected contention warnings
+    pub async fn get_write_lock_quiet(
+        &self,
+        resource: ObjectKey,
+        owner: &str,
+        timeout: Duration,
+    ) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
+        let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
+        let resource_str = format!("{}", resource);
+        match self {
+            Self::Distributed(lock) => match lock.lock_guard_quiet(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Standard(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+            Self::Local(lock) => match lock.lock_guard(resource, owner, timeout, ttl).await {
+                Ok(Some(guard)) => Ok(NamespaceLockGuard::Fast(guard)),
+                Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
+                Err(e) => Err(e),
+            },
+        }
+    }
+
     /// Acquire read lock (shared lock) with timeout
     /// Returns the guard if acquisition succeeds, or an error if it fails
     pub async fn get_read_lock(

crates/lock/src/types.rs

@@ -225,6 +225,8 @@ pub struct LockRequest {
     pub priority: LockPriority,
     /// Deadlock detection
     pub deadlock_detection: bool,
+    /// Suppress warning logs for expected contention failures
+    pub suppress_contention_logs: bool,
 }
 
 impl LockRequest {
@@ -240,6 +242,7 @@ impl LockRequest {
             metadata: LockMetadata::default(),
             priority: LockPriority::default(),
             deadlock_detection: false,
+            suppress_contention_logs: false,
         }
     }
 
@@ -272,6 +275,12 @@ impl LockRequest {
         self.deadlock_detection = enabled;
         self
     }
+
+    /// Suppress warning logs for expected contention failures
+    pub fn with_suppress_contention_logs(mut self, enabled: bool) -> Self {
+        self.suppress_contention_logs = enabled;
+        self
+    }
 }
 
 /// Lock response structure