mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 12:26:37 +00:00
refactor(lock): unify NamespaceLock client model and LockRequest API
- Refactor NamespaceLock to use a unified client vector and quorum mechanism, removing legacy local/distributed lock split and related code. - Update LockRequest to split timeout into acquire_timeout and ttl, and add builder methods for both. - Adjust all batch lock APIs to accept ttl and use new LockRequest fields. - Update all affected tests and documentation for the new API. Signed-off-by: junxiang Mu <1948535941@qq.com>
This commit is contained in:
+180
-47
@@ -37,27 +37,6 @@ impl LocalClient {
|
||||
pub fn get_lock_map(&self) -> Arc<LocalLockMap> {
|
||||
crate::get_global_lock_map()
|
||||
}
|
||||
|
||||
/// Convert LockRequest to batch operation
|
||||
async fn request_to_batch(&self, request: LockRequest) -> Result<bool> {
|
||||
let lock_map = self.get_lock_map();
|
||||
let resources = vec![request.resource];
|
||||
let timeout = request.timeout;
|
||||
|
||||
match request.lock_type {
|
||||
LockType::Exclusive => lock_map.lock_batch(&resources, &request.owner, timeout, None).await,
|
||||
LockType::Shared => lock_map.rlock_batch(&resources, &request.owner, timeout, None).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert LockId to resource for release
|
||||
async fn lock_id_to_batch_release(&self, lock_id: &LockId) -> Result<()> {
|
||||
let lock_map = self.get_lock_map();
|
||||
// For simplicity, we'll use the lock_id as resource name
|
||||
// In a real implementation, you might want to maintain a mapping
|
||||
let resources = vec![lock_id.as_str().to_string()];
|
||||
lock_map.unlock_batch(&resources, "unknown").await
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LocalClient {
|
||||
@@ -68,10 +47,10 @@ impl Default for LocalClient {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LockClient for LocalClient {
|
||||
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse> {
|
||||
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let lock_map = self.get_lock_map();
|
||||
let success = lock_map
|
||||
.lock_with_ttl_id(&request.resource, &request.owner, request.timeout, None)
|
||||
.lock_with_ttl_id(request)
|
||||
.await
|
||||
.map_err(|e| crate::error::LockError::internal(format!("Lock acquisition failed: {e}")))?;
|
||||
if success {
|
||||
@@ -82,7 +61,7 @@ impl LockClient for LocalClient {
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.timeout,
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
@@ -94,10 +73,10 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse> {
|
||||
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let lock_map = self.get_lock_map();
|
||||
let success = lock_map
|
||||
.rlock_with_ttl_id(&request.resource, &request.owner, request.timeout, None)
|
||||
.rlock_with_ttl_id(request)
|
||||
.await
|
||||
.map_err(|e| crate::error::LockError::internal(format!("Shared lock acquisition failed: {e}")))?;
|
||||
if success {
|
||||
@@ -108,7 +87,7 @@ impl LockClient for LocalClient {
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.timeout,
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
@@ -122,11 +101,19 @@ impl LockClient for LocalClient {
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let lock_map = self.get_lock_map();
|
||||
lock_map
|
||||
.unlock_by_id(lock_id)
|
||||
.await
|
||||
.map_err(|e| crate::error::LockError::internal(format!("Release failed: {e}")))?;
|
||||
Ok(true)
|
||||
|
||||
// Try to release the lock directly by ID
|
||||
match lock_map.unlock_by_id(lock_id).await {
|
||||
Ok(()) => Ok(true),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
// Try as read lock if exclusive unlock failed
|
||||
match lock_map.runlock_by_id(lock_id).await {
|
||||
Ok(()) => Ok(true),
|
||||
Err(_) => Err(crate::error::LockError::internal("Lock ID not found".to_string())),
|
||||
}
|
||||
}
|
||||
Err(e) => Err(crate::error::LockError::internal(format!("Release lock failed: {e}"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn refresh(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
@@ -140,15 +127,34 @@ impl LockClient for LocalClient {
|
||||
|
||||
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
|
||||
let lock_map = self.get_lock_map();
|
||||
if let Some((resource, owner)) = lock_map.lockid_map.get(lock_id).map(|v| v.clone()) {
|
||||
let is_locked = lock_map.is_locked(&resource).await;
|
||||
if is_locked {
|
||||
|
||||
// Check if the lock exists in our locks map
|
||||
let locks_guard = lock_map.locks.read().await;
|
||||
if let Some(entry) = locks_guard.get(lock_id) {
|
||||
let entry_guard = entry.read().await;
|
||||
|
||||
// Determine lock type and owner based on the entry
|
||||
if let Some(owner) = &entry_guard.writer {
|
||||
Ok(Some(LockInfo {
|
||||
id: lock_id.clone(),
|
||||
resource,
|
||||
lock_type: LockType::Exclusive, // 这里可进一步完善
|
||||
resource: lock_id.resource.clone(),
|
||||
lock_type: crate::types::LockType::Exclusive,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner,
|
||||
owner: owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: LockMetadata::default(),
|
||||
priority: LockPriority::Normal,
|
||||
wait_start_time: None,
|
||||
}))
|
||||
} else if !entry_guard.readers.is_empty() {
|
||||
Ok(Some(LockInfo {
|
||||
id: lock_id.clone(),
|
||||
resource: lock_id.resource.clone(),
|
||||
lock_type: crate::types::LockType::Shared,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: entry_guard.readers.iter().next().map(|(k, _)| k.clone()).unwrap_or_default(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
@@ -189,31 +195,44 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_local_client_acquire_exclusive() {
|
||||
let client = LocalClient::new();
|
||||
let request =
|
||||
LockRequest::new("test-resource", LockType::Exclusive, "test-owner").with_timeout(std::time::Duration::from_secs(30));
|
||||
let resource_name = format!("test-resource-exclusive-{}", uuid::Uuid::new_v4());
|
||||
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_secs(30));
|
||||
|
||||
let response = client.acquire_exclusive(request).await.unwrap();
|
||||
let response = client.acquire_exclusive(&request).await.unwrap();
|
||||
assert!(response.is_success());
|
||||
|
||||
// Clean up
|
||||
if let Some(lock_info) = response.lock_info() {
|
||||
let _ = client.release(&lock_info.id).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_client_acquire_shared() {
|
||||
let client = LocalClient::new();
|
||||
let request =
|
||||
LockRequest::new("test-resource", LockType::Shared, "test-owner").with_timeout(std::time::Duration::from_secs(30));
|
||||
let resource_name = format!("test-resource-shared-{}", uuid::Uuid::new_v4());
|
||||
let request = LockRequest::new(&resource_name, LockType::Shared, "test-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_secs(30));
|
||||
|
||||
let response = client.acquire_shared(request).await.unwrap();
|
||||
let response = client.acquire_shared(&request).await.unwrap();
|
||||
assert!(response.is_success());
|
||||
|
||||
// Clean up
|
||||
if let Some(lock_info) = response.lock_info() {
|
||||
let _ = client.release(&lock_info.id).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_client_release() {
|
||||
let client = LocalClient::new();
|
||||
let resource_name = format!("test-resource-release-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// First acquire a lock
|
||||
let request =
|
||||
LockRequest::new("test-resource", LockType::Exclusive, "test-owner").with_timeout(std::time::Duration::from_secs(30));
|
||||
let response = client.acquire_exclusive(request).await.unwrap();
|
||||
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_secs(30));
|
||||
let response = client.acquire_exclusive(&request).await.unwrap();
|
||||
assert!(response.is_success());
|
||||
|
||||
// Get the lock ID from the response
|
||||
@@ -230,4 +249,118 @@ mod tests {
|
||||
let client = LocalClient::new();
|
||||
assert!(client.is_local().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_client_read_write_lock_exclusion() {
|
||||
let client = LocalClient::new();
|
||||
let resource_name = format!("test-resource-exclusion-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// First, acquire an exclusive lock
|
||||
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_millis(10));
|
||||
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
|
||||
assert!(exclusive_response.is_success());
|
||||
|
||||
// Try to acquire a shared lock on the same resource - should fail
|
||||
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_millis(10));
|
||||
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
|
||||
assert!(!shared_response.is_success(), "Shared lock should fail when exclusive lock exists");
|
||||
|
||||
// Clean up exclusive lock
|
||||
if let Some(exclusive_info) = exclusive_response.lock_info() {
|
||||
let _ = client.release(&exclusive_info.id).await;
|
||||
}
|
||||
|
||||
// Now shared lock should succeed
|
||||
let shared_request2 = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_millis(10));
|
||||
let shared_response2 = client.acquire_shared(&shared_request2).await.unwrap();
|
||||
assert!(
|
||||
shared_response2.is_success(),
|
||||
"Shared lock should succeed after exclusive lock is released"
|
||||
);
|
||||
|
||||
// Clean up
|
||||
if let Some(shared_info) = shared_response2.lock_info() {
|
||||
let _ = client.release(&shared_info.id).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_local_client_read_write_lock_distinction() {
|
||||
let client = LocalClient::new();
|
||||
let resource_name = format!("test-resource-rw-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// Test exclusive lock
|
||||
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_secs(30));
|
||||
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
|
||||
assert!(exclusive_response.is_success());
|
||||
|
||||
if let Some(exclusive_info) = exclusive_response.lock_info() {
|
||||
assert_eq!(exclusive_info.lock_type, LockType::Exclusive);
|
||||
|
||||
// Check status should return correct lock type
|
||||
let status = client.check_status(&exclusive_info.id).await.unwrap();
|
||||
assert!(status.is_some());
|
||||
assert_eq!(status.unwrap().lock_type, LockType::Exclusive);
|
||||
|
||||
// Release exclusive lock
|
||||
let result = client.release(&exclusive_info.id).await.unwrap();
|
||||
assert!(result);
|
||||
}
|
||||
|
||||
// Test shared lock
|
||||
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
|
||||
.with_acquire_timeout(std::time::Duration::from_secs(30));
|
||||
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
|
||||
assert!(shared_response.is_success());
|
||||
|
||||
if let Some(shared_info) = shared_response.lock_info() {
|
||||
assert_eq!(shared_info.lock_type, LockType::Shared);
|
||||
|
||||
// Check status should return correct lock type
|
||||
let status = client.check_status(&shared_info.id).await.unwrap();
|
||||
assert!(status.is_some());
|
||||
assert_eq!(status.unwrap().lock_type, LockType::Shared);
|
||||
|
||||
// Release shared lock
|
||||
let result = client.release(&shared_info.id).await.unwrap();
|
||||
assert!(result);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_multiple_local_clients_exclusive_mutex() {
|
||||
let client1 = LocalClient::new();
|
||||
let client2 = LocalClient::new();
|
||||
let resource_name = format!("test-multi-client-mutex-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// client1 acquire exclusive lock
|
||||
let req1 = LockRequest::new(&resource_name, LockType::Exclusive, "owner1")
|
||||
.with_acquire_timeout(std::time::Duration::from_millis(50));
|
||||
let resp1 = client1.acquire_exclusive(&req1).await.unwrap();
|
||||
assert!(resp1.is_success(), "client1 should acquire exclusive lock");
|
||||
|
||||
// client2 try to acquire exclusive lock, should fail
|
||||
let req2 = LockRequest::new(&resource_name, LockType::Exclusive, "owner2")
|
||||
.with_acquire_timeout(std::time::Duration::from_millis(50));
|
||||
let resp2 = client2.acquire_exclusive(&req2).await.unwrap();
|
||||
assert!(!resp2.is_success(), "client2 should not acquire exclusive lock while client1 holds it");
|
||||
|
||||
// client1 release lock
|
||||
if let Some(lock_info) = resp1.lock_info() {
|
||||
let _ = client1.release(&lock_info.id).await;
|
||||
}
|
||||
|
||||
// client2 try again, should succeed
|
||||
let resp3 = client2.acquire_exclusive(&req2).await.unwrap();
|
||||
assert!(resp3.is_success(), "client2 should acquire exclusive lock after client1 releases it");
|
||||
|
||||
// clean up
|
||||
if let Some(lock_info) = resp3.lock_info() {
|
||||
let _ = client2.release(&lock_info.id).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,13 +27,13 @@ use crate::{
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Acquire exclusive lock
|
||||
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse>;
|
||||
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse>;
|
||||
|
||||
/// Acquire shared lock
|
||||
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse>;
|
||||
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse>;
|
||||
|
||||
/// Acquire lock (generic method)
|
||||
async fn acquire_lock(&self, request: LockRequest) -> Result<LockResponse> {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
match request.lock_type {
|
||||
crate::types::LockType::Exclusive => self.acquire_exclusive(request).await,
|
||||
crate::types::LockType::Shared => self.acquire_shared(request).await,
|
||||
@@ -101,7 +101,7 @@ mod tests {
|
||||
let request = crate::types::LockRequest::new("test-resource", LockType::Exclusive, "test-owner");
|
||||
|
||||
// Test lock acquisition
|
||||
let response = client.acquire_exclusive(request).await;
|
||||
let response = client.acquire_exclusive(&request).await;
|
||||
assert!(response.is_ok());
|
||||
|
||||
if let Ok(response) = response {
|
||||
|
||||
@@ -13,8 +13,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use rustfs_protos::{
|
||||
node_service_time_out_client,
|
||||
proto_gen::node_service::{GenerallyLockRequest, PingRequest},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tonic::Request;
|
||||
use tracing::info;
|
||||
|
||||
@@ -25,154 +30,220 @@ use crate::{
|
||||
|
||||
use super::LockClient;
|
||||
|
||||
/// RPC lock arguments for gRPC communication
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct LockArgs {
|
||||
pub uid: String,
|
||||
pub resources: Vec<String>,
|
||||
pub owner: String,
|
||||
pub source: String,
|
||||
pub quorum: u32,
|
||||
}
|
||||
|
||||
impl LockArgs {
|
||||
fn from_request(request: &LockRequest, _is_shared: bool) -> Self {
|
||||
Self {
|
||||
uid: request.metadata.operation_id.clone().unwrap_or_default(),
|
||||
resources: vec![request.resource.clone()],
|
||||
owner: request.owner.clone(),
|
||||
source: "remote".to_string(),
|
||||
quorum: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_lock_id(lock_id: &LockId) -> Self {
|
||||
Self {
|
||||
uid: lock_id.as_str().to_string(),
|
||||
resources: vec![lock_id.as_str().to_string()],
|
||||
owner: "remote".to_string(),
|
||||
source: "remote".to_string(),
|
||||
quorum: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remote lock client implementation
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct RemoteClient {
|
||||
addr: String,
|
||||
// Track active locks with their original owner information
|
||||
active_locks: Arc<RwLock<HashMap<LockId, String>>>, // lock_id -> owner
|
||||
}
|
||||
|
||||
impl Clone for RemoteClient {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
addr: self.addr.clone(),
|
||||
active_locks: self.active_locks.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteClient {
|
||||
pub fn new(endpoint: String) -> Self {
|
||||
Self { addr: endpoint }
|
||||
Self {
|
||||
addr: endpoint,
|
||||
active_locks: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_url(url: url::Url) -> Self {
|
||||
Self { addr: url.to_string() }
|
||||
Self {
|
||||
addr: url.to_string(),
|
||||
active_locks: Arc::new(RwLock::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a minimal LockRequest for unlock operations
|
||||
fn create_unlock_request(&self, lock_id: &LockId, owner: &str) -> LockRequest {
|
||||
LockRequest {
|
||||
lock_id: lock_id.clone(),
|
||||
resource: lock_id.resource.clone(),
|
||||
lock_type: crate::types::LockType::Exclusive, // Type doesn't matter for unlock
|
||||
owner: owner.to_string(),
|
||||
acquire_timeout: std::time::Duration::from_secs(30),
|
||||
ttl: std::time::Duration::from_secs(300),
|
||||
metadata: crate::types::LockMetadata::default(),
|
||||
priority: crate::types::LockPriority::Normal,
|
||||
deadlock_detection: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse> {
|
||||
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let args = LockArgs::from_request(&request, false);
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
let resp = client
|
||||
.lock(req)
|
||||
.await
|
||||
.map_err(|e| LockError::internal(e.to_string()))?
|
||||
.into_inner();
|
||||
|
||||
// Check for explicit error first
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
LockInfo {
|
||||
id: LockId::new_deterministic(&request.resource),
|
||||
resource: request.resource,
|
||||
lock_type: request.lock_type,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner,
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.timeout,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata,
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
// Save the lock information for later release
|
||||
let mut locks = self.active_locks.write().await;
|
||||
locks.insert(request.lock_id.clone(), request.owner.clone());
|
||||
|
||||
Ok(LockResponse::success(
|
||||
LockInfo {
|
||||
id: request.lock_id.clone(),
|
||||
resource: request.resource.clone(),
|
||||
lock_type: request.lock_type,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
"Lock acquisition failed on remote server".to_string(),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse> {
|
||||
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
info!("remote acquire_shared for {}", request.resource);
|
||||
let args = LockArgs::from_request(&request, true);
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
let resp = client
|
||||
.r_lock(req)
|
||||
.await
|
||||
.map_err(|e| LockError::internal(e.to_string()))?
|
||||
.into_inner();
|
||||
|
||||
// Check for explicit error first
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
Ok(LockResponse::success(
|
||||
LockInfo {
|
||||
id: LockId::new_deterministic(&request.resource),
|
||||
resource: request.resource,
|
||||
lock_type: request.lock_type,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner,
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.timeout,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata,
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
|
||||
// Check if the lock acquisition was successful
|
||||
if resp.success {
|
||||
// Save the lock information for later release
|
||||
let mut locks = self.active_locks.write().await;
|
||||
locks.insert(request.lock_id.clone(), request.owner.clone());
|
||||
|
||||
Ok(LockResponse::success(
|
||||
LockInfo {
|
||||
id: request.lock_id.clone(),
|
||||
resource: request.resource.clone(),
|
||||
lock_type: request.lock_type,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: request.owner.clone(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + request.ttl,
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: request.metadata.clone(),
|
||||
priority: request.priority,
|
||||
wait_start_time: None,
|
||||
},
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
} else {
|
||||
// Lock acquisition failed
|
||||
Ok(LockResponse::failure(
|
||||
"Shared lock acquisition failed on remote server".to_string(),
|
||||
std::time::Duration::ZERO,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote release for {}", lock_id);
|
||||
let args = LockArgs::from_lock_id(lock_id);
|
||||
|
||||
// Get the original owner for this lock
|
||||
let owner = {
|
||||
let locks = self.active_locks.read().await;
|
||||
locks.get(lock_id).cloned().unwrap_or_else(|| "remote".to_string())
|
||||
};
|
||||
|
||||
let unlock_request = self.create_unlock_request(lock_id, &owner);
|
||||
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
|
||||
// Try UnLock first (for exclusive locks)
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
|
||||
args: request_string.clone(),
|
||||
});
|
||||
let resp = client
|
||||
.un_lock(req)
|
||||
.await
|
||||
.map_err(|e| LockError::internal(e.to_string()))?
|
||||
.into_inner();
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
let resp = client.un_lock(req).await;
|
||||
|
||||
let success = if resp.is_err() {
|
||||
// If that fails, try RUnLock (for shared locks)
|
||||
let req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
let resp = client
|
||||
.r_un_lock(req)
|
||||
.await
|
||||
.map_err(|e| LockError::internal(e.to_string()))?
|
||||
.into_inner();
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
resp.success
|
||||
} else {
|
||||
let resp = resp.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
|
||||
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
resp.success
|
||||
};
|
||||
|
||||
// Remove the lock from our tracking if successful
|
||||
if success {
|
||||
let mut locks = self.active_locks.write().await;
|
||||
locks.remove(lock_id);
|
||||
}
|
||||
Ok(resp.success)
|
||||
|
||||
Ok(success)
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote refresh for {}", lock_id);
|
||||
let args = LockArgs::from_lock_id(lock_id);
|
||||
let refresh_request = self.create_unlock_request(lock_id, "remote");
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
|
||||
args: serde_json::to_string(&refresh_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
let resp = client
|
||||
.refresh(req)
|
||||
@@ -187,12 +258,13 @@ impl LockClient for RemoteClient {
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let args = LockArgs::from_lock_id(lock_id);
|
||||
let force_request = self.create_unlock_request(lock_id, "remote");
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
|
||||
args: serde_json::to_string(&force_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
let resp = client
|
||||
.force_un_lock(req)
|
||||
@@ -205,14 +277,93 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn check_status(&self, _lock_id: &LockId) -> Result<Option<LockInfo>> {
|
||||
// TODO: Implement remote status query
|
||||
Ok(None)
|
||||
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
|
||||
info!("remote check_status for {}", lock_id);
|
||||
|
||||
// Since there's no direct status query in the gRPC service,
|
||||
// we attempt a non-blocking lock acquisition to check if the resource is available
|
||||
let status_request = self.create_unlock_request(lock_id, "remote");
|
||||
let mut client = node_service_time_out_client(&self.addr)
|
||||
.await
|
||||
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = client.lock(req).await;
|
||||
|
||||
match resp {
|
||||
Ok(response) => {
|
||||
let resp = response.into_inner();
|
||||
if resp.success {
|
||||
// If we successfully acquired the lock, the resource was free
|
||||
// Immediately release it
|
||||
let release_req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
let _ = client.un_lock(release_req).await; // Best effort release
|
||||
|
||||
// Return None since no one was holding the lock
|
||||
Ok(None)
|
||||
} else {
|
||||
// Lock acquisition failed, meaning someone is holding it
|
||||
// We can't determine the exact details remotely, so return a generic status
|
||||
Ok(Some(LockInfo {
|
||||
id: lock_id.clone(),
|
||||
resource: lock_id.as_str().to_string(),
|
||||
lock_type: crate::types::LockType::Exclusive, // We can't know the exact type
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: "unknown".to_string(), // Remote client can't determine owner
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: crate::types::LockMetadata::default(),
|
||||
priority: crate::types::LockPriority::Normal,
|
||||
wait_start_time: None,
|
||||
}))
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
// Communication error or lock is held
|
||||
Ok(Some(LockInfo {
|
||||
id: lock_id.clone(),
|
||||
resource: lock_id.as_str().to_string(),
|
||||
lock_type: crate::types::LockType::Exclusive,
|
||||
status: crate::types::LockStatus::Acquired,
|
||||
owner: "unknown".to_string(),
|
||||
acquired_at: std::time::SystemTime::now(),
|
||||
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
|
||||
last_refreshed: std::time::SystemTime::now(),
|
||||
metadata: crate::types::LockMetadata::default(),
|
||||
priority: crate::types::LockPriority::Normal,
|
||||
wait_start_time: None,
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> Result<LockStats> {
|
||||
// TODO: Implement remote statistics
|
||||
Ok(LockStats::default())
|
||||
info!("remote get_stats from {}", self.addr);
|
||||
|
||||
// Since there's no direct statistics endpoint in the gRPC service,
|
||||
// we return basic stats indicating this is a remote client
|
||||
let stats = LockStats {
|
||||
last_updated: std::time::SystemTime::now(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// We could potentially enhance this by:
|
||||
// 1. Keeping local counters of operations performed
|
||||
// 2. Adding a stats gRPC method to the service
|
||||
// 3. Querying server health endpoints
|
||||
|
||||
// For now, return minimal stats indicating remote connectivity
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
@@ -220,7 +371,30 @@ impl LockClient for RemoteClient {
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
true
|
||||
// Use Ping interface to test if remote service is online
|
||||
let mut client = match node_service_time_out_client(&self.addr).await {
|
||||
Ok(client) => client,
|
||||
Err(_) => {
|
||||
info!("remote client {} connection failed", self.addr);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
let ping_req = Request::new(PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::new(),
|
||||
});
|
||||
|
||||
match client.ping(ping_req).await {
|
||||
Ok(_) => {
|
||||
info!("remote client {} is online", self.addr);
|
||||
true
|
||||
}
|
||||
Err(_) => {
|
||||
info!("remote client {} ping failed", self.addr);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
|
||||
Reference in New Issue
Block a user