gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-25 13:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

tmp

Browse Source 
Signed-off-by: root <root@PC.localdomain>
This commit is contained in:
root
2025-07-06 21:52:28 +08:00
committed by junxiang Mu
parent 4ccdeb9d2a
commit dc44cde081
24 changed files with 3308 additions and 4772 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/lock/src/client/local.rs
+124 -763
View File
@@ -12,279 +12,51 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use dashmap::DashMap;
use std::sync::Arc;
use tokio::sync::Mutex;
use crate::{
client::LockClient,
deadlock_detector::DeadlockDetector,
error::Result,
types::{
DeadlockDetectionResult, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType,
WaitQueueItem,
},
local::LocalLockMap,
types::{LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockType},
};
/// Local lock client
///
/// Uses global singleton LocalLockMap to ensure all clients access the same lock instance
#[derive(Debug, Clone)]
pub struct LocalClient {
/// Lock storage
locks: Arc<DashMap<String, LockInfo>>,
/// Deadlock detector
deadlock_detector: Arc<Mutex<DeadlockDetector>>,
/// Wait queues: resource -> wait queue
wait_queues: Arc<DashMap<String, Vec<WaitQueueItem>>>,
/// Statistics
stats: Arc<Mutex<LockStats>>,
}
pub struct LocalClient;
impl LocalClient {
/// Create new local client
pub fn new() -> Self {
Self {
locks: Arc::new(DashMap::new()),
deadlock_detector: Arc::new(Mutex::new(DeadlockDetector::new())),
wait_queues: Arc::new(DashMap::new()),
stats: Arc::new(Mutex::new(LockStats::default())),
Self
}
/// Get global lock map instance
pub fn get_lock_map(&self) -> Arc<LocalLockMap> {
crate::get_global_lock_map()
}
/// Convert LockRequest to batch operation
async fn request_to_batch(&self, request: LockRequest) -> Result<bool> {
let lock_map = self.get_lock_map();
let resources = vec![request.resource];
let timeout = request.timeout;
match request.lock_type {
LockType::Exclusive => lock_map.lock_batch(&resources, &request.owner, timeout, None).await,
LockType::Shared => lock_map.rlock_batch(&resources, &request.owner, timeout, None).await,
}
}
/// Acquire lock with priority and deadlock detection
async fn acquire_lock_with_priority(&self, request: LockRequest, lock_type: LockType) -> Result<LockResponse> {
let _start_time = std::time::SystemTime::now();
let lock_key = crate::utils::generate_lock_key(&request.resource, lock_type);
// Check deadlock detection
if request.deadlock_detection {
if let Ok(detection_result) = self.check_deadlock(&request).await {
if detection_result.has_deadlock {
return Ok(LockResponse::failure(
format!("Deadlock detected: {:?}", detection_result.deadlock_cycle),
crate::utils::duration_between(_start_time, std::time::SystemTime::now()),
));
}
}
}
// Atomic check + insert
match self.locks.entry(lock_key) {
dashmap::mapref::entry::Entry::Occupied(mut entry) => {
let existing = entry.get();
if existing.owner != request.owner {
// Add to wait queue
let wait_item = WaitQueueItem::new(&request.owner, lock_type, request.priority);
self.add_to_wait_queue(&request.resource, wait_item).await;
// Update deadlock detector
self.update_deadlock_detector(&request, &existing.owner).await;
// Check wait timeout
if let Some(wait_timeout) = request.wait_timeout {
if crate::utils::duration_between(_start_time, std::time::SystemTime::now()) > wait_timeout {
self.remove_from_wait_queue(&request.resource, &request.owner).await;
return Ok(LockResponse::failure(
"Wait timeout exceeded".to_string(),
crate::utils::duration_between(_start_time, std::time::SystemTime::now()),
));
}
}
let position = self.get_wait_position(&request.resource, &request.owner).await;
return Ok(LockResponse::waiting(
crate::utils::duration_between(_start_time, std::time::SystemTime::now()),
position,
));
}
// Update lock info (same owner can re-acquire)
let mut lock_info = existing.clone();
lock_info.last_refreshed = std::time::SystemTime::now();
lock_info.expires_at = std::time::SystemTime::now() + request.timeout;
lock_info.priority = request.priority;
entry.insert(lock_info.clone());
Ok(LockResponse::success(
lock_info,
crate::utils::duration_between(_start_time, std::time::SystemTime::now()),
))
}
dashmap::mapref::entry::Entry::Vacant(entry) => {
// Insert new lock
let lock_info = LockInfo {
id: LockId::new(),
resource: request.resource.clone(),
lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.timeout,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
entry.insert(lock_info.clone());
// Update deadlock detector
self.update_deadlock_detector(&request, "").await;
Ok(LockResponse::success(
lock_info,
crate::utils::duration_between(_start_time, std::time::SystemTime::now()),
))
}
}
}
/// Check for deadlock
async fn check_deadlock(&self, _request: &LockRequest) -> Result<DeadlockDetectionResult> {
let mut detector = self.deadlock_detector.lock().await;
Ok(detector.detect_deadlock())
}
/// Update deadlock detector
async fn update_deadlock_detector(&self, request: &LockRequest, current_owner: &str) {
let mut detector = self.deadlock_detector.lock().await;
if !current_owner.is_empty() {
// Add wait relationship
detector.add_wait_relationship(
&request.owner,
&request.resource,
vec![], // TODO: Get currently held resources
request.priority,
);
}
// Update resource holder
detector.update_resource_holder(&request.resource, &request.owner);
}
/// Add to wait queue
async fn add_to_wait_queue(&self, resource: &str, item: WaitQueueItem) {
let mut queue = self.wait_queues.entry(resource.to_string()).or_default();
queue.push(item);
// Sort by priority
queue.sort_by(|a, b| b.priority.cmp(&a.priority));
}
/// Remove from wait queue
async fn remove_from_wait_queue(&self, resource: &str, owner: &str) {
if let Some(mut queue) = self.wait_queues.get_mut(resource) {
queue.retain(|item| item.owner != owner);
}
}
/// Get wait position
async fn get_wait_position(&self, resource: &str, owner: &str) -> usize {
if let Some(queue) = self.wait_queues.get(resource) {
for (i, item) in queue.iter().enumerate() {
if item.owner == owner {
return i;
}
}
}
0
}
/// Process wait queue
async fn process_wait_queue(&self, resource: &str) {
// Simple implementation to avoid never_loop warning
if let Some(mut queue) = self.wait_queues.get_mut(resource) {
if !queue.is_empty() {
let _next_item = queue.remove(0);
// TODO: Process next item in queue
}
}
}
/// Acquire multiple locks atomically
pub async fn acquire_multiple_atomic(&self, requests: Vec<LockRequest>) -> Result<Vec<LockResponse>> {
let mut responses = Vec::new();
let mut acquired_locks = Vec::new();
for request in requests {
match self.acquire_lock_with_priority(request.clone(), LockType::Exclusive).await {
Ok(response) => {
if response.is_success() {
acquired_locks.push(request.resource.clone());
}
responses.push(response);
}
Err(e) => {
// Rollback acquired locks
for resource in acquired_locks {
let _ = self.force_release_by_resource(&resource).await;
}
return Err(e);
}
}
}
Ok(responses)
}
/// Release multiple locks atomically
pub async fn release_multiple_atomic(&self, lock_ids: Vec<LockId>) -> Result<Vec<bool>> {
let mut results = Vec::new();
for lock_id in lock_ids {
results.push(self.release(&lock_id).await?);
}
Ok(results)
}
/// Force release by resource
async fn force_release_by_resource(&self, resource: &str) -> Result<bool> {
let lock_key = crate::utils::generate_lock_key(resource, LockType::Exclusive);
if let Some((_, lock_info)) = self.locks.remove(&lock_key) {
// Update statistics
let mut stats = self.stats.lock().await;
stats.total_releases += 1;
stats.total_hold_time += crate::utils::duration_between(lock_info.acquired_at, std::time::SystemTime::now());
Ok(true)
} else {
Ok(false)
}
}
/// Check multiple lock status
pub async fn check_multiple_status(&self, lock_ids: Vec<LockId>) -> Result<Vec<Option<LockInfo>>> {
let mut results = Vec::new();
for lock_id in lock_ids {
results.push(self.check_status(&lock_id).await?);
}
Ok(results)
}
/// Refresh multiple locks atomically
pub async fn refresh_multiple_atomic(&self, lock_ids: Vec<LockId>) -> Result<Vec<bool>> {
let mut results = Vec::new();
for lock_id in lock_ids {
results.push(self.refresh(&lock_id).await?);
}
Ok(results)
}
/// Get deadlock statistics
pub async fn get_deadlock_stats(&self) -> Result<(usize, std::time::SystemTime)> {
let detector = self.deadlock_detector.lock().await;
let (count, time) = detector.get_stats();
Ok((count, time))
}
/// Detect deadlock
pub async fn detect_deadlock(&self) -> Result<DeadlockDetectionResult> {
let mut detector = self.deadlock_detector.lock().await;
Ok(detector.detect_deadlock())
}
/// Cleanup expired waits
pub async fn cleanup_expired_waits(&self, max_wait_time: std::time::Duration) {
let now = std::time::SystemTime::now();
for mut queue in self.wait_queues.iter_mut() {
queue.retain(|item| now.duration_since(item.wait_start_time).unwrap_or_default() <= max_wait_time);
}
/// Convert LockId to resource for release
async fn lock_id_to_batch_release(&self, lock_id: &LockId) -> Result<()> {
let lock_map = self.get_lock_map();
// For simplicity, we'll use the lock_id as resource name
// In a real implementation, you might want to maintain a mapping
let resources = vec![lock_id.as_str().to_string()];
lock_map.unlock_batch(&resources, "unknown").await
}
}
@@ -295,62 +67,71 @@ impl Default for LocalClient {
}
#[async_trait::async_trait]
impl super::LockClient for LocalClient {
impl LockClient for LocalClient {
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse> {
self.acquire_lock_with_priority(request, LockType::Exclusive).await
let lock_map = self.get_lock_map();
let success = lock_map
.lock_with_ttl_id(&request.resource, &request.owner, request.timeout, None)
.await
.map_err(|e| crate::error::LockError::internal(format!("Lock acquisition failed: {e}")))?;
if success {
let lock_info = LockInfo {
id: crate::types::LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.timeout,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO))
}
}
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse> {
self.acquire_lock_with_priority(request, LockType::Shared).await
let lock_map = self.get_lock_map();
let success = lock_map
.rlock_with_ttl_id(&request.resource, &request.owner, request.timeout, None)
.await
.map_err(|e| crate::error::LockError::internal(format!("Shared lock acquisition failed: {e}")))?;
if success {
let lock_info = LockInfo {
id: crate::types::LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: LockType::Shared,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.timeout,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO))
}
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
let _start_time = std::time::SystemTime::now();
// Find and remove the lock
let mut found = false;
let mut lock_info_opt = None;
for entry in self.locks.iter() {
if entry.id == *lock_id {
lock_info_opt = Some(entry.clone());
found = true;
break;
}
}
if found {
let lock_key = crate::utils::generate_lock_key(
&lock_info_opt.as_ref().unwrap().resource,
lock_info_opt.as_ref().unwrap().lock_type,
);
if let Some((_, lock_info)) = self.locks.remove(&lock_key) {
// Update statistics
let mut stats = self.stats.lock().await;
stats.total_releases += 1;
stats.total_hold_time += crate::utils::duration_between(lock_info.acquired_at, std::time::SystemTime::now());
// Process wait queue
self.process_wait_queue(&lock_info.resource).await;
Ok(true)
} else {
Ok(false)
}
} else {
Ok(false)
}
let lock_map = self.get_lock_map();
lock_map
.unlock_by_id(lock_id)
.await
.map_err(|e| crate::error::LockError::internal(format!("Release failed: {e}")))?;
Ok(true)
}
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
for mut entry in self.locks.iter_mut() {
if entry.id == *lock_id {
entry.last_refreshed = std::time::SystemTime::now();
entry.expires_at = std::time::SystemTime::now() + std::time::Duration::from_secs(30);
return Ok(true);
}
}
Ok(false)
async fn refresh(&self, _lock_id: &LockId) -> Result<bool> {
// For local locks, refresh is not needed as they don't expire automatically
Ok(true)
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
@@ -358,44 +139,41 @@ impl super::LockClient for LocalClient {
}
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
for entry in self.locks.iter() {
if entry.id == *lock_id {
// Check if lock has expired
if entry.expires_at < std::time::SystemTime::now() {
// Lock has expired, remove it
let lock_key = crate::utils::generate_lock_key(&entry.resource, entry.lock_type);
let _ = self.locks.remove(&lock_key);
return Ok(None);
}
return Ok(Some(entry.clone()));
let lock_map = self.get_lock_map();
if let Some((resource, owner)) = lock_map.lockid_map.get(lock_id).map(|v| v.clone()) {
let is_locked = lock_map.is_locked(&resource).await;
if is_locked {
Ok(Some(LockInfo {
id: lock_id.clone(),
resource,
lock_type: LockType::Exclusive, // 这里可进一步完善
status: crate::types::LockStatus::Acquired,
owner,
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
} else {
Ok(None)
}
} else {
Ok(None)
}
Ok(None)
}
async fn get_stats(&self) -> Result<LockStats> {
let mut stats = self.stats.lock().await;
stats.total_locks = self.locks.len();
stats.total_wait_queues = self.wait_queues.len();
// Calculate average hold time
if stats.total_releases > 0 {
stats.average_hold_time =
std::time::Duration::from_secs(stats.total_hold_time.as_secs() / stats.total_releases as u64);
}
Ok(stats.clone())
Ok(LockStats::default())
}
async fn close(&self) -> Result<()> {
// Cleanup all locks
self.locks.clear();
self.wait_queues.clear();
Ok(())
}
async fn is_online(&self) -> bool {
true // Local client is always online
true
}
async fn is_local(&self) -> bool {
@@ -406,21 +184,13 @@ impl super::LockClient for LocalClient {
#[cfg(test)]
mod tests {
use super::*;
use crate::types::{LockMetadata, LockPriority, LockType};
use crate::types::LockType;
#[tokio::test]
async fn test_local_client_acquire_exclusive() {
let client = LocalClient::new();
let request = LockRequest {
resource: "test_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "test_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let request =
LockRequest::new("test-resource", LockType::Exclusive, "test-owner").with_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(request).await.unwrap();
assert!(response.is_success());
@@ -429,16 +199,8 @@ mod tests {
#[tokio::test]
async fn test_local_client_acquire_shared() {
let client = LocalClient::new();
let request = LockRequest {
resource: "test_resource".to_string(),
lock_type: LockType::Shared,
owner: "test_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let request =
LockRequest::new("test-resource", LockType::Shared, "test-owner").with_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_shared(request).await.unwrap();
assert!(response.is_success());
@@ -447,426 +209,25 @@ mod tests {
#[tokio::test]
async fn test_local_client_release() {
let client = LocalClient::new();
let request = LockRequest {
resource: "test_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "test_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(request).await.unwrap();
assert!(response.is_success());
let lock_id = &response.lock_info().unwrap().id;
let result = client.release(lock_id).await.unwrap();
assert!(result);
}
#[tokio::test]
async fn test_local_client_concurrent_access() {
let client = Arc::new(LocalClient::new());
let mut handles = vec![];
for i in 0..10 {
let client_clone = client.clone();
let handle = tokio::spawn(async move {
let request = LockRequest {
resource: "concurrent_resource".to_string(),
lock_type: LockType::Exclusive,
owner: format!("owner_{i}"),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client_clone.acquire_exclusive(request).await.unwrap();
if response.is_success() {
let lock_id = &response.lock_info().unwrap().id;
let _ = client_clone.release(lock_id).await;
}
});
handles.push(handle);
}
for handle in handles {
handle.await.unwrap();
}
}
#[tokio::test]
async fn test_dashmap_performance() {
let client = LocalClient::new();
let start_time = std::time::Instant::now();
// Simulate high concurrent access
let mut handles = vec![];
for i in 0..100 {
let client_clone = Arc::new(client.clone());
let handle = tokio::spawn(async move {
let request = LockRequest {
resource: format!("resource_{i}"),
lock_type: LockType::Exclusive,
owner: format!("owner_{i}"),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client_clone.acquire_exclusive(request).await.unwrap();
if response.is_success() {
let lock_id = &response.lock_info().unwrap().id;
let _ = client_clone.release(lock_id).await;
}
});
handles.push(handle);
}
for handle in handles {
handle.await.unwrap();
}
let duration = start_time.elapsed();
println!("DashMap performance test completed in {duration:?}");
assert!(duration < std::time::Duration::from_secs(5));
}
#[tokio::test]
async fn test_atomic_operations() {
let client = LocalClient::new();
let request = LockRequest {
resource: "atomic_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "test_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
// Test atomic acquire
let response = client.acquire_exclusive(request).await.unwrap();
assert!(response.is_success());
// Test concurrent access to same resource
let client_clone = Arc::new(client);
let mut handles = vec![];
for i in 0..5 {
let client_clone = client_clone.clone();
let handle = tokio::spawn(async move {
let request = LockRequest {
resource: "atomic_resource".to_string(),
lock_type: LockType::Exclusive,
owner: format!("owner_{i}"),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client_clone.acquire_exclusive(request).await.unwrap();
response.is_waiting() // Should be waiting due to atomic operation
});
handles.push(handle);
}
for handle in handles {
let result = handle.await.unwrap();
assert!(result);
}
}
#[tokio::test]
async fn test_batch_atomic_operations() {
let client = LocalClient::new();
let requests = vec![
LockRequest {
resource: "batch_resource_1".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
LockRequest {
resource: "batch_resource_2".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
];
let responses = client.acquire_multiple_atomic(requests).await.unwrap();
assert_eq!(responses.len(), 2);
assert!(responses[0].is_success());
assert!(responses[1].is_success());
}
#[tokio::test]
async fn test_batch_atomic_rollback() {
let client = LocalClient::new();
// First acquire a lock
let first_request = LockRequest {
resource: "rollback_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(first_request).await.unwrap();
let request =
LockRequest::new("test-resource", LockType::Exclusive, "test-owner").with_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(request).await.unwrap();
assert!(response.is_success());
// Try to acquire same resource in batch (should fail and rollback)
let requests = vec![
LockRequest {
resource: "rollback_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_2".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
LockRequest {
resource: "rollback_resource_2".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_2".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
];
let responses = client.acquire_multiple_atomic(requests).await.unwrap();
assert_eq!(responses.len(), 2);
assert!(responses[0].is_waiting()); // Should be waiting
assert!(responses[1].is_success()); // Second should succeed
}
#[tokio::test]
async fn test_concurrent_atomic_operations() {
let client = Arc::new(LocalClient::new());
let mut handles = vec![];
for i in 0..10 {
let client_clone = client.clone();
let handle = tokio::spawn(async move {
let requests = vec![
LockRequest {
resource: format!("concurrent_batch_{i}"),
lock_type: LockType::Exclusive,
owner: format!("owner_{i}"),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
LockRequest {
resource: format!("concurrent_batch_{i}_2"),
lock_type: LockType::Exclusive,
owner: format!("owner_{i}"),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
},
];
let responses = client_clone.acquire_multiple_atomic(requests).await.unwrap();
assert_eq!(responses.len(), 2);
// Release locks
for response in responses {
if response.is_success() {
let lock_id = &response.lock_info().unwrap().id;
let _ = client_clone.release(lock_id).await;
}
}
});
handles.push(handle);
}
for handle in handles {
handle.await.unwrap();
// Get the lock ID from the response
if let Some(lock_info) = response.lock_info() {
let result = client.release(&lock_info.id).await.unwrap();
assert!(result);
} else {
panic!("No lock info in response");
}
}
#[tokio::test]
async fn test_priority_upgrade() {
async fn test_local_client_is_local() {
let client = LocalClient::new();
// Acquire lock with normal priority
let normal_request = LockRequest {
resource: "priority_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "normal_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(normal_request).await.unwrap();
assert!(response.is_success());
// Try to acquire with high priority (should be waiting)
let high_request = LockRequest {
resource: "priority_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "high_owner".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::High,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(high_request.clone()).await.unwrap();
assert!(response.is_waiting());
// Release normal priority lock
let lock_id = &response.lock_info().unwrap().id;
let _ = client.release(lock_id).await;
// High priority should now acquire
let response = client.acquire_exclusive(high_request).await.unwrap();
assert!(response.is_success());
}
#[tokio::test]
async fn test_deadlock_detection() {
let client = LocalClient::new();
// Create a potential deadlock scenario
let request1 = LockRequest {
resource: "resource_a".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: true,
metadata: LockMetadata::default(),
};
let request2 = LockRequest {
resource: "resource_b".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_2".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: true,
metadata: LockMetadata::default(),
};
// Acquire first lock
let response1 = client.acquire_exclusive(request1).await.unwrap();
assert!(response1.is_success());
// Acquire second lock
let response2 = client.acquire_exclusive(request2).await.unwrap();
assert!(response2.is_success());
// Try to create deadlock
let deadlock_request1 = LockRequest {
resource: "resource_b".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: true,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(deadlock_request1).await.unwrap();
assert!(response.is_waiting() || response.is_failure());
}
#[tokio::test]
async fn test_wait_timeout() {
let client = LocalClient::new();
// Acquire lock
let request1 = LockRequest {
resource: "timeout_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_1".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: None,
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let response = client.acquire_exclusive(request1).await.unwrap();
assert!(response.is_success());
// Try to acquire with short wait timeout
let request2 = LockRequest {
resource: "timeout_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "owner_2".to_string(),
timeout: std::time::Duration::from_secs(30),
wait_timeout: Some(std::time::Duration::from_millis(100)),
priority: LockPriority::Normal,
deadlock_detection: false,
metadata: LockMetadata::default(),
};
let start_time = std::time::Instant::now();
let response = client.acquire_exclusive(request2).await.unwrap();
let duration = start_time.elapsed();
assert!(response.is_failure() || response.is_waiting());
assert!(duration < std::time::Duration::from_secs(1));
}
#[tokio::test]
async fn test_deadlock_stats() {
let client = LocalClient::new();
let (count, last_time) = client.get_deadlock_stats().await.unwrap();
assert_eq!(count, 0);
assert!(last_time < std::time::SystemTime::now());
}
#[tokio::test]
async fn test_cleanup_expired_waits() {
let client = LocalClient::new();
// Add some wait items
let wait_item = WaitQueueItem::new("test_owner", LockType::Exclusive, LockPriority::Normal);
client.add_to_wait_queue("test_resource", wait_item).await;
// Cleanup with short timeout
client.cleanup_expired_waits(std::time::Duration::from_millis(1)).await;
// Wait queue should be empty
let position = client.get_wait_position("test_resource", "test_owner").await;
assert_eq!(position, 0);
assert!(client.is_local().await);
}
}
crates/lock/src/client/mod.rs
+9 -1
View File
@@ -25,13 +25,21 @@ use crate::{
/// Lock client trait
#[async_trait]
pub trait LockClient: Send + Sync {
pub trait LockClient: Send + Sync + std::fmt::Debug {
/// Acquire exclusive lock
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse>;
/// Acquire shared lock
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse>;
/// Acquire lock (generic method)
async fn acquire_lock(&self, request: LockRequest) -> Result<LockResponse> {
match request.lock_type {
crate::types::LockType::Exclusive => self.acquire_exclusive(request).await,
crate::types::LockType::Shared => self.acquire_shared(request).await,
}
}
/// Release lock
async fn release(&self, lock_id: &LockId) -> Result<bool>;
crates/lock/src/client/remote.rs
+125 -199
View File
@@ -14,284 +14,210 @@
use async_trait::async_trait;
use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
use serde::{Deserialize, Serialize};
use tonic::Request;
use tracing::info;
use crate::{
error::{LockError, Result},
lock_args::LockArgs,
types::{LockId, LockInfo, LockRequest, LockResponse, LockStats},
};
/// Remote lock client
use super::LockClient;
/// RPC lock arguments for gRPC communication
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockArgs {
pub uid: String,
pub resources: Vec<String>,
pub owner: String,
pub source: String,
pub quorum: u32,
}
impl LockArgs {
fn from_request(request: &LockRequest, _is_shared: bool) -> Self {
Self {
uid: request.metadata.operation_id.clone().unwrap_or_default(),
resources: vec![request.resource.clone()],
owner: request.owner.clone(),
source: "remote".to_string(),
quorum: 1,
}
}
fn from_lock_id(lock_id: &LockId) -> Self {
Self {
uid: lock_id.as_str().to_string(),
resources: vec![lock_id.as_str().to_string()],
owner: "remote".to_string(),
source: "remote".to_string(),
quorum: 1,
}
}
}
/// Remote lock client implementation
#[derive(Debug, Clone)]
pub struct RemoteClient {
addr: String,
}
impl RemoteClient {
/// Create new remote client from endpoint string (for trait兼容)
pub fn new(endpoint: String) -> Self {
Self { addr: endpoint }
}
/// Create new remote client from url::Url（兼容 namespace/distributed 场景）
pub fn from_url(url: url::Url) -> Self {
let addr = format!("{}://{}:{}", url.scheme(), url.host_str().unwrap(), url.port().unwrap());
Self { addr }
}
}
// 辅助方法：从 LockRequest 创建 LockArgs
impl LockArgs {
fn from_request(request: &LockRequest, _is_shared: bool) -> Self {
Self {
uid: uuid::Uuid::new_v4().to_string(),
resources: vec![request.resource.clone()],
owner: request.owner.clone(),
source: "remote_client".to_string(),
quorum: 1,
}
}
fn from_lock_id(lock_id: &LockId) -> Self {
Self {
uid: lock_id.to_string(),
resources: vec![],
owner: "remote_client".to_string(),
source: "remote_client".to_string(),
quorum: 1,
}
pub fn from_url(url: url::Url) -> Self {
Self { addr: url.to_string() }
}
}
#[async_trait]
impl super::LockClient for RemoteClient {
impl LockClient for RemoteClient {
async fn acquire_exclusive(&self, request: LockRequest) -> Result<LockResponse> {
info!("remote acquire_exclusive");
info!("remote acquire_exclusive for {}", request.resource);
let args = LockArgs::from_request(&request, false);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))? });
let resp = client.lock(req).await.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
});
let resp = client
.lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(LockResponse {
success: resp.success,
lock_info: None, // 可扩展: 解析resp内容
error: None,
wait_time: std::time::Duration::ZERO,
position_in_queue: None,
})
Ok(LockResponse::success(
LockInfo {
id: LockId::new_deterministic(&request.resource),
resource: request.resource,
lock_type: request.lock_type,
status: crate::types::LockStatus::Acquired,
owner: request.owner,
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.timeout,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata,
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
}
async fn acquire_shared(&self, request: LockRequest) -> Result<LockResponse> {
info!("remote acquire_shared");
info!("remote acquire_shared for {}", request.resource);
let args = LockArgs::from_request(&request, true);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))? });
let resp = client.r_lock(req).await.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
});
let resp = client
.r_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(LockResponse {
success: resp.success,
lock_info: None,
error: None,
wait_time: std::time::Duration::ZERO,
position_in_queue: None,
})
Ok(LockResponse::success(
LockInfo {
id: LockId::new_deterministic(&request.resource),
resource: request.resource,
lock_type: request.lock_type,
status: crate::types::LockStatus::Acquired,
owner: request.owner,
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.timeout,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata,
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote release");
info!("remote release for {}", lock_id);
let args = LockArgs::from_lock_id(lock_id);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))? });
let resp = client.un_lock(req).await.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
});
let resp = client
.un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
info!("remote refresh");
info!("remote refresh for {}", lock_id);
let args = LockArgs::from_lock_id(lock_id);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))? });
let resp = client.refresh(req).await.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
});
let resp = client
.refresh(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release");
info!("remote force_release for {}", lock_id);
let args = LockArgs::from_lock_id(lock_id);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest { args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))? });
let resp = client.force_un_lock(req).await.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?,
});
let resp = client
.force_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn check_status(&self, _lock_id: &LockId) -> Result<Option<LockInfo>> {
// 可扩展: 实现远程状态查询
// TODO: Implement remote status query
Ok(None)
}
async fn get_stats(&self) -> Result<LockStats> {
// 可扩展: 实现远程统计
// TODO: Implement remote statistics
Ok(LockStats::default())
}
async fn close(&self) -> Result<()> {
Ok(())
}
async fn is_online(&self) -> bool {
true
}
async fn is_local(&self) -> bool {
false
}
}
// 同时实现 Locker trait 以兼容现有调用
#[async_trait]
impl crate::Locker for RemoteClient {
async fn lock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote lock");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.lock(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn unlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote unlock");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.un_lock(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn rlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote rlock");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.r_lock(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn runlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote runlock");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.r_un_lock(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn refresh(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote refresh");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.refresh(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn force_unlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote force_unlock");
let args = serde_json::to_string(args).map_err(|e| LockError::internal(format!("Failed to serialize args: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client
.force_un_lock(request)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = response.error_info {
return Err(LockError::internal(error_info));
}
Ok(response.success)
}
async fn close(&self) {}
async fn is_online(&self) -> bool {
true