gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: NamespaceLock (nslock), AHM→Heal Crate, and Lock/Clippy Fixes (#1664)

Browse Source 
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: weisd <2057561+weisd@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
weisd
2026-01-30 13:13:41 +08:00
committed by GitHub
parent 1c085590ca
commit dce117840c
80 changed files with 3787 additions and 16746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/lock/src/client/local.rs
+73 -239
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use tokio::sync::RwLock;
@@ -21,23 +22,69 @@ use crate::{
LockResponse, LockStats, LockStatus, LockType, Result,
};
/// Local lock client using FastLock
#[derive(Debug, Clone)]
/// Default shard count for guard storage (must be power of 2)
const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
/// Local lock client using FastLock with sharded guard storage for better concurrency
#[derive(Debug)]
pub struct LocalClient {
guard_storage: Arc<RwLock<HashMap<LockId, FastLockGuard>>>,
/// Sharded guard storage to reduce lock contention
guard_storage: Vec<Arc<RwLock<HashMap<LockId, FastLockGuard>>>>,
/// Mask for fast shard index calculation (shard_count - 1)
shard_mask: usize,
/// Optional lock manager (if None, uses global singleton)
manager: Option<Arc<GlobalLockManager>>,
}
impl LocalClient {
/// Create new local client
/// Create new local client with default shard count
pub fn new() -> Self {
Self::with_shard_count(DEFAULT_GUARD_SHARD_COUNT)
}
/// Create new local client with custom shard count
/// Shard count must be a power of 2 for efficient masking
pub fn with_shard_count(shard_count: usize) -> Self {
assert!(shard_count.is_power_of_two(), "Shard count must be power of 2");
let guard_storage: Vec<Arc<RwLock<HashMap<LockId, FastLockGuard>>>> =
(0..shard_count).map(|_| Arc::new(RwLock::new(HashMap::new()))).collect();
Self {
guard_storage: Arc::new(RwLock::new(HashMap::new())),
guard_storage,
shard_mask: shard_count - 1,
manager: None,
}
}
/// Get the global lock manager
/// Create new local client with a specific lock manager
/// This allows simulating multi-node environments where each node has its own lock backend
pub fn with_manager(manager: Arc<GlobalLockManager>) -> Self {
Self {
guard_storage: (0..DEFAULT_GUARD_SHARD_COUNT)
.map(|_| Arc::new(RwLock::new(HashMap::new())))
.collect(),
shard_mask: DEFAULT_GUARD_SHARD_COUNT - 1,
manager: Some(manager),
}
}
/// Get the lock manager (injected manager if available, otherwise global singleton)
pub fn get_lock_manager(&self) -> Arc<GlobalLockManager> {
crate::get_global_lock_manager()
self.manager.clone().unwrap_or_else(crate::get_global_lock_manager)
}
/// Get the shard index for a given lock ID
fn get_shard_index(&self, lock_id: &LockId) -> usize {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
lock_id.hash(&mut hasher);
(hasher.finish() as usize) & self.shard_mask
}
/// Get the shard for a given lock ID
fn get_shard(&self, lock_id: &LockId) -> &Arc<RwLock<HashMap<LockId, FastLockGuard>>> {
let index = self.get_shard_index(lock_id);
&self.guard_storage[index]
}
}
@@ -49,67 +96,30 @@ impl Default for LocalClient {
#[async_trait::async_trait]
impl LockClient for LocalClient {
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
let lock_manager = self.get_lock_manager();
let lock_request = crate::ObjectLockRequest::new_write("", request.resource.clone(), request.owner.clone())
.with_acquire_timeout(request.acquire_timeout);
let lock_request = match request.lock_type {
LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
.with_acquire_timeout(request.acquire_timeout),
LockType::Shared => crate::ObjectLockRequest::new_read(request.resource.clone(), request.owner.clone())
.with_acquire_timeout(request.acquire_timeout),
};
match lock_manager.acquire_lock(lock_request).await {
Ok(guard) => {
let lock_id = LockId::new_deterministic(&request.resource);
let lock_id = LockId::new_unique(&request.resource);
// Store guard for later release
let mut guards = self.guard_storage.write().await;
guards.insert(lock_id.clone(), guard);
{
let shard = self.get_shard(&lock_id);
let mut guards = shard.write().await;
guards.insert(lock_id.clone(), guard);
}
let lock_info = LockInfo {
id: lock_id,
resource: request.resource.clone(),
lock_type: LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
}
Err(crate::fast_lock::LockResult::Timeout) => {
Ok(LockResponse::failure("Lock acquisition timeout", request.acquire_timeout))
}
Err(crate::fast_lock::LockResult::Conflict {
current_owner,
current_mode,
}) => Ok(LockResponse::failure(
format!("Lock conflict: resource held by {current_owner} in {current_mode:?} mode"),
std::time::Duration::ZERO,
)),
Err(crate::fast_lock::LockResult::Acquired) => {
unreachable!("Acquired should not be an error")
}
}
}
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
let lock_manager = self.get_lock_manager();
let lock_request = crate::ObjectLockRequest::new_read("", request.resource.clone(), request.owner.clone())
.with_acquire_timeout(request.acquire_timeout);
match lock_manager.acquire_lock(lock_request).await {
Ok(guard) => {
let lock_id = LockId::new_deterministic(&request.resource);
// Store guard for later release
let mut guards = self.guard_storage.write().await;
guards.insert(lock_id.clone(), guard);
let lock_info = LockInfo {
id: lock_id,
resource: request.resource.clone(),
lock_type: LockType::Shared,
lock_type: request.lock_type,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
@@ -138,7 +148,8 @@ impl LockClient for LocalClient {
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
let mut guards = self.guard_storage.write().await;
let shard = self.get_shard(lock_id);
let mut guards = shard.write().await;
if let Some(guard) = guards.remove(lock_id) {
// Guard automatically releases the lock when dropped
drop(guard);
@@ -159,7 +170,8 @@ impl LockClient for LocalClient {
}
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
let guards = self.guard_storage.read().await;
let shard = self.get_shard(lock_id);
let guards = shard.read().await;
if let Some(guard) = guards.get(lock_id) {
// We have an active guard for this lock
let lock_type = match guard.mode() {
@@ -200,181 +212,3 @@ impl LockClient for LocalClient {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LockType;
#[tokio::test]
async fn test_local_client_acquire_exclusive() {
let client = LocalClient::new();
let resource_name = format!("test-resource-exclusive-{}", uuid::Uuid::new_v4());
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(&request).await.unwrap();
assert!(response.is_success());
// Clean up
if let Some(lock_info) = response.lock_info() {
let _ = client.release(&lock_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_acquire_shared() {
let client = LocalClient::new();
let resource_name = format!("test-resource-shared-{}", uuid::Uuid::new_v4());
let request = LockRequest::new(&resource_name, LockType::Shared, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_shared(&request).await.unwrap();
assert!(response.is_success());
// Clean up
if let Some(lock_info) = response.lock_info() {
let _ = client.release(&lock_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_release() {
let client = LocalClient::new();
let resource_name = format!("test-resource-release-{}", uuid::Uuid::new_v4());
// First acquire a lock
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(&request).await.unwrap();
assert!(response.is_success());
// Get the lock ID from the response
if let Some(lock_info) = response.lock_info() {
let result = client.release(&lock_info.id).await.unwrap();
assert!(result);
} else {
panic!("No lock info in response");
}
}
#[tokio::test]
async fn test_local_client_is_local() {
let client = LocalClient::new();
assert!(client.is_local().await);
}
#[tokio::test]
async fn test_local_client_read_write_lock_exclusion() {
let client = LocalClient::new();
let resource_name = format!("test-resource-exclusion-{}", uuid::Uuid::new_v4());
// First, acquire an exclusive lock
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
assert!(exclusive_response.is_success());
// Try to acquire a shared lock on the same resource - should fail
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
assert!(!shared_response.is_success(), "Shared lock should fail when exclusive lock exists");
// Clean up exclusive lock
if let Some(exclusive_info) = exclusive_response.lock_info() {
let _ = client.release(&exclusive_info.id).await;
}
// Now shared lock should succeed
let shared_request2 = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let shared_response2 = client.acquire_shared(&shared_request2).await.unwrap();
assert!(
shared_response2.is_success(),
"Shared lock should succeed after exclusive lock is released"
);
// Clean up
if let Some(shared_info) = shared_response2.lock_info() {
let _ = client.release(&shared_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_read_write_lock_distinction() {
let client = LocalClient::new();
let resource_name = format!("test-resource-rw-{}", uuid::Uuid::new_v4());
// Test exclusive lock
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
assert!(exclusive_response.is_success());
if let Some(exclusive_info) = exclusive_response.lock_info() {
assert_eq!(exclusive_info.lock_type, LockType::Exclusive);
// Check status should return correct lock type
let status = client.check_status(&exclusive_info.id).await.unwrap();
assert!(status.is_some());
assert_eq!(status.unwrap().lock_type, LockType::Exclusive);
// Release exclusive lock
let result = client.release(&exclusive_info.id).await.unwrap();
assert!(result);
}
// Test shared lock
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
assert!(shared_response.is_success());
if let Some(shared_info) = shared_response.lock_info() {
assert_eq!(shared_info.lock_type, LockType::Shared);
// Check status should return correct lock type
let status = client.check_status(&shared_info.id).await.unwrap();
assert!(status.is_some());
assert_eq!(status.unwrap().lock_type, LockType::Shared);
// Release shared lock
let result = client.release(&shared_info.id).await.unwrap();
assert!(result);
}
}
#[tokio::test]
async fn test_multiple_local_clients_exclusive_mutex() {
let client1 = LocalClient::new();
let client2 = LocalClient::new();
let resource_name = format!("test-multi-client-mutex-{}", uuid::Uuid::new_v4());
// client1 acquire exclusive lock
let req1 = LockRequest::new(&resource_name, LockType::Exclusive, "owner1")
.with_acquire_timeout(std::time::Duration::from_millis(50));
let resp1 = client1.acquire_exclusive(&req1).await.unwrap();
assert!(resp1.is_success(), "client1 should acquire exclusive lock");
// client2 try to acquire exclusive lock, should fail
let req2 = LockRequest::new(&resource_name, LockType::Exclusive, "owner2")
.with_acquire_timeout(std::time::Duration::from_millis(50));
let resp2 = client2.acquire_exclusive(&req2).await.unwrap();
assert!(!resp2.is_success(), "client2 should not acquire exclusive lock while client1 holds it");
// client1 release lock
if let Some(lock_info) = resp1.lock_info() {
let _ = client1.release(&lock_info.id).await;
}
// client2 try again, should succeed
let resp3 = client2.acquire_exclusive(&req2).await.unwrap();
assert!(resp3.is_success(), "client2 should acquire exclusive lock after client1 releases it");
// clean up
if let Some(lock_info) = resp3.lock_info() {
let _ = client2.release(&lock_info.id).await;
}
}
}
crates/lock/src/client/mod.rs
+2 -46
View File
@@ -15,26 +15,15 @@
pub mod local;
// pub mod remote;
use crate::{LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, Result};
use crate::{LockId, LockInfo, LockRequest, LockResponse, LockStats, Result};
use async_trait::async_trait;
use std::sync::Arc;
/// Lock client trait
#[async_trait]
pub trait LockClient: Send + Sync + std::fmt::Debug {
/// Acquire exclusive lock
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse>;
/// Acquire shared lock
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse>;
/// Acquire lock (generic method)
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
match request.lock_type {
LockType::Exclusive => self.acquire_exclusive(request).await,
LockType::Shared => self.acquire_shared(request).await,
}
}
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse>;
/// Release lock
async fn release(&self, lock_id: &LockId) -> Result<bool>;
@@ -75,36 +64,3 @@ impl ClientFactory {
// Arc::new(remote::RemoteClient::new(endpoint))
// }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::LockType;
#[tokio::test]
async fn test_local_client_basic_operations() {
let client = ClientFactory::create_local();
let request = LockRequest::new("test-resource", LockType::Exclusive, "test-owner");
// Test lock acquisition
let response = client.acquire_exclusive(&request).await;
assert!(response.is_ok());
if let Ok(response) = response
&& response.success
{
let lock_info = response.lock_info.unwrap();
// Test status check
let status = client.check_status(&lock_info.id).await;
assert!(status.is_ok());
assert!(status.unwrap().is_some());
// Test lock release
let released = client.release(&lock_info.id).await;
assert!(released.is_ok());
assert!(released.unwrap());
}
}
}
crates/lock/src/distributed_lock.rs
+367
View File
@@ -0,0 +1,367 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
ObjectKey,
client::LockClient,
error::{LockError, Result},
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
};
use std::sync::{Arc, LazyLock};
use std::time::Duration;
use tokio::sync::mpsc;
use tracing::warn;
use uuid::Uuid;
/// Generate a new aggregate lock ID for multiple client locks
fn generate_aggregate_lock_id(resource: &ObjectKey) -> LockId {
LockId {
resource: resource.clone(),
uuid: Uuid::new_v4().to_string(),
}
}
#[derive(Debug, Clone)]
struct UnlockJob {
/// Entries to release: each (LockId, client) pair will be released independently.
entries: Vec<(LockId, Arc<dyn LockClient>)>,
}
#[derive(Debug)]
struct UnlockRuntime {
tx: mpsc::Sender<UnlockJob>,
}
// Global unlock runtime with background worker
static UNLOCK_RUNTIME: LazyLock<UnlockRuntime> = LazyLock::new(|| {
// Larger buffer to reduce contention during bursts
let (tx, mut rx) = mpsc::channel::<UnlockJob>(8192);
// Spawn background worker when first used; assumes a Tokio runtime is available
tokio::spawn(async move {
while let Some(job) = rx.recv().await {
// Best-effort release across all (LockId, client) entries.
let mut any_ok = false;
for (lock_id, client) in job.entries.into_iter() {
if client.release(&lock_id).await.unwrap_or(false) {
any_ok = true;
}
}
if !any_ok {
tracing::warn!("DistributedLockGuard background release failed for one or more entries");
} else {
tracing::debug!("DistributedLockGuard background released one or more entries");
}
}
});
UnlockRuntime { tx }
});
/// A RAII guard for distributed locks that releases the lock asynchronously when dropped.
#[derive(Debug)]
pub struct DistributedLockGuard {
/// The public-facing lock id. For multi-client scenarios this is typically
/// an aggregate id; for single-client it is the only id.
lock_id: LockId,
/// All underlying (LockId, client) entries that should be released when the
/// guard is dropped.
entries: Vec<(LockId, Arc<dyn LockClient>)>,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
}
impl DistributedLockGuard {
/// Create a new guard.
///
/// - `lock_id` is the id returned to the caller (`lock_id()`).
/// - `entries` is the full list of underlying (LockId, client) pairs
/// that should be released when this guard is dropped.
pub(crate) fn new(lock_id: LockId, entries: Vec<(LockId, Arc<dyn LockClient>)>) -> Self {
Self {
lock_id,
entries,
disarmed: false,
}
}
/// Get the lock id associated with this guard
pub fn lock_id(&self) -> &LockId {
&self.lock_id
}
/// Manually disarm the guard so dropping it won't release the lock.
/// Call this if you explicitly released the lock elsewhere.
pub fn disarm(&mut self) {
self.disarmed = true;
}
/// Check if the guard has been disarmed (lock already released)
pub fn is_disarmed(&self) -> bool {
self.disarmed
}
/// Manually release the lock early.
/// This sends a release job to the background worker and then disarms the guard
/// to prevent double-release on drop.
/// Returns true if the lock was released (or was already released), false otherwise.
pub fn release(&mut self) -> bool {
if self.disarmed {
// Lock was already released, return true to indicate lock is in released state
return true;
}
let job = UnlockJob {
entries: self.entries.clone(),
};
// Try a non-blocking send to avoid panics
let success = if let Err(err) = UNLOCK_RUNTIME.tx.try_send(job) {
// Channel full or closed; best-effort fallback: spawn a detached task
let entries = self.entries.clone();
tracing::warn!(
"DistributedLockGuard channel send failed ({}), spawning fallback unlock task for {} entries",
err,
entries.len()
);
// If runtime is not available, this will panic; but in RustFS we are inside Tokio contexts.
let handle = tokio::spawn(async move {
let futures_iter = entries
.into_iter()
.map(|(lock_id, client)| async move { client.release(&lock_id).await.unwrap_or(false) });
let _ = futures::future::join_all(futures_iter).await;
});
// Explicitly drop the JoinHandle to acknowledge detaching the task.
drop(handle);
true // Consider it successful even if we had to use fallback
} else {
true
};
// Disarm to prevent double-release on drop
self.disarmed = true;
success
}
}
impl Drop for DistributedLockGuard {
fn drop(&mut self) {
// Call release() to handle the actual release logic
// If already disarmed, release() will return early
// Setting disarmed in release() is harmless here since we're dropping anyway
let _ = self.release();
}
}
/// Distributed lock handler for distributed use cases
/// Uses quorum-based acquisition and aggregate lock ID mapping
#[derive(Debug)]
pub struct DistributedLock {
/// Lock clients for this namespace
clients: Vec<Arc<dyn LockClient>>,
/// Namespace identifier
namespace: String,
/// Quorum size for operations (majority for distributed)
quorum: usize,
}
impl DistributedLock {
/// Create new distributed lock
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
let q = if clients.len() <= 1 {
1
} else {
quorum.clamp(1, clients.len())
};
Self {
clients,
namespace,
quorum: q,
}
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
&self.namespace
}
/// Get resource key for this namespace
pub fn get_resource_key(&self, resource: &ObjectKey) -> String {
format!("{}:{}", self.namespace, resource)
}
/// Get clients (for health check and stats)
pub(crate) fn clients(&self) -> &[Arc<dyn LockClient>] {
&self.clients
}
/// Acquire a lock and return a RAII guard
pub(crate) async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<DistributedLockGuard>> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
let (resp, individual_locks) = self.acquire_lock_quorum(request).await?;
if resp.success {
// Use aggregate lock_id from LockResponse's LockInfo
// The aggregate id is what we expose to callers; individual_locks carries
// the real (LockId, client) pairs that must be released.
let aggregate_lock_id = resp
.lock_info
.as_ref()
.map(|info| info.id.clone())
.unwrap_or_else(|| LockId::new_unique(&request.resource));
Ok(Some(DistributedLockGuard::new(aggregate_lock_id, individual_locks)))
} else {
// Check if it's a timeout or quorum failure
if let Some(error_msg) = &resp.error {
warn!("acquire_lock_quorum error: {}", error_msg);
if error_msg.contains("quorum") {
// This is a quorum failure - return appropriate error
// Extract achieved count from error message or use individual_locks.len()
let achieved = individual_locks.len();
Err(LockError::QuorumNotReached {
required: self.quorum,
achieved,
})
} else if error_msg.contains("timeout") || resp.wait_time >= request.acquire_timeout {
// This is a timeout - return None so caller can convert to timeout error
Ok(None)
} else {
// Other failure - return None for backward compatibility
Ok(None)
}
} else {
Ok(None)
}
}
}
/// Convenience: acquire exclusive lock as a guard
pub async fn lock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<DistributedLockGuard>> {
let req = LockRequest::new(resource, LockType::Exclusive, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
/// Convenience: acquire shared lock as a guard
pub async fn rlock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<DistributedLockGuard>> {
let req = LockRequest::new(resource, LockType::Shared, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
/// Quorum-based lock acquisition: success if at least `self.quorum` clients succeed.
/// Collects all individual lock_ids from successful clients and creates an aggregate lock_id.
/// Returns the LockResponse with aggregate lock_id and individual lock mappings.
async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<(LockId, Arc<dyn LockClient>)>)> {
let futs: Vec<_> = self
.clients
.iter()
.enumerate()
.map(|(idx, client)| async move { (idx, client.acquire_lock(request).await) })
.collect();
let results = futures::future::join_all(futs).await;
// Store all individual lock_ids and their corresponding clients
let mut individual_locks: Vec<(LockId, Arc<dyn LockClient>)> = Vec::new();
for (idx, result) in results {
match result {
Ok(resp) => {
if resp.success {
// Collect individual lock_id and client for each successful acquisition
if let Some(lock_info) = &resp.lock_info
&& idx < self.clients.len()
{
// Save the individual lock_id returned by each client
individual_locks.push((lock_info.id.clone(), self.clients[idx].clone()));
}
} else {
tracing::warn!(
"Failed to acquire lock on client from response: {}, error: {}",
idx,
resp.error.unwrap_or_else(|| "unknown error".to_string())
);
}
}
Err(e) => {
tracing::warn!("Failed to acquire lock on client {}: {}", idx, e);
}
}
}
if individual_locks.len() >= self.quorum {
// Generate a new aggregate lock_id for multiple client locks
let aggregate_lock_id = generate_aggregate_lock_id(&request.resource);
tracing::debug!(
"Generated aggregate lock_id {} for {} individual locks on resource {}",
aggregate_lock_id,
individual_locks.len(),
request.resource
);
let resp = LockResponse::success(
LockInfo {
id: aggregate_lock_id,
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
Duration::ZERO,
);
Ok((resp, individual_locks))
} else {
// Rollback: release all locks that were successfully acquired
let rollback_count = individual_locks.len();
for (individual_lock_id, client) in individual_locks {
if let Err(e) = client.release(&individual_lock_id).await {
tracing::warn!("Failed to rollback lock {} on client: {}", individual_lock_id, e);
}
}
let resp = LockResponse::failure(
format!("Failed to acquire quorum: {}/{} required", rollback_count, self.quorum),
Duration::ZERO,
);
Ok((resp, Vec::new()))
}
}
}
crates/lock/src/fast_lock/disabled_manager.rs
+8 -127
View File
@@ -51,51 +51,22 @@ impl DisabledLockManager {
}
/// Always succeeds - returns a no-op guard
pub async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(bucket, object, owner);
pub async fn acquire_read_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>>) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(key, owner);
self.acquire_lock(request).await
}
/// Always succeeds - returns a no-op guard
pub async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
version: impl Into<Arc<str>>,
key: ObjectKey,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(bucket, object, owner).with_version(version);
let request = ObjectLockRequest::new_write(key, owner);
self.acquire_lock(request).await
}
/// Always succeeds - returns a no-op guard
pub async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_write(bucket, object, owner);
self.acquire_lock(request).await
}
/// Always succeeds - returns a no-op guard
pub async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
version: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_write(bucket, object, owner).with_version(version);
self.acquire_lock(request).await
}
/// Always succeeds - all locks acquired
pub async fn acquire_locks_batch(&self, batch_request: BatchLockRequest) -> BatchLockResult {
let successful_locks: Vec<ObjectKey> = batch_request.requests.iter().map(|req| req.key.clone()).collect();
@@ -161,42 +132,12 @@ impl LockManager for DisabledLockManager {
self.acquire_lock(request).await
}
async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock(bucket, object, owner).await
async fn acquire_read_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock(key, owner).await
}
async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock_versioned(bucket, object, version, owner).await
}
async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock(bucket, object, owner).await
}
async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock_versioned(bucket, object, version, owner).await
async fn acquire_write_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock(key, owner).await
}
async fn acquire_locks_batch(&self, batch_request: BatchLockRequest) -> BatchLockResult {
@@ -235,63 +176,3 @@ impl LockManager for DisabledLockManager {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_disabled_manager_basic_operations() {
let manager = DisabledLockManager::new();
// All operations should succeed immediately
let read_guard = manager
.acquire_read_lock("bucket", "object", "owner1")
.await
.expect("Disabled manager should always succeed");
let write_guard = manager
.acquire_write_lock("bucket", "object", "owner2")
.await
.expect("Disabled manager should always succeed");
// Guards should indicate they are disabled
assert!(read_guard.is_disabled());
assert!(write_guard.is_disabled());
}
#[tokio::test]
async fn test_disabled_manager_batch_operations() {
let manager = DisabledLockManager::new();
let batch = BatchLockRequest::new("owner")
.add_read_lock("bucket", "obj1")
.add_write_lock("bucket", "obj2")
.with_all_or_nothing(true);
let result = manager.acquire_locks_batch(batch).await;
assert!(result.all_acquired);
assert_eq!(result.successful_locks.len(), 2);
assert!(result.failed_locks.is_empty());
}
#[tokio::test]
async fn test_disabled_manager_metrics() {
let manager = DisabledLockManager::new();
// Metrics should indicate empty/disabled state
let metrics = manager.get_metrics();
assert!(metrics.is_empty());
assert_eq!(manager.total_lock_count(), 0);
assert!(manager.get_pool_stats().is_empty());
}
#[tokio::test]
async fn test_disabled_manager_cleanup() {
let manager = DisabledLockManager::new();
// Cleanup should be no-op
assert_eq!(manager.cleanup_expired().await, 0);
assert_eq!(manager.cleanup_expired_traditional().await, 0);
}
}
crates/lock/src/fast_lock/guard.rs
-1055
View File
File diff suppressed because it is too large Load Diff
crates/lock/src/fast_lock/integration_example.rs
-255
View File
@@ -1,255 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Example integration of FastObjectLockManager in set_disk.rs
// This shows how to replace the current slow lock system
use crate::fast_lock::{BatchLockRequest, FastObjectLockManager, ObjectLockRequest};
use std::sync::Arc;
use std::time::Duration;
/// Example integration into SetDisks structure
pub struct SetDisksWithFastLock {
/// Replace the old namespace_lock with fast lock manager
pub fast_lock_manager: Arc<FastObjectLockManager>,
pub locker_owner: String,
// ... other fields remain the same
}
impl SetDisksWithFastLock {
/// Example: Replace get_object_reader with fast locking
pub async fn get_object_reader_fast(
&self,
bucket: &str,
object: &str,
version: Option<&str>,
// ... other parameters
) -> Result<(), Box<dyn std::error::Error>> {
// Fast path: Try to acquire read lock immediately
let _read_guard = if let Some(v) = version {
// Version-specific lock
self.fast_lock_manager
.acquire_read_lock_versioned(bucket, object, v, self.locker_owner.as_str())
.await
.map_err(|_| "Lock acquisition failed")?
} else {
// Latest version lock
self.fast_lock_manager
.acquire_read_lock(bucket, object, self.locker_owner.as_str())
.await
.map_err(|_| "Lock acquisition failed")?
};
// Critical section: Read object
// The lock is automatically released when _read_guard goes out of scope
// ... actual read operation logic
Ok(())
}
/// Example: Replace put_object with fast locking
pub async fn put_object_fast(
&self,
bucket: &str,
object: &str,
version: Option<&str>,
// ... other parameters
) -> Result<(), Box<dyn std::error::Error>> {
// Acquire exclusive write lock with timeout
let request = ObjectLockRequest::new_write(bucket, object, self.locker_owner.as_str())
.with_acquire_timeout(Duration::from_secs(5))
.with_lock_timeout(Duration::from_secs(30));
let request = if let Some(v) = version {
request.with_version(v)
} else {
request
};
let _write_guard = self
.fast_lock_manager
.acquire_lock(request)
.await
.map_err(|_| "Lock acquisition failed")?;
// Critical section: Write object
// ... actual write operation logic
Ok(())
// Lock automatically released when _write_guard drops
}
/// Example: Replace delete_objects with batch fast locking
pub async fn delete_objects_fast(
&self,
bucket: &str,
objects: Vec<(&str, Option<&str>)>, // (object_name, version)
) -> Result<Vec<String>, Box<dyn std::error::Error>> {
// Create batch request for atomic locking
let mut batch = BatchLockRequest::new(self.locker_owner.as_str()).with_all_or_nothing(true); // Either lock all or fail
// Add all objects to batch (sorted internally to prevent deadlocks)
for (object, version) in &objects {
let mut request = ObjectLockRequest::new_write(bucket, *object, self.locker_owner.as_str());
if let Some(v) = version {
request = request.with_version(*v);
}
batch.requests.push(request);
}
// Acquire all locks atomically
let batch_result = self.fast_lock_manager.acquire_locks_batch(batch).await;
if !batch_result.all_acquired {
return Err("Failed to acquire all locks for batch delete".into());
}
// Critical section: Delete all objects
let mut deleted = Vec::new();
for (object, _version) in objects {
// ... actual delete operation logic
deleted.push(object.to_string());
}
// All locks automatically released when guards go out of scope
Ok(deleted)
}
/// Example: Health check integration
pub fn get_lock_health(&self) -> crate::fast_lock::metrics::AggregatedMetrics {
self.fast_lock_manager.get_metrics()
}
/// Example: Cleanup integration
pub async fn cleanup_expired_locks(&self) -> usize {
self.fast_lock_manager.cleanup_expired().await
}
}
/// Performance comparison demonstration
pub mod performance_comparison {
use super::*;
use std::time::Instant;
pub async fn benchmark_fast_vs_old() {
let fast_manager = Arc::new(FastObjectLockManager::new());
let owner = "benchmark_owner";
// Benchmark fast lock acquisition
let start = Instant::now();
let mut guards = Vec::new();
for i in 0..1000 {
let guard = fast_manager
.acquire_write_lock("bucket", format!("object_{i}"), owner)
.await
.expect("Failed to acquire fast lock");
guards.push(guard);
}
let fast_duration = start.elapsed();
println!("Fast lock: 1000 acquisitions in {fast_duration:?}");
// Release all
drop(guards);
// Compare with metrics
let metrics = fast_manager.get_metrics();
println!("Fast path rate: {:.2}%", metrics.shard_metrics.fast_path_rate() * 100.0);
println!("Average wait time: {:?}", metrics.shard_metrics.avg_wait_time());
println!("Total operations/sec: {:.2}", metrics.ops_per_second());
}
}
/// Migration guide from old to new system
pub mod migration_guide {
/*
Step-by-step migration from old lock system:
1. Replace namespace_lock field:
OLD: pub namespace_lock: Arc<rustfs_lock::NamespaceLock>
NEW: pub fast_lock_manager: Arc<FastObjectLockManager>
2. Replace lock acquisition:
OLD: self.namespace_lock.lock_guard(object, &self.locker_owner, timeout, ttl).await?
NEW: self.fast_lock_manager.acquire_write_lock(bucket, object, &self.locker_owner).await?
3. Replace read lock acquisition:
OLD: self.namespace_lock.rlock_guard(object, &self.locker_owner, timeout, ttl).await?
NEW: self.fast_lock_manager.acquire_read_lock(bucket, object, &self.locker_owner).await?
4. Add version support where needed:
NEW: self.fast_lock_manager.acquire_write_lock_versioned(bucket, object, version, owner).await?
5. Replace batch operations:
OLD: Multiple individual lock_guard calls in loop
NEW: Single BatchLockRequest with all objects
6. Remove manual lock release (RAII handles it automatically)
OLD: guard.disarm() or explicit release
NEW: Just let guard go out of scope
Expected performance improvements:
- 10-50x faster lock acquisition
- 90%+ fast path success rate
- Sub-millisecond lock operations
- No deadlock issues with batch operations
- Automatic cleanup and monitoring
*/
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_integration_example() {
let fast_manager = Arc::new(FastObjectLockManager::new());
let set_disks = SetDisksWithFastLock {
fast_lock_manager: fast_manager,
locker_owner: "test_owner".to_string(),
};
// Test read operation
assert!(set_disks.get_object_reader_fast("bucket", "object", None).await.is_ok());
// Test write operation
assert!(set_disks.put_object_fast("bucket", "object", Some("v1")).await.is_ok());
// Test batch delete
let objects = vec![("obj1", None), ("obj2", Some("v1"))];
let result = set_disks.delete_objects_fast("bucket", objects).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_version_locking() {
let fast_manager = Arc::new(FastObjectLockManager::new());
// Should be able to lock different versions simultaneously
let guard_v1 = fast_manager
.acquire_write_lock_versioned("bucket", "object", "v1", "owner1")
.await
.expect("Failed to lock v1");
let guard_v2 = fast_manager
.acquire_write_lock_versioned("bucket", "object", "v2", "owner2")
.await
.expect("Failed to lock v2");
// Both locks should coexist
assert!(!guard_v1.is_released());
assert!(!guard_v2.is_released());
}
}
crates/lock/src/fast_lock/integration_test.rs
-166
View File
@@ -1,166 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Integration tests for performance optimizations
#[cfg(test)]
mod tests {
use crate::fast_lock::FastObjectLockManager;
use tokio::time::Duration;
#[tokio::test]
async fn test_object_pool_integration() {
let manager = FastObjectLockManager::new();
// Create many locks to test pool efficiency
let mut guards = Vec::new();
for i in 0..100 {
let bucket = format!("test-bucket-{}", i % 10); // Reuse some bucket names
let object = format!("test-object-{i}");
let guard = manager
.acquire_write_lock(bucket.as_str(), object.as_str(), "test-owner")
.await
.expect("Failed to acquire lock");
guards.push(guard);
}
// Drop all guards to return objects to pool
drop(guards);
// Wait a moment for cleanup
tokio::time::sleep(Duration::from_millis(100)).await;
// Get pool statistics from all shards
let pool_stats = manager.get_pool_stats();
let (hits, misses, releases, pool_size) = pool_stats.iter().fold((0, 0, 0, 0), |acc, stats| {
(acc.0 + stats.0, acc.1 + stats.1, acc.2 + stats.2, acc.3 + stats.3)
});
let hit_rate = if hits + misses > 0 {
hits as f64 / (hits + misses) as f64
} else {
0.0
};
println!("Pool stats - Hits: {hits}, Misses: {misses}, Releases: {releases}, Pool size: {pool_size}");
println!("Hit rate: {:.2}%", hit_rate * 100.0);
// We should see some pool activity
assert!(hits + misses > 0, "Pool should have been used");
}
#[tokio::test]
async fn test_optimized_notification_system() {
let manager = FastObjectLockManager::new();
// Test that notifications work by measuring timing
let start = std::time::Instant::now();
// Acquire two read locks on different objects (should be fast)
let guard1 = manager
.acquire_read_lock("bucket", "object1", "reader1")
.await
.expect("Failed to acquire first read lock");
let guard2 = manager
.acquire_read_lock("bucket", "object2", "reader2")
.await
.expect("Failed to acquire second read lock");
let duration = start.elapsed();
println!("Two read locks on different objects took: {duration:?}");
// Should be very fast since no contention
assert!(duration < Duration::from_millis(10), "Read locks should be fast with no contention");
drop(guard1);
drop(guard2);
// Test same object contention
let start = std::time::Instant::now();
let guard1 = manager
.acquire_read_lock("bucket", "same-object", "reader1")
.await
.expect("Failed to acquire first read lock on same object");
let guard2 = manager
.acquire_read_lock("bucket", "same-object", "reader2")
.await
.expect("Failed to acquire second read lock on same object");
let duration = start.elapsed();
println!("Two read locks on same object took: {duration:?}");
// Should still be fast since read locks are compatible
assert!(duration < Duration::from_millis(10), "Compatible read locks should be fast");
drop(guard1);
drop(guard2);
}
#[tokio::test]
async fn test_fast_path_optimization() {
let manager = FastObjectLockManager::new();
// First acquisition should be fast path
let start = std::time::Instant::now();
let guard1 = manager
.acquire_read_lock("bucket", "object", "reader1")
.await
.expect("Failed to acquire first read lock");
let first_duration = start.elapsed();
// Second read lock should also be fast path
let start = std::time::Instant::now();
let guard2 = manager
.acquire_read_lock("bucket", "object", "reader2")
.await
.expect("Failed to acquire second read lock");
let second_duration = start.elapsed();
println!("First lock: {first_duration:?}, Second lock: {second_duration:?}");
// Both should be very fast (sub-millisecond typically)
assert!(first_duration < Duration::from_millis(10));
assert!(second_duration < Duration::from_millis(10));
drop(guard1);
drop(guard2);
}
#[tokio::test]
async fn test_batch_operations_optimization() {
let manager = FastObjectLockManager::new();
// Test batch operation with sorted keys
let batch = crate::fast_lock::BatchLockRequest::new("batch-owner")
.add_read_lock("bucket", "obj1")
.add_read_lock("bucket", "obj2")
.add_write_lock("bucket", "obj3")
.with_all_or_nothing(false);
let start = std::time::Instant::now();
let result = manager.acquire_locks_batch(batch).await;
let duration = start.elapsed();
println!("Batch operation took: {duration:?}");
assert!(result.all_acquired, "All locks should be acquired");
assert_eq!(result.successful_locks.len(), 3);
assert!(result.failed_locks.is_empty());
// Batch should be reasonably fast
assert!(duration < Duration::from_millis(100));
}
}
crates/lock/src/fast_lock/manager.rs
+16 -235
View File
@@ -77,100 +77,54 @@ impl FastObjectLockManager {
}
/// Acquire shared (read) lock
pub async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(bucket, object, owner);
pub async fn acquire_read_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>>) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(key, owner);
self.acquire_lock(request).await
}
/// Acquire shared (read) lock for specific version
pub async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
version: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_read(bucket, object, owner).with_version(version);
self.acquire_lock(request).await
}
/// Acquire exclusive (write) lock
pub async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
// let bucket = bucket.into();
// let object = object.into();
// let owner = owner.into();
// error!("acquire_write_lock: bucket={:?}, object={:?}, owner={:?}", bucket, object, owner);
let request = ObjectLockRequest::new_write(bucket, object, owner);
self.acquire_lock(request).await
}
/// Acquire exclusive (write) lock for specific version
pub async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
version: impl Into<Arc<str>>,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_write(bucket, object, owner).with_version(version);
pub async fn acquire_write_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>>) -> Result<FastLockGuard, LockResult> {
let request = ObjectLockRequest::new_write(key, owner);
self.acquire_lock(request).await
}
/// Acquire high-priority read lock - optimized for database queries
pub async fn acquire_high_priority_read_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
key: ObjectKey,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request =
ObjectLockRequest::new_read(bucket, object, owner).with_priority(crate::fast_lock::types::LockPriority::High);
let request = ObjectLockRequest::new_read(key, owner).with_priority(crate::fast_lock::types::LockPriority::High);
self.acquire_lock(request).await
}
/// Acquire high-priority write lock - optimized for database queries
pub async fn acquire_high_priority_write_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
key: ObjectKey,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request =
ObjectLockRequest::new_write(bucket, object, owner).with_priority(crate::fast_lock::types::LockPriority::High);
let request = ObjectLockRequest::new_write(key, owner).with_priority(crate::fast_lock::types::LockPriority::High);
self.acquire_lock(request).await
}
/// Acquire critical priority read lock - for system operations
pub async fn acquire_critical_read_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
key: ObjectKey,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request =
ObjectLockRequest::new_read(bucket, object, owner).with_priority(crate::fast_lock::types::LockPriority::Critical);
let request = ObjectLockRequest::new_read(key, owner).with_priority(crate::fast_lock::types::LockPriority::Critical);
self.acquire_lock(request).await
}
/// Acquire critical priority write lock - for system operations
pub async fn acquire_critical_write_lock(
&self,
bucket: impl Into<Arc<str>>,
object: impl Into<Arc<str>>,
key: ObjectKey,
owner: impl Into<Arc<str>>,
) -> Result<FastLockGuard, LockResult> {
let request =
ObjectLockRequest::new_write(bucket, object, owner).with_priority(crate::fast_lock::types::LockPriority::Critical);
let request = ObjectLockRequest::new_write(key, owner).with_priority(crate::fast_lock::types::LockPriority::Critical);
self.acquire_lock(request).await
}
@@ -440,42 +394,12 @@ impl LockManager for FastObjectLockManager {
self.acquire_lock(request).await
}
async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock(bucket, object, owner).await
async fn acquire_read_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock(key, owner).await
}
async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_read_lock_versioned(bucket, object, version, owner).await
}
async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock(bucket, object, owner).await
}
async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock_versioned(bucket, object, version, owner).await
async fn acquire_write_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult> {
self.acquire_write_lock(key, owner).await
}
async fn acquire_locks_batch(&self, batch_request: BatchLockRequest) -> BatchLockResult {
@@ -514,146 +438,3 @@ impl LockManager for FastObjectLockManager {
false
}
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::time::Duration;
#[tokio::test]
async fn test_manager_basic_operations() {
let manager = FastObjectLockManager::new();
// Test read lock
let read_guard = manager
.acquire_read_lock("bucket", "object", "owner1")
.await
.expect("Failed to acquire read lock");
// Should be able to acquire another read lock
let read_guard2 = manager
.acquire_read_lock("bucket", "object", "owner2")
.await
.expect("Failed to acquire second read lock");
drop(read_guard);
drop(read_guard2);
// Test write lock
let write_guard = manager
.acquire_write_lock("bucket", "object", "owner1")
.await
.expect("Failed to acquire write lock");
drop(write_guard);
}
#[tokio::test]
async fn test_manager_contention() {
let manager = Arc::new(FastObjectLockManager::new());
// Acquire write lock
let write_guard = manager
.acquire_write_lock("bucket", "object", "owner1")
.await
.expect("Failed to acquire write lock");
// Try to acquire read lock (should timeout)
let manager_clone = manager.clone();
let read_result =
tokio::time::timeout(Duration::from_millis(100), manager_clone.acquire_read_lock("bucket", "object", "owner2")).await;
assert!(read_result.is_err()); // Should timeout
drop(write_guard);
// Now read lock should succeed
let read_guard = manager
.acquire_read_lock("bucket", "object", "owner2")
.await
.expect("Failed to acquire read lock after write lock released");
drop(read_guard);
}
#[tokio::test]
async fn test_versioned_locks() {
let manager = FastObjectLockManager::new();
// Acquire lock on version v1
let v1_guard = manager
.acquire_write_lock_versioned("bucket", "object", "v1", "owner1")
.await
.expect("Failed to acquire v1 lock");
// Should be able to acquire lock on version v2 simultaneously
let v2_guard = manager
.acquire_write_lock_versioned("bucket", "object", "v2", "owner2")
.await
.expect("Failed to acquire v2 lock");
drop(v1_guard);
drop(v2_guard);
}
#[tokio::test]
async fn test_batch_operations() {
let manager = FastObjectLockManager::new();
let batch = BatchLockRequest::new("owner")
.add_read_lock("bucket", "obj1")
.add_write_lock("bucket", "obj2")
.with_all_or_nothing(true);
let result = manager.acquire_locks_batch(batch).await;
assert!(result.all_acquired);
assert_eq!(result.successful_locks.len(), 2);
assert!(result.failed_locks.is_empty());
}
#[tokio::test]
async fn test_metrics() {
let manager = FastObjectLockManager::new();
// Perform some operations
let _guard1 = manager.acquire_read_lock("bucket", "obj1", "owner").await.unwrap();
let _guard2 = manager.acquire_write_lock("bucket", "obj2", "owner").await.unwrap();
let metrics = manager.get_metrics();
assert!(metrics.shard_metrics.total_acquisitions() > 0);
assert!(metrics.shard_metrics.fast_path_rate() > 0.0);
}
#[tokio::test]
async fn test_cleanup() {
let config = LockConfig {
max_idle_time: Duration::from_secs(1), // Use 1 second for easier testing
..Default::default()
};
let manager = FastObjectLockManager::with_config(config);
// Acquire and release some locks
{
let _guard = manager.acquire_read_lock("bucket", "obj1", "owner1").await.unwrap();
let _guard2 = manager.acquire_read_lock("bucket", "obj2", "owner2").await.unwrap();
} // Locks are released here
// Check lock count before cleanup
let count_before = manager.total_lock_count();
assert!(count_before >= 2, "Should have at least 2 locks before cleanup");
// Wait for idle timeout
tokio::time::sleep(Duration::from_secs(2)).await;
// Force cleanup with traditional method to ensure cleanup for testing
let cleaned = manager.cleanup_expired_traditional().await;
let count_after = manager.total_lock_count();
// The test should pass if cleanup works at all
assert!(
cleaned > 0 || count_after < count_before,
"Cleanup should either clean locks or they should be cleaned by other means"
);
}
}
crates/lock/src/fast_lock/manager_trait.rs
+2 -30
View File
@@ -31,38 +31,10 @@ pub trait LockManager: Send + Sync {
async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult>;
/// Acquire shared (read) lock
async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult>;
/// Acquire shared (read) lock for specific version
async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult>;
async fn acquire_read_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult>;
/// Acquire exclusive (write) lock
async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult>;
/// Acquire exclusive (write) lock for specific version
async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> Result<FastLockGuard, LockResult>;
async fn acquire_write_lock(&self, key: ObjectKey, owner: impl Into<Arc<str>> + Send) -> Result<FastLockGuard, LockResult>;
/// Acquire multiple locks atomically
async fn acquire_locks_batch(&self, batch_request: BatchLockRequest) -> BatchLockResult;
crates/lock/src/fast_lock/mod.rs
+9 -8
View File
@@ -26,8 +26,6 @@
pub mod disabled_manager;
pub mod guard;
pub mod integration_example;
pub mod integration_test;
pub mod manager;
pub mod manager_trait;
pub mod metrics;
@@ -37,6 +35,9 @@ pub mod shard;
pub mod state;
pub mod types;
#[cfg(test)]
mod tests;
// Re-export main types
pub use disabled_manager::DisabledLockManager;
pub use guard::FastLockGuard;
@@ -45,19 +46,19 @@ pub use manager_trait::LockManager;
use std::time::Duration;
pub use types::*;
/// Default RustFS specific timeouts in seconds
pub(crate) const DEFAULT_RUSTFS_MAX_ACQUIRE_TIMEOUT: u64 = 120;
/// Maximum acquire timeout in seconds (for slow storage / high contention; override via env)
pub(crate) const DEFAULT_RUSTFS_MAX_ACQUIRE_TIMEOUT: u64 = 60;
/// Default RustFS acquire timeout in seconds
pub(crate) const DEFAULT_RUSTFS_ACQUIRE_TIMEOUT: u64 = 60;
/// Default acquire timeout in seconds (how long to wait for a lock before giving up)
pub(crate) const DEFAULT_RUSTFS_ACQUIRE_TIMEOUT: u64 = 10;
/// Default shard count (must be power of 2)
pub const DEFAULT_SHARD_COUNT: usize = 1024;
/// Default lock timeout
/// Default lock timeout (lease TTL; lock is released if not refreshed within this duration)
pub const DEFAULT_LOCK_TIMEOUT: Duration = Duration::from_secs(30);
/// Default acquire timeout - increased for network block storage workloads (e.g., Hetzner Ceph)
/// Default acquire timeout - common value for local/low-latency; use env to increase for slow storage
pub const DEFAULT_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(DEFAULT_RUSTFS_ACQUIRE_TIMEOUT);
/// Maximum acquire timeout for high-load scenarios
crates/lock/src/fast_lock/shard.rs
+8 -4
View File
@@ -759,13 +759,17 @@ mod tests {
let shard = LockShard::new(0);
// First acquire a lock that will block the batch operation
let blocking_request = ObjectLockRequest::new_write("bucket", "obj1", "blocking_owner");
let blocking_request = ObjectLockRequest::new_write(ObjectKey::new("bucket", "obj1"), "blocking_owner")
.with_acquire_timeout(Duration::from_secs(1));
shard.acquire_lock(&blocking_request).await.unwrap();
// Now try a batch operation that should fail and clean up properly
// Use short acquire timeout so the test fails fast when obj1 is already locked
// (default is 60s which would make this test very slow)
let requests = vec![
ObjectLockRequest::new_read("bucket", "obj2", "batch_owner"), // This should succeed
ObjectLockRequest::new_write("bucket", "obj1", "batch_owner"), // This should fail due to existing lock
ObjectLockRequest::new_read(ObjectKey::new("bucket", "obj2"), "batch_owner")
.with_acquire_timeout(Duration::from_millis(100)), // This should succeed
ObjectLockRequest::new_write(ObjectKey::new("bucket", "obj1"), "batch_owner")
.with_acquire_timeout(Duration::from_millis(100)), // This should fail due to existing lock
];
let result = shard.acquire_locks_batch(requests, true).await;
crates/lock/src/fast_lock/tests.rs
+532
View File
@@ -0,0 +1,532 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod fast_lock_tests {
use crate::fast_lock::FastObjectLockManager;
use crate::fast_lock::types::{LockConfig, LockMode, LockPriority, LockResult, ObjectKey, ObjectLockRequest};
use std::sync::Arc;
use std::time::Duration;
use tokio::time::sleep;
/// Helper function to create a test lock manager
fn create_test_manager() -> FastObjectLockManager {
let config = LockConfig {
shard_count: 4, // Use smaller shard count for tests
default_lock_timeout: Duration::from_secs(30),
default_acquire_timeout: Duration::from_secs(5),
..LockConfig::default()
};
FastObjectLockManager::with_config(config)
}
#[tokio::test]
async fn test_basic_write_lock_acquire_release() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner: Arc<str> = Arc::from("test-owner");
// Acquire write lock
let mut guard = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("Should acquire write lock");
// Verify guard properties
assert_eq!(guard.key(), &key);
assert_eq!(guard.mode(), LockMode::Exclusive);
assert_eq!(guard.owner(), &owner);
assert!(!guard.is_released());
// Manually release lock
assert!(guard.release(), "Should release lock successfully");
assert!(guard.is_released(), "Guard should be marked as released");
// Try to acquire again - should succeed
let guard2 = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("Should acquire write lock again after release");
drop(guard2);
}
#[tokio::test]
async fn test_basic_read_lock_acquire_release() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner: Arc<str> = Arc::from("test-owner");
// Acquire read lock
let mut guard = manager
.acquire_read_lock(key.clone(), owner.clone())
.await
.expect("Should acquire read lock");
// Verify guard properties
assert_eq!(guard.key(), &key);
assert_eq!(guard.mode(), LockMode::Shared);
assert_eq!(guard.owner(), &owner);
assert!(!guard.is_released());
// Manually release lock
assert!(guard.release(), "Should release lock successfully");
assert!(guard.is_released(), "Guard should be marked as released");
}
#[tokio::test]
async fn test_lock_auto_release_on_drop() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner1: Arc<str> = Arc::from("owner1");
let owner2: Arc<str> = Arc::from("owner2");
// Acquire lock and drop guard
{
let guard = manager
.acquire_write_lock(key.clone(), owner1.clone())
.await
.expect("Should acquire write lock");
assert!(!guard.is_released());
// Guard is dropped here, lock should be automatically released
}
// Wait a bit to ensure cleanup
sleep(Duration::from_millis(10)).await;
// Another owner should be able to acquire the lock
let guard2 = manager
.acquire_write_lock(key.clone(), owner2.clone())
.await
.expect("Should acquire write lock after previous guard dropped");
drop(guard2);
}
#[tokio::test]
async fn test_multiple_read_locks() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner1: Arc<str> = Arc::from("owner1");
let owner2: Arc<str> = Arc::from("owner2");
let owner3: Arc<str> = Arc::from("owner3");
// Multiple read locks should be allowed
let mut guard1 = manager
.acquire_read_lock(key.clone(), owner1.clone())
.await
.expect("Should acquire first read lock");
let mut guard2 = manager
.acquire_read_lock(key.clone(), owner2.clone())
.await
.expect("Should acquire second read lock");
let mut guard3 = manager
.acquire_read_lock(key.clone(), owner3.clone())
.await
.expect("Should acquire third read lock");
// All guards should be valid
assert_eq!(guard1.mode(), LockMode::Shared);
assert_eq!(guard2.mode(), LockMode::Shared);
assert_eq!(guard3.mode(), LockMode::Shared);
// Release all
assert!(guard1.release());
assert!(guard2.release());
assert!(guard3.release());
}
#[tokio::test]
async fn test_write_lock_excludes_read_lock() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let writer: Arc<str> = Arc::from("writer");
let reader: Arc<str> = Arc::from("reader");
// Acquire write lock
let mut write_guard = manager
.acquire_write_lock(key.clone(), writer.clone())
.await
.expect("Should acquire write lock");
// Try to acquire read lock - should timeout
let read_request =
ObjectLockRequest::new_read(key.clone(), reader.clone()).with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(read_request).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"Read lock should timeout when write lock is held"
);
// Release write lock
assert!(write_guard.release());
// Now read lock should succeed
let mut read_guard = manager
.acquire_read_lock(key.clone(), reader.clone())
.await
.expect("Should acquire read lock after write lock released");
assert!(read_guard.release());
}
#[tokio::test]
async fn test_read_lock_excludes_write_lock() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let reader: Arc<str> = Arc::from("reader");
let writer: Arc<str> = Arc::from("writer");
// Acquire read lock
let mut read_guard = manager
.acquire_read_lock(key.clone(), reader.clone())
.await
.expect("Should acquire read lock");
// Try to acquire write lock - should timeout
let write_request =
ObjectLockRequest::new_write(key.clone(), writer.clone()).with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(write_request).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"Write lock should timeout when read lock is held"
);
// Release read lock
assert!(read_guard.release());
// Now write lock should succeed
let mut write_guard = manager
.acquire_write_lock(key.clone(), writer.clone())
.await
.expect("Should acquire write lock after read lock released");
assert!(write_guard.release());
}
#[tokio::test]
async fn test_write_lock_excludes_write_lock() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner1: Arc<str> = Arc::from("owner1");
let owner2: Arc<str> = Arc::from("owner2");
// Acquire first write lock
let mut guard1 = manager
.acquire_write_lock(key.clone(), owner1.clone())
.await
.expect("Should acquire first write lock");
// Try to acquire second write lock - should timeout
let request2 = ObjectLockRequest::new_write(key.clone(), owner2.clone()).with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(request2).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"Second write lock should timeout when first write lock is held"
);
// Release first lock
assert!(guard1.release());
// Now second write lock should succeed
let mut guard2 = manager
.acquire_write_lock(key.clone(), owner2.clone())
.await
.expect("Should acquire second write lock after first released");
assert!(guard2.release());
}
#[tokio::test]
async fn test_same_owner_reentrant_write_lock() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner: Arc<str> = Arc::from("owner");
// Acquire first write lock
let mut guard1 = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("Should acquire first write lock");
// Same owner trying to acquire again - should timeout (not reentrant)
let request2 = ObjectLockRequest::new_write(key.clone(), owner.clone()).with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(request2).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"Same owner should not be able to acquire lock again (not reentrant)"
);
assert!(guard1.release());
}
#[tokio::test]
async fn test_different_keys_no_conflict() {
let manager = create_test_manager();
let key1 = ObjectKey::new("bucket1", "object1");
let key2 = ObjectKey::new("bucket2", "object2");
let owner: Arc<str> = Arc::from("owner");
// Acquire locks on different keys simultaneously
let mut guard1 = manager
.acquire_write_lock(key1.clone(), owner.clone())
.await
.expect("Should acquire lock on key1");
let mut guard2 = manager
.acquire_write_lock(key2.clone(), owner.clone())
.await
.expect("Should acquire lock on key2");
// Both should be valid
assert_eq!(guard1.key(), &key1);
assert_eq!(guard2.key(), &key2);
assert!(guard1.release());
assert!(guard2.release());
}
#[tokio::test]
async fn test_versioned_keys() {
let manager = create_test_manager();
let base_key = ObjectKey::new("bucket", "object");
let versioned_key = ObjectKey::with_version("bucket", "object", "v1");
let owner: Arc<str> = Arc::from("owner");
// Acquire lock on base key
let mut guard1 = manager
.acquire_write_lock(base_key.clone(), owner.clone())
.await
.expect("Should acquire lock on base key");
// Should be able to acquire lock on versioned key (different keys)
let mut guard2 = manager
.acquire_write_lock(versioned_key.clone(), owner.clone())
.await
.expect("Should acquire lock on versioned key");
assert_eq!(guard1.key(), &base_key);
assert_eq!(guard2.key(), &versioned_key);
assert!(guard1.release());
assert!(guard2.release());
}
#[tokio::test]
async fn test_concurrent_read_locks() {
let manager = Arc::new(create_test_manager());
let key = ObjectKey::new("test-bucket", "test-object");
let num_readers = 10;
let mut handles = Vec::new();
// Spawn multiple readers
for i in 0..num_readers {
let manager = manager.clone();
let key = key.clone();
let owner: Arc<str> = Arc::from(format!("reader-{}", i));
let handle = tokio::spawn(async move {
let mut guard = manager.acquire_read_lock(key, owner).await.expect("Should acquire read lock");
// Hold lock for a bit
sleep(Duration::from_millis(10)).await;
assert!(guard.release());
});
handles.push(handle);
}
// Wait for all readers
for handle in handles {
handle.await.expect("Reader task should complete");
}
}
#[tokio::test]
async fn test_concurrent_write_lock_contention() {
let manager = Arc::new(create_test_manager());
let key = ObjectKey::new("test-bucket", "test-object");
let num_writers = 5;
let mut handles = Vec::new();
// Spawn multiple writers - they should serialize
for i in 0..num_writers {
let manager = manager.clone();
let key = key.clone();
let owner: Arc<str> = Arc::from(format!("writer-{}", i));
let handle = tokio::spawn(async move {
let mut guard = manager
.acquire_write_lock(key, owner)
.await
.expect("Should acquire write lock");
// Hold lock for a bit
sleep(Duration::from_millis(10)).await;
assert!(guard.release());
});
handles.push(handle);
}
// Wait for all writers - they should complete sequentially
for handle in handles {
handle.await.expect("Writer task should complete");
}
}
#[tokio::test]
async fn test_lock_timeout() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner1: Arc<str> = Arc::from("owner1");
let owner2: Arc<str> = Arc::from("owner2");
// Acquire first lock
let mut guard1 = manager
.acquire_write_lock(key.clone(), owner1.clone())
.await
.expect("Should acquire first lock");
// Try to acquire with short timeout - should timeout
let request = ObjectLockRequest::new_write(key.clone(), owner2.clone()).with_acquire_timeout(Duration::from_millis(50));
let result = manager.acquire_lock(request).await;
assert!(matches!(result, Err(LockResult::Timeout)), "Should timeout when lock is held");
assert!(guard1.release());
}
#[tokio::test]
async fn test_lock_priority() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let normal_owner: Arc<str> = Arc::from("normal");
let high_owner: Arc<str> = Arc::from("high");
// Acquire normal priority lock
let normal_request = ObjectLockRequest::new_write(key.clone(), normal_owner.clone())
.with_priority(LockPriority::Normal)
.with_acquire_timeout(Duration::from_secs(1));
let mut normal_guard = manager
.acquire_lock(normal_request)
.await
.expect("Should acquire normal priority lock");
// Try high priority lock - should still timeout (write locks are exclusive)
let high_request = ObjectLockRequest::new_write(key.clone(), high_owner.clone())
.with_priority(LockPriority::High)
.with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(high_request).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"High priority write lock should still timeout when normal write lock is held"
);
assert!(normal_guard.release());
}
#[tokio::test]
async fn test_double_release() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner: Arc<str> = Arc::from("owner");
let mut guard = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("Should acquire lock");
// First release should succeed
assert!(guard.release(), "First release should succeed");
assert!(guard.is_released(), "Guard should be marked as released");
// Second release should fail
assert!(!guard.release(), "Second release should fail");
}
#[tokio::test]
async fn test_lock_info() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let owner: Arc<str> = Arc::from("owner");
let mut guard = manager
.acquire_write_lock(key.clone(), owner.clone())
.await
.expect("Should acquire lock");
// Get lock info
let lock_info = guard.lock_info();
assert!(lock_info.is_some(), "Should have lock info");
if let Some(info) = lock_info {
assert_eq!(info.key, key);
assert_eq!(info.mode, LockMode::Exclusive);
assert_eq!(info.owner, owner);
}
// Release lock
assert!(guard.release());
// Lock info should be None after release
let lock_info_after = guard.lock_info();
assert!(lock_info_after.is_none(), "Lock info should be None after release");
}
#[tokio::test]
async fn test_read_write_mixed_scenario() {
let manager = create_test_manager();
let key = ObjectKey::new("test-bucket", "test-object");
let reader1: Arc<str> = Arc::from("reader1");
let reader2: Arc<str> = Arc::from("reader2");
let writer: Arc<str> = Arc::from("writer");
// Acquire two read locks
let mut read_guard1 = manager
.acquire_read_lock(key.clone(), reader1.clone())
.await
.expect("Should acquire first read lock");
let mut read_guard2 = manager
.acquire_read_lock(key.clone(), reader2.clone())
.await
.expect("Should acquire second read lock");
// Writer should timeout
let write_request =
ObjectLockRequest::new_write(key.clone(), writer.clone()).with_acquire_timeout(Duration::from_millis(100));
let result = manager.acquire_lock(write_request).await;
assert!(
matches!(result, Err(LockResult::Timeout)),
"Write lock should timeout when read locks are held"
);
// Release one read lock
assert!(read_guard1.release());
// Writer should still timeout (other read lock still held)
let write_request2 =
ObjectLockRequest::new_write(key.clone(), writer.clone()).with_acquire_timeout(Duration::from_millis(100));
let result2 = manager.acquire_lock(write_request2).await;
assert!(
matches!(result2, Err(LockResult::Timeout)),
"Write lock should still timeout when read lock is held"
);
// Release second read lock
assert!(read_guard2.release());
// Now writer should succeed
let mut write_guard = manager
.acquire_write_lock(key.clone(), writer.clone())
.await
.expect("Should acquire write lock after all read locks released");
assert!(write_guard.release());
}
}
crates/lock/src/fast_lock/types.rs
+94 -14
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::fast_lock::guard::FastLockGuard;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use smartstring::SmartString;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
@@ -28,6 +28,88 @@ pub struct ObjectKey {
pub version: Option<Arc<str>>, // None means latest version
}
impl Serialize for ObjectKey {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
use serde::ser::SerializeStruct;
let mut state = serializer.serialize_struct("ObjectKey", 3)?;
state.serialize_field("bucket", self.bucket.as_ref())?;
state.serialize_field("object", self.object.as_ref())?;
state.serialize_field("version", &self.version.as_ref().map(|v| v.as_ref()))?;
state.end()
}
}
impl<'de> Deserialize<'de> for ObjectKey {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
use serde::de::{self, MapAccess, Visitor};
use std::fmt;
#[derive(Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum Field {
Bucket,
Object,
Version,
}
struct ObjectKeyVisitor;
impl<'de> Visitor<'de> for ObjectKeyVisitor {
type Value = ObjectKey;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("struct ObjectKey")
}
fn visit_map<V>(self, mut map: V) -> Result<ObjectKey, V::Error>
where
V: MapAccess<'de>,
{
let mut bucket = None;
let mut object = None;
let mut version = None;
while let Some(key) = map.next_key()? {
match key {
Field::Bucket => {
if bucket.is_some() {
return Err(de::Error::duplicate_field("bucket"));
}
let s: String = map.next_value()?;
bucket = Some(Arc::from(s));
}
Field::Object => {
if object.is_some() {
return Err(de::Error::duplicate_field("object"));
}
let s: String = map.next_value()?;
object = Some(Arc::from(s));
}
Field::Version => {
if version.is_some() {
return Err(de::Error::duplicate_field("version"));
}
let opt: Option<String> = map.next_value()?;
version = opt.map(Arc::from);
}
}
}
let bucket = bucket.ok_or_else(|| de::Error::missing_field("bucket"))?;
let object = object.ok_or_else(|| de::Error::missing_field("object"))?;
Ok(ObjectKey { bucket, object, version })
}
}
const FIELDS: &[&str] = &["bucket", "object", "version"];
deserializer.deserialize_struct("ObjectKey", FIELDS, ObjectKeyVisitor)
}
}
impl ObjectKey {
pub fn new(bucket: impl Into<Arc<str>>, object: impl Into<Arc<str>>) -> Self {
Self {
@@ -198,9 +280,9 @@ pub struct ObjectLockRequest {
}
impl ObjectLockRequest {
pub fn new_read(bucket: impl Into<Arc<str>>, object: impl Into<Arc<str>>, owner: impl Into<Arc<str>>) -> Self {
pub fn new_read(key: ObjectKey, owner: impl Into<Arc<str>>) -> Self {
Self {
key: ObjectKey::new(bucket, object),
key,
mode: LockMode::Shared,
owner: owner.into(),
acquire_timeout: crate::fast_lock::DEFAULT_ACQUIRE_TIMEOUT,
@@ -209,9 +291,9 @@ impl ObjectLockRequest {
}
}
pub fn new_write(bucket: impl Into<Arc<str>>, object: impl Into<Arc<str>>, owner: impl Into<Arc<str>>) -> Self {
pub fn new_write(key: ObjectKey, owner: impl Into<Arc<str>>) -> Self {
Self {
key: ObjectKey::new(bucket, object),
key,
mode: LockMode::Exclusive,
owner: owner.into(),
acquire_timeout: crate::fast_lock::DEFAULT_ACQUIRE_TIMEOUT,
@@ -317,15 +399,13 @@ impl BatchLockRequest {
}
}
pub fn add_read_lock(mut self, bucket: impl Into<Arc<str>>, object: impl Into<Arc<str>>) -> Self {
self.requests
.push(ObjectLockRequest::new_read(bucket, object, self.owner.clone()));
pub fn add_read_lock(mut self, key: ObjectKey) -> Self {
self.requests.push(ObjectLockRequest::new_read(key, self.owner.clone()));
self
}
pub fn add_write_lock(mut self, bucket: impl Into<Arc<str>>, object: impl Into<Arc<str>>) -> Self {
self.requests
.push(ObjectLockRequest::new_write(bucket, object, self.owner.clone()));
pub fn add_write_lock(mut self, key: ObjectKey) -> Self {
self.requests.push(ObjectLockRequest::new_write(key, self.owner.clone()));
self
}
@@ -366,7 +446,7 @@ mod tests {
#[test]
fn test_lock_request() {
let req = ObjectLockRequest::new_read("bucket", "object", "owner")
let req = ObjectLockRequest::new_read(ObjectKey::new("bucket", "object"), "owner")
.with_version("v1")
.with_priority(LockPriority::High);
@@ -378,8 +458,8 @@ mod tests {
#[test]
fn test_batch_request() {
let batch = BatchLockRequest::new("owner")
.add_read_lock("bucket", "obj1")
.add_write_lock("bucket", "obj2");
.add_read_lock(ObjectKey::new("bucket", "obj1"))
.add_write_lock(ObjectKey::new("bucket", "obj2"));
assert_eq!(batch.requests.len(), 2);
assert_eq!(batch.requests[0].mode, LockMode::Shared);
crates/lock/src/guard.rs
-117
View File
@@ -1,117 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{LockClient, LockId};
use std::sync::{Arc, LazyLock};
use tokio::sync::mpsc;
#[derive(Debug, Clone)]
struct UnlockJob {
lock_id: LockId,
clients: Vec<Arc<dyn LockClient>>, // cloned Arcs; cheap and shares state
}
#[derive(Debug)]
struct UnlockRuntime {
tx: mpsc::Sender<UnlockJob>,
}
// Global unlock runtime with background worker
static UNLOCK_RUNTIME: LazyLock<UnlockRuntime> = LazyLock::new(|| {
// Larger buffer to reduce contention during bursts
let (tx, mut rx) = mpsc::channel::<UnlockJob>(8192);
// Spawn background worker when first used; assumes a Tokio runtime is available
tokio::spawn(async move {
while let Some(job) = rx.recv().await {
// Best-effort release across clients; try all, success if any succeeds
let mut any_ok = false;
let lock_id = job.lock_id.clone();
for client in job.clients.into_iter() {
if client.release(&lock_id).await.unwrap_or(false) {
any_ok = true;
}
}
if !any_ok {
tracing::warn!("LockGuard background release failed for {}", lock_id);
} else {
tracing::debug!("LockGuard background released {}", lock_id);
}
}
});
UnlockRuntime { tx }
});
/// A RAII guard that releases the lock asynchronously when dropped.
#[derive(Debug)]
pub struct LockGuard {
lock_id: LockId,
clients: Vec<Arc<dyn LockClient>>,
/// If true, Drop will not try to release (used if user manually released).
disarmed: bool,
}
impl LockGuard {
pub(crate) fn new(lock_id: LockId, clients: Vec<Arc<dyn LockClient>>) -> Self {
Self {
lock_id,
clients,
disarmed: false,
}
}
/// Get the lock id associated with this guard
pub fn lock_id(&self) -> &LockId {
&self.lock_id
}
/// Manually disarm the guard so dropping it won't release the lock.
/// Call this if you explicitly released the lock elsewhere.
pub fn disarm(&mut self) {
self.disarmed = true;
}
}
impl Drop for LockGuard {
fn drop(&mut self) {
if self.disarmed {
return;
}
let job = UnlockJob {
lock_id: self.lock_id.clone(),
clients: self.clients.clone(),
};
// Try a non-blocking send to avoid panics in Drop
if let Err(err) = UNLOCK_RUNTIME.tx.try_send(job) {
// Channel full or closed; best-effort fallback: spawn a detached task
let lock_id = self.lock_id.clone();
let clients = self.clients.clone();
tracing::warn!("LockGuard channel send failed ({}), spawning fallback unlock task for {}", err, lock_id);
// If runtime is not available, this will panic; but in RustFS we are inside Tokio contexts.
let handle = tokio::spawn(async move {
let futures_iter = clients.into_iter().map(|client| {
let id = lock_id.clone();
async move { client.release(&id).await.unwrap_or(false) }
});
let _ = futures::future::join_all(futures_iter).await;
});
// Explicitly drop the JoinHandle to acknowledge detaching the task.
drop(handle);
}
}
}
crates/lock/src/lib.rs
+11 -132
View File
@@ -17,6 +17,8 @@
// ============================================================================
// Application Layer Modules
pub mod distributed_lock;
pub mod local_lock;
pub mod namespace;
// Abstraction Layer Modules
@@ -27,7 +29,6 @@ pub mod fast_lock;
// Core Modules
pub mod error;
pub mod guard;
pub mod types;
// ============================================================================
@@ -38,6 +39,7 @@ pub mod types;
pub use crate::{
// Client interfaces
client::{LockClient, local::LocalClient},
distributed_lock::DistributedLockGuard,
// Error types
error::{LockError, Result},
// Fast Lock System exports
@@ -45,9 +47,8 @@ pub use crate::{
BatchLockRequest, BatchLockResult, DisabledLockManager, FastLockGuard, FastObjectLockManager, LockManager, LockMode,
LockResult, ObjectKey, ObjectLockInfo, ObjectLockRequest, metrics::AggregatedMetrics,
},
guard::LockGuard,
// Main components
namespace::{NamespaceLock, NamespaceLockManager},
namespace::{NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper},
// Core types
types::{
HealthInfo, HealthStatus, LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockStatus,
@@ -81,6 +82,7 @@ use std::sync::Arc;
use std::sync::OnceLock;
/// Enum wrapper for different lock manager implementations
#[derive(Debug)]
pub enum GlobalLockManager {
Enabled(Arc<FastObjectLockManager>),
Disabled(DisabledLockManager),
@@ -157,51 +159,23 @@ impl LockManager for GlobalLockManager {
async fn acquire_read_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
key: ObjectKey,
owner: impl Into<Arc<str>> + Send,
) -> std::result::Result<FastLockGuard, LockResult> {
match self {
Self::Enabled(manager) => manager.acquire_read_lock(bucket, object, owner).await,
Self::Disabled(manager) => manager.acquire_read_lock(bucket, object, owner).await,
}
}
async fn acquire_read_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> std::result::Result<FastLockGuard, LockResult> {
match self {
Self::Enabled(manager) => manager.acquire_read_lock_versioned(bucket, object, version, owner).await,
Self::Disabled(manager) => manager.acquire_read_lock_versioned(bucket, object, version, owner).await,
Self::Enabled(manager) => manager.acquire_read_lock(key, owner).await,
Self::Disabled(manager) => manager.acquire_read_lock(key, owner).await,
}
}
async fn acquire_write_lock(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
key: ObjectKey,
owner: impl Into<Arc<str>> + Send,
) -> std::result::Result<FastLockGuard, LockResult> {
match self {
Self::Enabled(manager) => manager.acquire_write_lock(bucket, object, owner).await,
Self::Disabled(manager) => manager.acquire_write_lock(bucket, object, owner).await,
}
}
async fn acquire_write_lock_versioned(
&self,
bucket: impl Into<Arc<str>> + Send,
object: impl Into<Arc<str>> + Send,
version: impl Into<Arc<str>> + Send,
owner: impl Into<Arc<str>> + Send,
) -> std::result::Result<FastLockGuard, LockResult> {
match self {
Self::Enabled(manager) => manager.acquire_write_lock_versioned(bucket, object, version, owner).await,
Self::Disabled(manager) => manager.acquire_write_lock_versioned(bucket, object, version, owner).await,
Self::Enabled(manager) => manager.acquire_write_lock(key, owner).await,
Self::Disabled(manager) => manager.acquire_write_lock(key, owner).await,
}
}
@@ -290,98 +264,3 @@ pub fn get_global_fast_lock_manager() -> Arc<FastObjectLockManager> {
panic!("Cannot get FastObjectLockManager when locks are disabled. Use get_global_lock_manager() instead.");
})
}
// ============================================================================
// Convenience Functions
// ============================================================================
/// Create a new namespace lock
pub fn create_namespace_lock(namespace: String, _distributed: bool) -> NamespaceLock {
// The distributed behavior is now determined by the type of clients added to the NamespaceLock
// This function just creates an empty NamespaceLock
NamespaceLock::new(namespace)
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_global_lock_manager_basic() {
let manager = get_global_lock_manager();
// Should be able to acquire locks
let guard = manager.acquire_read_lock("bucket", "object", "owner").await;
assert!(guard.is_ok());
// Test metrics
let _metrics = manager.get_metrics();
// Even if locks are disabled, metrics should be available (empty or real)
// shard_count is usize so always >= 0
}
#[tokio::test]
async fn test_disabled_manager_direct() {
let manager = DisabledLockManager::new();
// All operations should succeed immediately
let guard = manager.acquire_read_lock("bucket", "object", "owner").await;
assert!(guard.is_ok());
assert!(guard.unwrap().is_disabled());
// Metrics should be empty
let metrics = manager.get_metrics();
assert!(metrics.is_empty());
assert_eq!(manager.total_lock_count(), 0);
}
#[tokio::test]
async fn test_enabled_manager_direct() {
let manager = FastObjectLockManager::new();
// Operations should work normally
let guard = manager.acquire_read_lock("bucket", "object", "owner").await;
assert!(guard.is_ok());
assert!(!guard.unwrap().is_disabled());
// Should have real metrics
let _metrics = manager.get_metrics();
// Note: total_lock_count might be > 0 due to previous lock acquisition
}
#[tokio::test]
async fn test_global_manager_enum_wrapper() {
// Test the GlobalLockManager enum directly
let enabled_manager = GlobalLockManager::Enabled(Arc::new(FastObjectLockManager::new()));
let disabled_manager = GlobalLockManager::Disabled(DisabledLockManager::new());
assert!(!enabled_manager.is_disabled());
assert!(disabled_manager.is_disabled());
// Test trait methods work for both
let enabled_guard = enabled_manager.acquire_read_lock("bucket", "obj", "owner").await;
let disabled_guard = disabled_manager.acquire_read_lock("bucket", "obj", "owner").await;
assert!(enabled_guard.is_ok());
assert!(disabled_guard.is_ok());
assert!(!enabled_guard.unwrap().is_disabled());
assert!(disabled_guard.unwrap().is_disabled());
}
#[tokio::test]
async fn test_batch_operations_work() {
let manager = get_global_lock_manager();
let batch = BatchLockRequest::new("owner")
.add_read_lock("bucket", "obj1")
.add_write_lock("bucket", "obj2");
let result = manager.acquire_locks_batch(batch).await;
// Should succeed regardless of whether locks are enabled or disabled
assert!(result.all_acquired);
assert_eq!(result.successful_locks.len(), 2);
assert!(result.failed_locks.is_empty());
}
}
crates/lock/src/local_lock.rs
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
GlobalLockManager, ObjectKey,
error::Result,
fast_lock::{FastLockGuard, LockManager, LockMode, ObjectLockRequest},
types::{LockPriority, LockRequest, LockType},
};
use std::sync::Arc;
use std::time::Duration;
/// Local lock handler using GlobalLockManager
/// Directly uses FastObjectLockManager for high-performance local locking
#[derive(Debug)]
pub struct LocalLock {
/// Global lock manager for fast local locks
manager: Arc<GlobalLockManager>,
/// Namespace identifier
namespace: String,
}
impl LocalLock {
/// Create new local lock
pub fn new(namespace: String, manager: Arc<GlobalLockManager>) -> Self {
Self { namespace, manager }
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
&self.namespace
}
/// Get resource key for this namespace
pub fn get_resource_key(&self, resource: &ObjectKey) -> String {
format!("{}:{}", self.namespace, resource)
}
/// Acquire a lock and return a RAII guard
pub(crate) async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<FastLockGuard>> {
// Convert LockRequest to ObjectLockRequest
let object_key = request.resource.clone();
let mode = match request.lock_type {
LockType::Exclusive => LockMode::Exclusive,
LockType::Shared => LockMode::Shared,
};
let owner: Arc<str> = request.owner.clone().into();
// Convert LockPriority from types::LockPriority to fast_lock::types::LockPriority
let fast_priority = match request.priority {
LockPriority::Low => crate::fast_lock::types::LockPriority::Low,
LockPriority::Normal => crate::fast_lock::types::LockPriority::Normal,
LockPriority::High => crate::fast_lock::types::LockPriority::High,
LockPriority::Critical => crate::fast_lock::types::LockPriority::Critical,
};
let object_request = ObjectLockRequest {
key: object_key,
mode,
owner,
acquire_timeout: request.acquire_timeout,
lock_timeout: request.ttl,
priority: fast_priority,
};
match self.manager.as_ref().acquire_lock(object_request).await {
Ok(guard) => Ok(Some(guard)),
Err(_) => Ok(None),
}
}
/// Convenience: acquire exclusive lock as a guard
pub async fn lock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<FastLockGuard>> {
let req = LockRequest::new(resource, LockType::Exclusive, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
/// Convenience: acquire shared lock as a guard
pub async fn rlock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<FastLockGuard>> {
let req = LockRequest::new(resource, LockType::Shared, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
}
crates/lock/src/namespace.rs
-586
View File
@@ -1,586 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use std::sync::Arc;
use std::time::Duration;
use crate::{
client::LockClient,
error::{LockError, Result},
guard::LockGuard,
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
};
/// Namespace lock for managing locks by resource namespaces
#[derive(Debug)]
pub struct NamespaceLock {
/// Lock clients for this namespace
clients: Vec<Arc<dyn LockClient>>,
/// Namespace identifier
namespace: String,
/// Quorum size for operations (1 for local, majority for distributed)
quorum: usize,
}
impl NamespaceLock {
/// Create new namespace lock
pub fn new(namespace: String) -> Self {
Self {
clients: Vec::new(),
namespace,
quorum: 1,
}
}
/// Create namespace lock with clients
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
let quorum = if clients.len() > 1 {
// For multiple clients (distributed mode), require majority
(clients.len() / 2) + 1
} else {
// For single client (local mode), only need 1
1
};
Self {
clients,
namespace,
quorum,
}
}
/// Create namespace lock with clients and an explicit quorum size.
/// Quorum will be clamped into [1, clients.len()]. For single client, quorum is always 1.
pub fn with_clients_and_quorum(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
let q = if clients.len() <= 1 {
1
} else {
quorum.clamp(1, clients.len())
};
Self {
clients,
namespace,
quorum: q,
}
}
/// Create namespace lock with client (compatibility)
pub fn with_client(client: Arc<dyn LockClient>) -> Self {
Self::with_clients("default".to_string(), vec![client])
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
&self.namespace
}
/// Get resource key for this namespace
pub fn get_resource_key(&self, resource: &str) -> String {
format!("{}:{}", self.namespace, resource)
}
/// Acquire lock using clients with transactional semantics (all-or-nothing)
pub async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For single client, use it directly
if self.clients.len() == 1 {
return self.clients[0].acquire_lock(request).await;
}
// Quorum-based acquisition for distributed mode
let (resp, _idxs) = self.acquire_lock_quorum(request).await?;
Ok(resp)
}
/// Acquire a lock and return a RAII guard that will release asynchronously on Drop.
/// This is a thin wrapper around `acquire_lock` and will only create a guard when acquisition succeeds.
pub async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<LockGuard>> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
if self.clients.len() == 1 {
let resp = self.clients[0].acquire_lock(request).await?;
if resp.success {
return Ok(Some(LockGuard::new(
LockId::new_deterministic(&request.resource),
vec![self.clients[0].clone()],
)));
}
return Ok(None);
}
let (resp, idxs) = self.acquire_lock_quorum(request).await?;
if resp.success {
let subset: Vec<_> = idxs.into_iter().filter_map(|i| self.clients.get(i).cloned()).collect();
Ok(Some(LockGuard::new(LockId::new_deterministic(&request.resource), subset)))
} else {
Ok(None)
}
}
/// Convenience: acquire exclusive lock as a guard
pub async fn lock_guard(&self, resource: &str, owner: &str, timeout: Duration, ttl: Duration) -> Result<Option<LockGuard>> {
let req = LockRequest::new(self.get_resource_key(resource), LockType::Exclusive, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
/// Convenience: acquire shared lock as a guard
pub async fn rlock_guard(&self, resource: &str, owner: &str, timeout: Duration, ttl: Duration) -> Result<Option<LockGuard>> {
let req = LockRequest::new(self.get_resource_key(resource), LockType::Shared, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
self.acquire_guard(&req).await
}
/// Quorum-based lock acquisition: success if at least `self.quorum` clients succeed.
/// Returns the LockResponse and the indices of clients that acquired the lock.
async fn acquire_lock_quorum(&self, request: &LockRequest) -> Result<(LockResponse, Vec<usize>)> {
let futs: Vec<_> = self
.clients
.iter()
.enumerate()
.map(|(idx, client)| async move { (idx, client.acquire_lock(request).await) })
.collect();
let results = futures::future::join_all(futs).await;
let mut successful_clients = Vec::new();
for (idx, res) in results {
if let Ok(resp) = res
&& resp.success
{
successful_clients.push(idx);
}
}
if successful_clients.len() >= self.quorum {
let resp = LockResponse::success(
LockInfo {
id: LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
Duration::ZERO,
);
Ok((resp, successful_clients))
} else {
if !successful_clients.is_empty() {
self.rollback_acquisitions(request, &successful_clients).await;
}
let resp = LockResponse::failure(
format!("Failed to acquire quorum: {}/{} required", successful_clients.len(), self.quorum),
Duration::ZERO,
);
Ok((resp, Vec::new()))
}
}
/// Rollback lock acquisitions on specified clients
async fn rollback_acquisitions(&self, request: &LockRequest, client_indices: &[usize]) {
let lock_id = LockId::new_deterministic(&request.resource);
let rollback_futures: Vec<_> = client_indices
.iter()
.filter_map(|&idx| self.clients.get(idx))
.map(|client| async {
if let Err(e) = client.release(&lock_id).await {
tracing::warn!("Failed to rollback lock on client: {}", e);
}
})
.collect();
futures::future::join_all(rollback_futures).await;
tracing::info!(
"Rolled back {} lock acquisitions for resource: {}",
client_indices.len(),
request.resource
);
}
/// Release lock using clients
pub async fn release_lock(&self, lock_id: &LockId) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For single client, use it directly
if self.clients.len() == 1 {
return self.clients[0].release(lock_id).await;
}
// For multiple clients, try to release from all clients
let futures: Vec<_> = self
.clients
.iter()
.map(|client| {
let id = lock_id.clone();
async move { client.release(&id).await }
})
.collect();
let results = futures::future::join_all(futures).await;
let successful = results.into_iter().filter_map(|r| r.ok()).filter(|&r| r).count();
// For release, if any succeed, consider it successful
Ok(successful > 0)
}
/// Get health information
pub async fn get_health(&self) -> crate::types::HealthInfo {
let lock_stats = self.get_stats().await;
let mut health = crate::types::HealthInfo {
node_id: self.namespace.clone(),
lock_stats,
..Default::default()
};
// Check client status
let mut connected_clients = 0;
for client in &self.clients {
if client.is_online().await {
connected_clients += 1;
}
}
health.status = if connected_clients > 0 {
crate::types::HealthStatus::Healthy
} else {
crate::types::HealthStatus::Degraded
};
health.connected_nodes = connected_clients;
health.total_nodes = self.clients.len();
health
}
/// Get namespace statistics
pub async fn get_stats(&self) -> crate::types::LockStats {
let mut stats = crate::types::LockStats::default();
// Try to get stats from clients
for client in &self.clients {
if let Ok(client_stats) = client.get_stats().await {
stats.successful_acquires += client_stats.successful_acquires;
stats.failed_acquires += client_stats.failed_acquires;
}
}
stats
}
}
impl Default for NamespaceLock {
fn default() -> Self {
Self::new("default".to_string())
}
}
/// Namespace lock manager trait
#[async_trait]
pub trait NamespaceLockManager: Send + Sync {
/// Batch get write lock
async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool>;
/// Batch release write lock
async fn unlock_batch(&self, resources: &[String], owner: &str) -> Result<()>;
/// Batch get read lock
async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool>;
/// Batch release read lock
async fn runlock_batch(&self, resources: &[String], owner: &str) -> Result<()>;
}
#[async_trait]
impl NamespaceLockManager for NamespaceLock {
async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// Transactional batch lock: all resources must be locked or none
let mut acquired_resources = Vec::new();
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let request = LockRequest::new(&namespaced_resource, LockType::Exclusive, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
let response = self.acquire_lock(&request).await?;
if response.success {
acquired_resources.push(namespaced_resource);
} else {
// Rollback all previously acquired locks
self.rollback_batch_locks(&acquired_resources, owner).await;
return Ok(false);
}
}
Ok(true)
}
async fn unlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// Release all locks (best effort)
let release_futures: Vec<_> = resources
.iter()
.map(|resource| {
let namespaced_resource = self.get_resource_key(resource);
let lock_id = LockId::new_deterministic(&namespaced_resource);
async move {
if let Err(e) = self.release_lock(&lock_id).await {
tracing::warn!("Failed to release lock for resource {}: {}", resource, e);
}
}
})
.collect();
futures::future::join_all(release_futures).await;
Ok(())
}
async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// Transactional batch read lock: all resources must be locked or none
let mut acquired_resources = Vec::new();
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let request = LockRequest::new(&namespaced_resource, LockType::Shared, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
let response = self.acquire_lock(&request).await?;
if response.success {
acquired_resources.push(namespaced_resource);
} else {
// Rollback all previously acquired read locks
self.rollback_batch_locks(&acquired_resources, owner).await;
return Ok(false);
}
}
Ok(true)
}
async fn runlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// Release all read locks (best effort)
let release_futures: Vec<_> = resources
.iter()
.map(|resource| {
let namespaced_resource = self.get_resource_key(resource);
let lock_id = LockId::new_deterministic(&namespaced_resource);
async move {
if let Err(e) = self.release_lock(&lock_id).await {
tracing::warn!("Failed to release read lock for resource {}: {}", resource, e);
}
}
})
.collect();
futures::future::join_all(release_futures).await;
Ok(())
}
}
impl NamespaceLock {
/// Rollback batch lock acquisitions
async fn rollback_batch_locks(&self, acquired_resources: &[String], _owner: &str) {
let rollback_futures: Vec<_> = acquired_resources
.iter()
.map(|resource| {
let lock_id = LockId::new_deterministic(resource);
async move {
if let Err(e) = self.release_lock(&lock_id).await {
tracing::warn!("Failed to rollback lock for resource {}: {}", resource, e);
}
}
})
.collect();
futures::future::join_all(rollback_futures).await;
tracing::info!("Rolled back {} batch lock acquisitions", acquired_resources.len());
}
}
#[cfg(test)]
mod tests {
use crate::LocalClient;
use super::*;
#[tokio::test]
async fn test_namespace_lock_local() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
let resources = vec!["test1".to_string(), "test2".to_string()];
// Test batch lock
let result = ns_lock
.lock_batch(&resources, "test_owner", Duration::from_millis(100), Duration::from_secs(10))
.await;
assert!(result.is_ok());
assert!(result.unwrap());
// Test batch unlock
let result = ns_lock.unlock_batch(&resources, "test_owner").await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_guard_acquire_and_drop_release() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
// Acquire guard
let guard = ns_lock
.lock_guard("guard-resource", "owner", Duration::from_millis(100), Duration::from_secs(5))
.await
.unwrap();
assert!(guard.is_some());
let lock_id = guard.as_ref().unwrap().lock_id().clone();
// Drop guard to trigger background release
drop(guard);
// Give background worker a moment to process
tokio::time::sleep(Duration::from_millis(50)).await;
// Re-acquire should succeed (previous lock released)
let req = LockRequest::new(&lock_id.resource, LockType::Exclusive, "owner").with_ttl(Duration::from_secs(2));
let resp = ns_lock.acquire_lock(&req).await.unwrap();
assert!(resp.success);
// Cleanup
let _ = ns_lock.release_lock(&LockId::new_deterministic(&lock_id.resource)).await;
}
#[tokio::test]
async fn test_connection_health() {
let local_lock = NamespaceLock::new("test-namespace".to_string());
let health = local_lock.get_health().await;
assert_eq!(health.status, crate::types::HealthStatus::Degraded); // No clients
}
#[tokio::test]
async fn test_namespace_lock_creation() {
let ns_lock = NamespaceLock::new("test-namespace".to_string());
assert_eq!(ns_lock.namespace(), "test-namespace");
}
#[tokio::test]
async fn test_namespace_lock_new_local() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
assert_eq!(ns_lock.namespace(), "default");
assert_eq!(ns_lock.clients.len(), 1);
assert!(ns_lock.clients[0].is_local().await);
// Test that it can perform lock operations
let resources = vec!["test-resource".to_string()];
let result = ns_lock
.lock_batch(&resources, "test-owner", Duration::from_millis(100), Duration::from_secs(10))
.await;
assert!(result.is_ok());
assert!(result.unwrap());
}
#[tokio::test]
async fn test_namespace_lock_resource_key() {
let ns_lock = NamespaceLock::new("test-namespace".to_string());
// Test resource key generation
let resource_key = ns_lock.get_resource_key("test-resource");
assert_eq!(resource_key, "test-namespace:test-resource");
}
#[tokio::test]
async fn test_transactional_batch_lock() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
let resources = vec!["resource1".to_string(), "resource2".to_string(), "resource3".to_string()];
// First, acquire one of the resources to simulate conflict
let conflicting_request = LockRequest::new(ns_lock.get_resource_key("resource2"), LockType::Exclusive, "other_owner")
.with_ttl(Duration::from_secs(10));
let response = ns_lock.acquire_lock(&conflicting_request).await.unwrap();
assert!(response.success);
// Now try batch lock - should fail and rollback
let result = ns_lock
.lock_batch(&resources, "test_owner", Duration::from_millis(10), Duration::from_secs(5))
.await;
assert!(result.is_ok());
assert!(!result.unwrap()); // Should fail due to conflict
// Verify that no locks were left behind (all rolled back)
for resource in &resources {
if resource != "resource2" {
// Skip the one we intentionally locked
let check_request = LockRequest::new(ns_lock.get_resource_key(resource), LockType::Exclusive, "verify_owner")
.with_ttl(Duration::from_secs(1));
let check_response = ns_lock.acquire_lock(&check_request).await.unwrap();
assert!(check_response.success, "Resource {resource} should be available after rollback");
// Clean up
let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(resource));
let _ = ns_lock.release_lock(&lock_id).await;
}
}
}
#[tokio::test]
async fn test_distributed_lock_consistency() {
// Create a namespace with multiple local clients to simulate distributed scenario
let client1: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let clients = vec![client1, client2];
// LocalClient shares a global in-memory map. For exclusive locks, only one can acquire at a time.
// In real distributed setups the quorum should be tied to EC write quorum. Here we use quorum=1 for success.
let ns_lock = NamespaceLock::with_clients_and_quorum("test-namespace".to_string(), clients, 1);
let request = LockRequest::new("test-resource", LockType::Shared, "test_owner").with_ttl(Duration::from_secs(2));
// This should succeed only if ALL clients can acquire the lock
let response = ns_lock.acquire_lock(&request).await.unwrap();
// Since we're using separate LocalClient instances, they don't share state
// so this test demonstrates the consistency check
assert!(response.success); // Either all succeed or rollback happens
}
}
crates/lock/src/namespace/mod.rs
+336
View File
@@ -0,0 +1,336 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
ObjectKey,
client::LockClient,
distributed_lock::{DistributedLock, DistributedLockGuard},
error::Result,
fast_lock::FastLockGuard,
local_lock::LocalLock,
types::{LockId, LockRequest},
};
use std::sync::Arc;
use std::time::Duration;
#[cfg(test)]
mod tests;
/// Unified guard for namespace locks
/// Supports both DistributedLockGuard (for Distributed locks) and FastLockGuard (for Local locks)
#[derive(Debug)]
pub enum NamespaceLockGuard {
/// Standard guard for Distributed locks
Standard(DistributedLockGuard),
/// Fast guard for Local locks using GlobalLockManager
Fast(FastLockGuard),
}
/// Wrapper for NamespaceLock that provides convenient lock acquisition methods
/// This wrapper holds the lock instance and resource information for easy lock acquisition
#[derive(Debug)]
pub struct NamespaceLockWrapper {
lock: NamespaceLock,
resource: ObjectKey,
owner: String,
}
impl NamespaceLockWrapper {
/// Create a new wrapper with the lock, resource, and owner
pub fn new(lock: NamespaceLock, resource: ObjectKey, owner: String) -> Self {
Self { lock, resource, owner }
}
/// Acquire write lock (exclusive lock) with timeout
/// Returns the guard if acquisition succeeds, or an error if it fails
pub async fn get_write_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
self.lock.get_write_lock(self.resource.clone(), &self.owner, timeout).await
}
/// Acquire read lock (shared lock) with timeout
/// Returns the guard if acquisition succeeds, or an error if it fails
pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
self.lock.get_read_lock(self.resource.clone(), &self.owner, timeout).await
}
}
impl NamespaceLockGuard {
/// Get the lock ID if available (only for Standard guards)
pub fn lock_id(&self) -> Option<&LockId> {
match self {
Self::Standard(guard) => Some(guard.lock_id()),
Self::Fast(_) => None,
}
}
/// Get the object key if available (only for Fast guards)
pub fn key(&self) -> Option<&ObjectKey> {
match self {
Self::Standard(_) => None,
Self::Fast(guard) => Some(guard.key()),
}
}
/// Manually release the lock early
pub fn release(&mut self) -> bool {
match self {
Self::Standard(guard) => {
// DistributedLockGuard::release() actually releases the lock and then disarms
guard.release()
}
Self::Fast(guard) => guard.release(),
}
}
/// Check if the lock has been released
pub fn is_released(&self) -> bool {
match self {
Self::Standard(guard) => {
// Check if the guard has been disarmed, which indicates the lock was released
guard.is_disarmed()
}
Self::Fast(guard) => guard.is_released(),
}
}
}
/// Namespace lock for managing locks by resource namespaces
/// Supports DistributedLock and LocalLock
#[derive(Debug)]
pub enum NamespaceLock {
/// Distributed lock (distributed use case)
Distributed(DistributedLock),
/// Local lock using GlobalLockManager (high-performance local locking)
Local(LocalLock),
}
impl NamespaceLock {
/// Create new namespace lock with single client (local use case)
/// Uses DistributedLock with quorum=1 for single client
pub fn new(namespace: String, client: Arc<dyn LockClient>) -> Self {
Self::Distributed(DistributedLock::new(namespace, vec![client], 1))
}
/// Create namespace lock with client (compatibility)
/// Uses DistributedLock with quorum=1 for single client
pub fn with_client(client: Arc<dyn LockClient>) -> Self {
Self::Distributed(DistributedLock::new("default".to_string(), vec![client], 1))
}
/// Create namespace lock with GlobalLockManager (high-performance local locking)
pub fn with_local_manager(namespace: String, manager: Arc<crate::GlobalLockManager>) -> Self {
Self::Local(LocalLock::new(namespace, manager))
}
/// Create namespace lock with clients
/// Uses DistributedLock with appropriate quorum
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
// Multiple clients: use DistributedLock with majority quorum
let quorum = if clients.len() > 1 { (clients.len() / 2) + 1 } else { 1 };
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
}
/// Create namespace lock with clients and an explicit quorum size.
/// Quorum will be clamped into [1, clients.len()].
pub fn with_clients_and_quorum(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
match self {
Self::Distributed(lock) => lock.namespace(),
Self::Local(lock) => lock.namespace(),
}
}
/// Get resource key for this namespace
pub fn get_resource_key(&self, resource: &ObjectKey) -> String {
match self {
Self::Distributed(lock) => lock.get_resource_key(resource),
Self::Local(lock) => lock.get_resource_key(resource),
}
}
/// Acquire a lock and return a RAII guard that will release asynchronously on Drop.
/// This is a thin wrapper around `acquire_lock` and will only create a guard when acquisition succeeds.
pub async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<NamespaceLockGuard>> {
match self {
Self::Distributed(lock) => lock
.acquire_guard(request)
.await
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
Self::Local(lock) => lock.acquire_guard(request).await.map(|opt| opt.map(NamespaceLockGuard::Fast)),
}
}
/// Convenience: acquire exclusive lock as a guard
pub async fn lock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<NamespaceLockGuard>> {
match self {
Self::Distributed(lock) => lock
.lock_guard(resource, owner, timeout, ttl)
.await
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
Self::Local(lock) => lock
.lock_guard(resource, owner, timeout, ttl)
.await
.map(|opt| opt.map(NamespaceLockGuard::Fast)),
}
}
/// Convenience: acquire shared lock as a guard
pub async fn rlock_guard(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
ttl: Duration,
) -> Result<Option<NamespaceLockGuard>> {
match self {
Self::Distributed(lock) => lock
.rlock_guard(resource, owner, timeout, ttl)
.await
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
Self::Local(lock) => lock
.rlock_guard(resource, owner, timeout, ttl)
.await
.map(|opt| opt.map(NamespaceLockGuard::Fast)),
}
}
/// Acquire write lock (exclusive lock) with timeout
/// Returns the guard if acquisition succeeds, or an error if it fails
pub async fn get_write_lock(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
let resource_str = format!("{}", resource);
match self.lock_guard(resource, owner, timeout, ttl).await {
Ok(Some(guard)) => Ok(guard),
Ok(None) => {
// None can mean timeout or other failure - check if it's a quorum error
// For distributed locks, quorum errors are already converted to LockError::QuorumNotReached
// So if we get None here, it's likely a timeout
Err(crate::error::LockError::timeout(resource_str, timeout))
}
Err(e) => Err(e),
}
}
/// Acquire read lock (shared lock) with timeout
/// Returns the guard if acquisition succeeds, or an error if it fails
pub async fn get_read_lock(
&self,
resource: ObjectKey,
owner: &str,
timeout: Duration,
) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
let resource_str = format!("{}", resource);
match self.rlock_guard(resource, owner, timeout, ttl).await {
Ok(Some(guard)) => Ok(guard),
Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
Err(e) => Err(e),
}
}
/// Get health information
pub async fn get_health(&self) -> crate::types::HealthInfo {
let lock_stats = self.get_stats().await;
let namespace = self.namespace().to_string();
let mut health = crate::types::HealthInfo {
node_id: namespace.clone(),
lock_stats,
..Default::default()
};
match self {
Self::Distributed(lock) => {
// Check client status - parallelize async calls for better performance
let clients = lock.clients();
let client_checks: Vec<_> = clients.iter().map(|client| client.is_online()).collect();
let results = futures::future::join_all(client_checks).await;
let connected_clients = results.iter().filter(|&&online| online).count();
let quorum = if clients.len() > 1 { (clients.len() / 2) + 1 } else { 1 };
health.status = if connected_clients >= quorum {
crate::types::HealthStatus::Healthy
} else {
crate::types::HealthStatus::Degraded
};
health.connected_nodes = connected_clients;
health.total_nodes = clients.len();
}
Self::Local(_) => {
// Local locks are always healthy (they use GlobalLockManager which is always available)
health.status = crate::types::HealthStatus::Healthy;
health.connected_nodes = 1;
health.total_nodes = 1;
}
}
health
}
/// Get namespace statistics
pub async fn get_stats(&self) -> crate::types::LockStats {
let mut stats = crate::types::LockStats::default();
match self {
Self::Distributed(lock) => {
// Parallelize stats collection for better performance
let stats_futures: Vec<_> = lock.clients().iter().map(|client| client.get_stats()).collect();
let results = futures::future::join_all(stats_futures).await;
for result in results {
match result {
Ok(client_stats) => {
stats.successful_acquires += client_stats.successful_acquires;
stats.failed_acquires += client_stats.failed_acquires;
}
Err(e) => {
tracing::debug!("Failed to get stats from client: {}", e);
}
}
}
}
Self::Local(_) => {
// Local locks use GlobalLockManager which doesn't expose detailed stats
// Stats are tracked internally but not exposed through the same interface
// We leave stats at default (0) for now
}
}
stats
}
}
impl Default for NamespaceLock {
fn default() -> Self {
use crate::client::ClientFactory;
Self::new("default".to_string(), ClientFactory::create_local())
}
}
crates/lock/src/namespace/tests.rs
+370
View File
@@ -0,0 +1,370 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::GlobalLockManager;
use crate::client::{ClientFactory, local::LocalClient};
use crate::types::LockType;
use std::sync::Arc;
use std::time::Duration;
fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object),
version: None,
}
}
#[tokio::test]
async fn test_namespace_lock_new() {
let client = ClientFactory::create_local();
let lock = NamespaceLock::new("test-namespace".to_string(), client);
assert_eq!(lock.namespace(), "test-namespace");
}
#[tokio::test]
async fn test_namespace_lock_with_client() {
let client = ClientFactory::create_local();
let lock = NamespaceLock::with_client(client);
assert_eq!(lock.namespace(), "default");
}
#[tokio::test]
async fn test_namespace_lock_with_local_manager() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("local-ns".to_string(), manager);
assert_eq!(lock.namespace(), "local-ns");
}
#[tokio::test]
async fn test_namespace_lock_with_clients() {
let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
let lock = NamespaceLock::with_clients("multi-client".to_string(), clients);
assert_eq!(lock.namespace(), "multi-client");
}
#[tokio::test]
async fn test_namespace_lock_get_resource_key() {
let client = ClientFactory::create_local();
let lock = NamespaceLock::new("test-ns".to_string(), client);
let resource = create_test_object_key("bucket", "object");
let key = lock.get_resource_key(&resource);
assert!(key.contains("test-ns"));
assert!(key.contains("bucket"));
assert!(key.contains("object"));
}
#[tokio::test]
async fn test_namespace_lock_acquire_guard_local() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("test-local".to_string(), manager);
let resource = create_test_object_key("bucket", "object");
let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner1")
.with_acquire_timeout(Duration::from_secs(5))
.with_ttl(Duration::from_secs(30));
let guard_opt = lock.acquire_guard(&request).await.unwrap();
assert!(guard_opt.is_some());
if let Some(NamespaceLockGuard::Fast(guard)) = guard_opt {
assert_eq!(guard.key(), &resource);
assert!(!guard.is_released());
// Test release
let mut guard = guard;
assert!(guard.release());
assert!(guard.is_released());
} else {
panic!("Expected Fast guard");
}
}
#[tokio::test]
async fn test_namespace_lock_get_write_lock_local() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("test-write".to_string(), manager);
let resource = create_test_object_key("bucket", "object");
let guard = lock
.get_write_lock(resource.clone(), "owner1", Duration::from_secs(5))
.await
.unwrap();
match guard {
NamespaceLockGuard::Fast(guard) => {
assert_eq!(guard.key(), &resource);
assert!(!guard.is_released());
}
NamespaceLockGuard::Standard(_) => {
panic!("Expected Fast guard for local lock");
}
}
}
#[tokio::test]
async fn test_namespace_lock_get_read_lock_local() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("test-read".to_string(), manager);
let resource = create_test_object_key("bucket", "object");
let guard = lock
.get_read_lock(resource.clone(), "owner1", Duration::from_secs(5))
.await
.unwrap();
match guard {
NamespaceLockGuard::Fast(guard) => {
assert_eq!(guard.key(), &resource);
assert!(!guard.is_released());
}
NamespaceLockGuard::Standard(_) => {
panic!("Expected Fast guard for local lock");
}
}
}
#[tokio::test]
async fn test_namespace_lock_guard_release() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("test-release".to_string(), manager);
let resource = create_test_object_key("bucket", "object");
let mut guard = lock.get_write_lock(resource, "owner1", Duration::from_secs(5)).await.unwrap();
assert!(!guard.is_released());
assert!(guard.release());
assert!(guard.is_released());
}
#[tokio::test]
async fn test_namespace_lock_wrapper() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("wrapper-test".to_string(), manager);
let resource = create_test_object_key("bucket", "object");
let wrapper = NamespaceLockWrapper::new(lock, resource.clone(), "owner1".to_string());
let guard = wrapper.get_write_lock(Duration::from_secs(5)).await.unwrap();
match guard {
NamespaceLockGuard::Fast(guard) => {
assert_eq!(guard.key(), &resource);
}
_ => panic!("Expected Fast guard"),
}
}
#[tokio::test]
async fn test_namespace_lock_get_health_local() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("health-test".to_string(), manager);
let health = lock.get_health().await;
assert_eq!(health.node_id, "health-test");
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
assert_eq!(health.connected_nodes, 1);
assert_eq!(health.total_nodes, 1);
}
#[tokio::test]
async fn test_namespace_lock_get_stats_local() {
let manager = Arc::new(GlobalLockManager::new());
let lock = NamespaceLock::with_local_manager("stats-test".to_string(), manager);
let stats = lock.get_stats().await;
// Local locks don't expose detailed stats, so defaults should be 0
assert_eq!(stats.successful_acquires, 0);
assert_eq!(stats.failed_acquires, 0);
}
#[tokio::test]
async fn test_namespace_lock_default() {
let lock = NamespaceLock::default();
assert_eq!(lock.namespace(), "default");
}
#[tokio::test]
async fn test_namespace_lock_guard_lock_id() {
let client = ClientFactory::create_local();
let lock = NamespaceLock::new("test-id".to_string(), client);
let resource = create_test_object_key("bucket", "object");
let request = LockRequest::new(resource, LockType::Exclusive, "owner1")
.with_acquire_timeout(Duration::from_secs(5))
.with_ttl(Duration::from_secs(30));
if let Some(NamespaceLockGuard::Standard(guard)) = lock.acquire_guard(&request).await.unwrap() {
// lock_id() returns &LockId, not Option, so we just check it's not empty
let lock_id = guard.lock_id();
assert!(!lock_id.uuid.is_empty());
}
}
#[tokio::test]
async fn test_namespace_lock_distributed_multi_node_simulation() {
// Simulate a 3-node distributed environment where each node has its own lock backend
let manager1 = Arc::new(GlobalLockManager::new());
let manager2 = Arc::new(GlobalLockManager::new());
let manager3 = Arc::new(GlobalLockManager::new());
// Create 3 clients, each bound to its own manager (simulating independent nodes)
let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));
let clients = vec![client1, client2, client3];
// Create NamespaceLock with 3 clients (quorum will be 2)
let lock = NamespaceLock::with_clients("multi-node".to_string(), clients);
assert_eq!(lock.namespace(), "multi-node");
let resource = create_test_object_key("test-bucket", "test-object");
// Test 1: Owner A acquires write lock successfully
let mut guard_a = lock
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
.await
.expect("Owner A should acquire write lock");
// Verify it's a Standard guard (DistributedLock path)
match &guard_a {
NamespaceLockGuard::Standard(_) => {
// Expected for distributed lock
}
NamespaceLockGuard::Fast(_) => {
panic!("Expected Standard guard for distributed lock");
}
}
// Test 2: Owner B tries to acquire write lock while A holds it - should fail
// Since all 3 backends are holding locks from owner-a, owner-b cannot acquire on any backend
// This means 0 successes < quorum(2), so acquisition should fail
let result_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
.await;
assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");
// Verify the error is a timeout or quorum failure (since quorum cannot be reached)
if let Err(err) = result_b {
// The error should indicate timeout or quorum failure
let err_str = err.to_string().to_lowercase();
assert!(
err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
"Error should be timeout or quorum related, got: {}",
err
);
}
// Test 3: Release owner A's lock
assert!(guard_a.release(), "Should release guard_a successfully");
assert!(guard_a.is_released(), "Guard A should be marked as released");
// Test 4: Owner B should now be able to acquire the lock
let guard_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
.await
.expect("Owner B should acquire write lock after A releases");
match &guard_b {
NamespaceLockGuard::Standard(_) => {
// Expected for distributed lock
}
NamespaceLockGuard::Fast(_) => {
panic!("Expected Standard guard for distributed lock");
}
}
// Test 5: Verify health check shows 3 nodes
let health = lock.get_health().await;
assert_eq!(health.node_id, "multi-node");
assert_eq!(health.total_nodes, 3);
assert_eq!(health.connected_nodes, 3);
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
// Cleanup
drop(guard_b);
}
#[tokio::test]
async fn test_namespace_lock_distributed_with_clients_and_quorum() {
// Same 3-node setup as multi-node simulation; use explicit quorum via with_clients_and_quorum
let manager1 = Arc::new(GlobalLockManager::new());
let manager2 = Arc::new(GlobalLockManager::new());
let manager3 = Arc::new(GlobalLockManager::new());
let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));
let clients = vec![client1, client2, client3];
// Create NamespaceLock with explicit quorum=2 (same as with_clients default for 3 nodes)
let lock = NamespaceLock::with_clients_and_quorum("multi-node".to_string(), clients, 2);
assert_eq!(lock.namespace(), "multi-node");
let resource = create_test_object_key("test-bucket", "test-object");
// Owner A acquires write lock successfully
let mut guard_a = lock
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
.await
.expect("Owner A should acquire write lock");
match &guard_a {
NamespaceLockGuard::Standard(_) => {}
NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
}
// Owner B tries to acquire while A holds it - should fail (quorum not reached)
let result_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
.await;
assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");
if let Err(err) = result_b {
let err_str = err.to_string().to_lowercase();
assert!(
err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
"Error should be timeout or quorum related, got: {}",
err
);
}
// Release owner A's lock
assert!(guard_a.release(), "Should release guard_a successfully");
assert!(guard_a.is_released(), "Guard A should be marked as released");
// Owner B should now acquire the lock
let guard_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
.await
.expect("Owner B should acquire write lock after A releases");
match &guard_b {
NamespaceLockGuard::Standard(_) => {}
NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
}
// Health check: 3 nodes, Healthy
let health = lock.get_health().await;
assert_eq!(health.node_id, "multi-node");
assert_eq!(health.total_nodes, 3);
assert_eq!(health.connected_nodes, 3);
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
drop(guard_b);
}
crates/lock/src/types.rs
+18 -138
View File
@@ -16,6 +16,8 @@ use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use uuid::Uuid;
use crate::ObjectKey;
/// Lock type enumeration
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum LockType {
@@ -56,7 +58,7 @@ pub struct LockInfo {
/// Unique identifier for the lock
pub id: LockId,
/// Resource path
pub resource: String,
pub resource: ObjectKey,
/// Lock type
pub lock_type: LockType,
/// Lock status
@@ -102,56 +104,27 @@ impl LockInfo {
/// Lock ID type
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct LockId {
pub resource: String,
pub resource: ObjectKey,
pub uuid: String,
}
impl LockId {
/// Generate new lock ID for a resource
pub fn new(resource: &str) -> Self {
pub fn new(resource: ObjectKey) -> Self {
Self {
resource: resource.to_string(),
resource,
uuid: Uuid::new_v4().to_string(),
}
}
/// Generate deterministic lock ID for a resource (same resource = same ID)
pub fn new_deterministic(resource: &str) -> Self {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
resource.hash(&mut hasher);
let hash = hasher.finish();
/// Generate unique lock ID for a resource
/// Each call generates a different ID, even for the same resource
pub fn new_unique(resource: &ObjectKey) -> Self {
// Use UUID v4 (random) to ensure uniqueness
// Each call generates a new unique ID regardless of the resource
Self {
resource: resource.to_string(),
uuid: format!("{hash:016x}"),
}
}
/// Create lock ID from resource and uuid
pub fn from_parts(resource: impl Into<String>, uuid: impl Into<String>) -> Self {
Self {
resource: resource.into(),
uuid: uuid.into(),
}
}
/// Create lock ID from string (for compatibility, expects "resource:uuid")
pub fn from_string(id: impl Into<String>) -> Self {
let s = id.into();
if let Some((resource, uuid)) = s.split_once(":") {
Self {
resource: resource.to_string(),
uuid: uuid.to_string(),
}
} else {
// fallback: treat as uuid only
Self {
resource: "unknown".to_string(),
uuid: s,
}
resource: resource.clone(),
uuid: Uuid::new_v4().to_string(),
}
}
@@ -163,7 +136,7 @@ impl LockId {
impl Default for LockId {
fn default() -> Self {
Self::new("default")
Self::new(ObjectKey::new("default", "default"))
}
}
@@ -237,7 +210,7 @@ pub struct LockRequest {
/// Lock ID
pub lock_id: LockId,
/// Resource path
pub resource: String,
pub resource: ObjectKey,
/// Lock type
pub lock_type: LockType,
/// Lock owner
@@ -256,11 +229,10 @@ pub struct LockRequest {
impl LockRequest {
/// Create new lock request
pub fn new(resource: impl Into<String>, lock_type: LockType, owner: impl Into<String>) -> Self {
let resource_str = resource.into();
pub fn new(resource: ObjectKey, lock_type: LockType, owner: impl Into<String>) -> Self {
Self {
lock_id: LockId::new_deterministic(&resource_str),
resource: resource_str,
lock_id: LockId::new_unique(&resource),
resource,
lock_type,
owner: owner.into(),
acquire_timeout: Duration::from_secs(10), // Default 10 seconds to acquire
@@ -611,95 +583,3 @@ impl WaitQueueItem {
self.wait_start_time.elapsed().unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_lock_id() {
let id1 = LockId::new("test-resource");
let id2 = LockId::new("test-resource");
assert_ne!(id1, id2);
let id3 = LockId::from_string("test-resource:test-uuid");
assert_eq!(id3.as_str(), "test-resource:test-uuid");
}
#[test]
fn test_lock_metadata() {
let metadata = LockMetadata::new()
.with_client_info("test-client")
.with_operation_id("test-op")
.with_priority(1)
.with_tag("key", "value");
assert_eq!(metadata.client_info, Some("test-client".to_string()));
assert_eq!(metadata.operation_id, Some("test-op".to_string()));
assert_eq!(metadata.priority, Some(1));
assert_eq!(metadata.tags.get("key"), Some(&"value".to_string()));
}
#[test]
fn test_lock_request() {
let request = LockRequest::new("test-resource", LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(60))
.with_priority(LockPriority::High)
.with_deadlock_detection(true);
assert_eq!(request.resource, "test-resource");
assert_eq!(request.lock_type, LockType::Exclusive);
assert_eq!(request.owner, "test-owner");
assert_eq!(request.acquire_timeout, Duration::from_secs(60));
assert_eq!(request.priority, LockPriority::High);
assert!(request.deadlock_detection);
}
#[test]
fn test_lock_response() {
let lock_info = LockInfo {
id: LockId::new("test-resource"),
resource: "test".to_string(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "test".to_string(),
acquired_at: SystemTime::now(),
expires_at: SystemTime::now() + Duration::from_secs(30),
last_refreshed: SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
};
let success = LockResponse::success(lock_info.clone(), Duration::ZERO);
assert!(success.is_success());
let failure = LockResponse::failure("error", Duration::ZERO);
assert!(failure.is_failure());
let waiting = LockResponse::waiting(Duration::ZERO, 1);
assert!(waiting.is_waiting());
}
#[test]
fn test_timestamp_conversion() {
let now = SystemTime::now();
let timestamp = system_time_to_timestamp(now);
let converted = timestamp_to_system_time(timestamp);
// Allow for small time differences
let diff = now.duration_since(converted).unwrap_or(Duration::ZERO);
assert!(diff < Duration::from_secs(1));
}
#[test]
fn test_serialization() {
let request = LockRequest::new("test", LockType::Exclusive, "owner");
let serialized = serde_json::to_string(&request).unwrap();
let deserialized: LockRequest = serde_json::from_str(&serialized).unwrap();
assert_eq!(request.resource, deserialized.resource);
assert_eq!(request.lock_type, deserialized.lock_type);
assert_eq!(request.owner, deserialized.owner);
}
}