mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-10 07:06:53 +00:00
refactor: NamespaceLock (nslock), AHM→Heal Crate, and Lock/Clippy Fixes (#1664)
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: weisd <2057561+weisd@users.noreply.github.com> Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
weisd
@@ -0,0 +1,336 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{
|
||||
ObjectKey,
|
||||
client::LockClient,
|
||||
distributed_lock::{DistributedLock, DistributedLockGuard},
|
||||
error::Result,
|
||||
fast_lock::FastLockGuard,
|
||||
local_lock::LocalLock,
|
||||
types::{LockId, LockRequest},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// Unified guard for namespace locks
|
||||
/// Supports both DistributedLockGuard (for Distributed locks) and FastLockGuard (for Local locks)
|
||||
#[derive(Debug)]
|
||||
pub enum NamespaceLockGuard {
|
||||
/// Standard guard for Distributed locks
|
||||
Standard(DistributedLockGuard),
|
||||
/// Fast guard for Local locks using GlobalLockManager
|
||||
Fast(FastLockGuard),
|
||||
}
|
||||
|
||||
/// Wrapper for NamespaceLock that provides convenient lock acquisition methods
|
||||
/// This wrapper holds the lock instance and resource information for easy lock acquisition
|
||||
#[derive(Debug)]
|
||||
pub struct NamespaceLockWrapper {
|
||||
lock: NamespaceLock,
|
||||
resource: ObjectKey,
|
||||
owner: String,
|
||||
}
|
||||
|
||||
impl NamespaceLockWrapper {
|
||||
/// Create a new wrapper with the lock, resource, and owner
|
||||
pub fn new(lock: NamespaceLock, resource: ObjectKey, owner: String) -> Self {
|
||||
Self { lock, resource, owner }
|
||||
}
|
||||
|
||||
/// Acquire write lock (exclusive lock) with timeout
|
||||
/// Returns the guard if acquisition succeeds, or an error if it fails
|
||||
pub async fn get_write_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
|
||||
self.lock.get_write_lock(self.resource.clone(), &self.owner, timeout).await
|
||||
}
|
||||
|
||||
/// Acquire read lock (shared lock) with timeout
|
||||
/// Returns the guard if acquisition succeeds, or an error if it fails
|
||||
pub async fn get_read_lock(&self, timeout: Duration) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
|
||||
self.lock.get_read_lock(self.resource.clone(), &self.owner, timeout).await
|
||||
}
|
||||
}
|
||||
|
||||
impl NamespaceLockGuard {
|
||||
/// Get the lock ID if available (only for Standard guards)
|
||||
pub fn lock_id(&self) -> Option<&LockId> {
|
||||
match self {
|
||||
Self::Standard(guard) => Some(guard.lock_id()),
|
||||
Self::Fast(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the object key if available (only for Fast guards)
|
||||
pub fn key(&self) -> Option<&ObjectKey> {
|
||||
match self {
|
||||
Self::Standard(_) => None,
|
||||
Self::Fast(guard) => Some(guard.key()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
pub fn release(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => {
|
||||
// DistributedLockGuard::release() actually releases the lock and then disarms
|
||||
guard.release()
|
||||
}
|
||||
Self::Fast(guard) => guard.release(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the lock has been released
|
||||
pub fn is_released(&self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => {
|
||||
// Check if the guard has been disarmed, which indicates the lock was released
|
||||
guard.is_disarmed()
|
||||
}
|
||||
Self::Fast(guard) => guard.is_released(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Namespace lock for managing locks by resource namespaces
|
||||
/// Supports DistributedLock and LocalLock
|
||||
#[derive(Debug)]
|
||||
pub enum NamespaceLock {
|
||||
/// Distributed lock (distributed use case)
|
||||
Distributed(DistributedLock),
|
||||
/// Local lock using GlobalLockManager (high-performance local locking)
|
||||
Local(LocalLock),
|
||||
}
|
||||
|
||||
impl NamespaceLock {
|
||||
/// Create new namespace lock with single client (local use case)
|
||||
/// Uses DistributedLock with quorum=1 for single client
|
||||
pub fn new(namespace: String, client: Arc<dyn LockClient>) -> Self {
|
||||
Self::Distributed(DistributedLock::new(namespace, vec![client], 1))
|
||||
}
|
||||
|
||||
/// Create namespace lock with client (compatibility)
|
||||
/// Uses DistributedLock with quorum=1 for single client
|
||||
pub fn with_client(client: Arc<dyn LockClient>) -> Self {
|
||||
Self::Distributed(DistributedLock::new("default".to_string(), vec![client], 1))
|
||||
}
|
||||
|
||||
/// Create namespace lock with GlobalLockManager (high-performance local locking)
|
||||
pub fn with_local_manager(namespace: String, manager: Arc<crate::GlobalLockManager>) -> Self {
|
||||
Self::Local(LocalLock::new(namespace, manager))
|
||||
}
|
||||
|
||||
/// Create namespace lock with clients
|
||||
/// Uses DistributedLock with appropriate quorum
|
||||
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
|
||||
// Multiple clients: use DistributedLock with majority quorum
|
||||
let quorum = if clients.len() > 1 { (clients.len() / 2) + 1 } else { 1 };
|
||||
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Create namespace lock with clients and an explicit quorum size.
|
||||
/// Quorum will be clamped into [1, clients.len()].
|
||||
pub fn with_clients_and_quorum(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Get namespace identifier
|
||||
pub fn namespace(&self) -> &str {
|
||||
match self {
|
||||
Self::Distributed(lock) => lock.namespace(),
|
||||
Self::Local(lock) => lock.namespace(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get resource key for this namespace
|
||||
pub fn get_resource_key(&self, resource: &ObjectKey) -> String {
|
||||
match self {
|
||||
Self::Distributed(lock) => lock.get_resource_key(resource),
|
||||
Self::Local(lock) => lock.get_resource_key(resource),
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a lock and return a RAII guard that will release asynchronously on Drop.
|
||||
/// This is a thin wrapper around `acquire_lock` and will only create a guard when acquisition succeeds.
|
||||
pub async fn acquire_guard(&self, request: &LockRequest) -> Result<Option<NamespaceLockGuard>> {
|
||||
match self {
|
||||
Self::Distributed(lock) => lock
|
||||
.acquire_guard(request)
|
||||
.await
|
||||
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
|
||||
Self::Local(lock) => lock.acquire_guard(request).await.map(|opt| opt.map(NamespaceLockGuard::Fast)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: acquire exclusive lock as a guard
|
||||
pub async fn lock_guard(
|
||||
&self,
|
||||
resource: ObjectKey,
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
ttl: Duration,
|
||||
) -> Result<Option<NamespaceLockGuard>> {
|
||||
match self {
|
||||
Self::Distributed(lock) => lock
|
||||
.lock_guard(resource, owner, timeout, ttl)
|
||||
.await
|
||||
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
|
||||
Self::Local(lock) => lock
|
||||
.lock_guard(resource, owner, timeout, ttl)
|
||||
.await
|
||||
.map(|opt| opt.map(NamespaceLockGuard::Fast)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: acquire shared lock as a guard
|
||||
pub async fn rlock_guard(
|
||||
&self,
|
||||
resource: ObjectKey,
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
ttl: Duration,
|
||||
) -> Result<Option<NamespaceLockGuard>> {
|
||||
match self {
|
||||
Self::Distributed(lock) => lock
|
||||
.rlock_guard(resource, owner, timeout, ttl)
|
||||
.await
|
||||
.map(|opt| opt.map(NamespaceLockGuard::Standard)),
|
||||
Self::Local(lock) => lock
|
||||
.rlock_guard(resource, owner, timeout, ttl)
|
||||
.await
|
||||
.map(|opt| opt.map(NamespaceLockGuard::Fast)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire write lock (exclusive lock) with timeout
|
||||
/// Returns the guard if acquisition succeeds, or an error if it fails
|
||||
pub async fn get_write_lock(
|
||||
&self,
|
||||
resource: ObjectKey,
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
|
||||
let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
|
||||
let resource_str = format!("{}", resource);
|
||||
match self.lock_guard(resource, owner, timeout, ttl).await {
|
||||
Ok(Some(guard)) => Ok(guard),
|
||||
Ok(None) => {
|
||||
// None can mean timeout or other failure - check if it's a quorum error
|
||||
// For distributed locks, quorum errors are already converted to LockError::QuorumNotReached
|
||||
// So if we get None here, it's likely a timeout
|
||||
Err(crate::error::LockError::timeout(resource_str, timeout))
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire read lock (shared lock) with timeout
|
||||
/// Returns the guard if acquisition succeeds, or an error if it fails
|
||||
pub async fn get_read_lock(
|
||||
&self,
|
||||
resource: ObjectKey,
|
||||
owner: &str,
|
||||
timeout: Duration,
|
||||
) -> std::result::Result<NamespaceLockGuard, crate::error::LockError> {
|
||||
let ttl = crate::fast_lock::DEFAULT_LOCK_TIMEOUT;
|
||||
let resource_str = format!("{}", resource);
|
||||
match self.rlock_guard(resource, owner, timeout, ttl).await {
|
||||
Ok(Some(guard)) => Ok(guard),
|
||||
Ok(None) => Err(crate::error::LockError::timeout(resource_str, timeout)),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get health information
|
||||
pub async fn get_health(&self) -> crate::types::HealthInfo {
|
||||
let lock_stats = self.get_stats().await;
|
||||
let namespace = self.namespace().to_string();
|
||||
let mut health = crate::types::HealthInfo {
|
||||
node_id: namespace.clone(),
|
||||
lock_stats,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
match self {
|
||||
Self::Distributed(lock) => {
|
||||
// Check client status - parallelize async calls for better performance
|
||||
let clients = lock.clients();
|
||||
let client_checks: Vec<_> = clients.iter().map(|client| client.is_online()).collect();
|
||||
|
||||
let results = futures::future::join_all(client_checks).await;
|
||||
let connected_clients = results.iter().filter(|&&online| online).count();
|
||||
|
||||
let quorum = if clients.len() > 1 { (clients.len() / 2) + 1 } else { 1 };
|
||||
|
||||
health.status = if connected_clients >= quorum {
|
||||
crate::types::HealthStatus::Healthy
|
||||
} else {
|
||||
crate::types::HealthStatus::Degraded
|
||||
};
|
||||
health.connected_nodes = connected_clients;
|
||||
health.total_nodes = clients.len();
|
||||
}
|
||||
Self::Local(_) => {
|
||||
// Local locks are always healthy (they use GlobalLockManager which is always available)
|
||||
health.status = crate::types::HealthStatus::Healthy;
|
||||
health.connected_nodes = 1;
|
||||
health.total_nodes = 1;
|
||||
}
|
||||
}
|
||||
|
||||
health
|
||||
}
|
||||
|
||||
/// Get namespace statistics
|
||||
pub async fn get_stats(&self) -> crate::types::LockStats {
|
||||
let mut stats = crate::types::LockStats::default();
|
||||
|
||||
match self {
|
||||
Self::Distributed(lock) => {
|
||||
// Parallelize stats collection for better performance
|
||||
let stats_futures: Vec<_> = lock.clients().iter().map(|client| client.get_stats()).collect();
|
||||
|
||||
let results = futures::future::join_all(stats_futures).await;
|
||||
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(client_stats) => {
|
||||
stats.successful_acquires += client_stats.successful_acquires;
|
||||
stats.failed_acquires += client_stats.failed_acquires;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!("Failed to get stats from client: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Self::Local(_) => {
|
||||
// Local locks use GlobalLockManager which doesn't expose detailed stats
|
||||
// Stats are tracked internally but not exposed through the same interface
|
||||
// We leave stats at default (0) for now
|
||||
}
|
||||
}
|
||||
|
||||
stats
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NamespaceLock {
|
||||
fn default() -> Self {
|
||||
use crate::client::ClientFactory;
|
||||
Self::new("default".to_string(), ClientFactory::create_local())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::GlobalLockManager;
|
||||
use crate::client::{ClientFactory, local::LocalClient};
|
||||
use crate::types::LockType;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
|
||||
ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_new() {
|
||||
let client = ClientFactory::create_local();
|
||||
let lock = NamespaceLock::new("test-namespace".to_string(), client);
|
||||
assert_eq!(lock.namespace(), "test-namespace");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_with_client() {
|
||||
let client = ClientFactory::create_local();
|
||||
let lock = NamespaceLock::with_client(client);
|
||||
assert_eq!(lock.namespace(), "default");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_with_local_manager() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("local-ns".to_string(), manager);
|
||||
assert_eq!(lock.namespace(), "local-ns");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_with_clients() {
|
||||
let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
|
||||
let lock = NamespaceLock::with_clients("multi-client".to_string(), clients);
|
||||
assert_eq!(lock.namespace(), "multi-client");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_get_resource_key() {
|
||||
let client = ClientFactory::create_local();
|
||||
let lock = NamespaceLock::new("test-ns".to_string(), client);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
let key = lock.get_resource_key(&resource);
|
||||
assert!(key.contains("test-ns"));
|
||||
assert!(key.contains("bucket"));
|
||||
assert!(key.contains("object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_acquire_guard_local() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("test-local".to_string(), manager);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner1")
|
||||
.with_acquire_timeout(Duration::from_secs(5))
|
||||
.with_ttl(Duration::from_secs(30));
|
||||
|
||||
let guard_opt = lock.acquire_guard(&request).await.unwrap();
|
||||
assert!(guard_opt.is_some());
|
||||
|
||||
if let Some(NamespaceLockGuard::Fast(guard)) = guard_opt {
|
||||
assert_eq!(guard.key(), &resource);
|
||||
assert!(!guard.is_released());
|
||||
|
||||
// Test release
|
||||
let mut guard = guard;
|
||||
assert!(guard.release());
|
||||
assert!(guard.is_released());
|
||||
} else {
|
||||
panic!("Expected Fast guard");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_get_write_lock_local() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("test-write".to_string(), manager);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let guard = lock
|
||||
.get_write_lock(resource.clone(), "owner1", Duration::from_secs(5))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match guard {
|
||||
NamespaceLockGuard::Fast(guard) => {
|
||||
assert_eq!(guard.key(), &resource);
|
||||
assert!(!guard.is_released());
|
||||
}
|
||||
NamespaceLockGuard::Standard(_) => {
|
||||
panic!("Expected Fast guard for local lock");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_get_read_lock_local() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("test-read".to_string(), manager);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let guard = lock
|
||||
.get_read_lock(resource.clone(), "owner1", Duration::from_secs(5))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match guard {
|
||||
NamespaceLockGuard::Fast(guard) => {
|
||||
assert_eq!(guard.key(), &resource);
|
||||
assert!(!guard.is_released());
|
||||
}
|
||||
NamespaceLockGuard::Standard(_) => {
|
||||
panic!("Expected Fast guard for local lock");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_guard_release() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("test-release".to_string(), manager);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let mut guard = lock.get_write_lock(resource, "owner1", Duration::from_secs(5)).await.unwrap();
|
||||
|
||||
assert!(!guard.is_released());
|
||||
assert!(guard.release());
|
||||
assert!(guard.is_released());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_wrapper() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("wrapper-test".to_string(), manager);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
let wrapper = NamespaceLockWrapper::new(lock, resource.clone(), "owner1".to_string());
|
||||
|
||||
let guard = wrapper.get_write_lock(Duration::from_secs(5)).await.unwrap();
|
||||
|
||||
match guard {
|
||||
NamespaceLockGuard::Fast(guard) => {
|
||||
assert_eq!(guard.key(), &resource);
|
||||
}
|
||||
_ => panic!("Expected Fast guard"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_get_health_local() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("health-test".to_string(), manager);
|
||||
|
||||
let health = lock.get_health().await;
|
||||
assert_eq!(health.node_id, "health-test");
|
||||
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
|
||||
assert_eq!(health.connected_nodes, 1);
|
||||
assert_eq!(health.total_nodes, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_get_stats_local() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let lock = NamespaceLock::with_local_manager("stats-test".to_string(), manager);
|
||||
|
||||
let stats = lock.get_stats().await;
|
||||
// Local locks don't expose detailed stats, so defaults should be 0
|
||||
assert_eq!(stats.successful_acquires, 0);
|
||||
assert_eq!(stats.failed_acquires, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_default() {
|
||||
let lock = NamespaceLock::default();
|
||||
assert_eq!(lock.namespace(), "default");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_guard_lock_id() {
|
||||
let client = ClientFactory::create_local();
|
||||
let lock = NamespaceLock::new("test-id".to_string(), client);
|
||||
let resource = create_test_object_key("bucket", "object");
|
||||
|
||||
let request = LockRequest::new(resource, LockType::Exclusive, "owner1")
|
||||
.with_acquire_timeout(Duration::from_secs(5))
|
||||
.with_ttl(Duration::from_secs(30));
|
||||
|
||||
if let Some(NamespaceLockGuard::Standard(guard)) = lock.acquire_guard(&request).await.unwrap() {
|
||||
// lock_id() returns &LockId, not Option, so we just check it's not empty
|
||||
let lock_id = guard.lock_id();
|
||||
assert!(!lock_id.uuid.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_multi_node_simulation() {
|
||||
// Simulate a 3-node distributed environment where each node has its own lock backend
|
||||
let manager1 = Arc::new(GlobalLockManager::new());
|
||||
let manager2 = Arc::new(GlobalLockManager::new());
|
||||
let manager3 = Arc::new(GlobalLockManager::new());
|
||||
|
||||
// Create 3 clients, each bound to its own manager (simulating independent nodes)
|
||||
let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
|
||||
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
|
||||
let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));
|
||||
|
||||
let clients = vec![client1, client2, client3];
|
||||
|
||||
// Create NamespaceLock with 3 clients (quorum will be 2)
|
||||
let lock = NamespaceLock::with_clients("multi-node".to_string(), clients);
|
||||
assert_eq!(lock.namespace(), "multi-node");
|
||||
|
||||
let resource = create_test_object_key("test-bucket", "test-object");
|
||||
|
||||
// Test 1: Owner A acquires write lock successfully
|
||||
let mut guard_a = lock
|
||||
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("Owner A should acquire write lock");
|
||||
|
||||
// Verify it's a Standard guard (DistributedLock path)
|
||||
match &guard_a {
|
||||
NamespaceLockGuard::Standard(_) => {
|
||||
// Expected for distributed lock
|
||||
}
|
||||
NamespaceLockGuard::Fast(_) => {
|
||||
panic!("Expected Standard guard for distributed lock");
|
||||
}
|
||||
}
|
||||
|
||||
// Test 2: Owner B tries to acquire write lock while A holds it - should fail
|
||||
// Since all 3 backends are holding locks from owner-a, owner-b cannot acquire on any backend
|
||||
// This means 0 successes < quorum(2), so acquisition should fail
|
||||
let result_b = lock
|
||||
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
|
||||
.await;
|
||||
|
||||
assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");
|
||||
|
||||
// Verify the error is a timeout or quorum failure (since quorum cannot be reached)
|
||||
if let Err(err) = result_b {
|
||||
// The error should indicate timeout or quorum failure
|
||||
let err_str = err.to_string().to_lowercase();
|
||||
assert!(
|
||||
err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
|
||||
"Error should be timeout or quorum related, got: {}",
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
// Test 3: Release owner A's lock
|
||||
assert!(guard_a.release(), "Should release guard_a successfully");
|
||||
assert!(guard_a.is_released(), "Guard A should be marked as released");
|
||||
|
||||
// Test 4: Owner B should now be able to acquire the lock
|
||||
let guard_b = lock
|
||||
.get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("Owner B should acquire write lock after A releases");
|
||||
|
||||
match &guard_b {
|
||||
NamespaceLockGuard::Standard(_) => {
|
||||
// Expected for distributed lock
|
||||
}
|
||||
NamespaceLockGuard::Fast(_) => {
|
||||
panic!("Expected Standard guard for distributed lock");
|
||||
}
|
||||
}
|
||||
|
||||
// Test 5: Verify health check shows 3 nodes
|
||||
let health = lock.get_health().await;
|
||||
assert_eq!(health.node_id, "multi-node");
|
||||
assert_eq!(health.total_nodes, 3);
|
||||
assert_eq!(health.connected_nodes, 3);
|
||||
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
|
||||
|
||||
// Cleanup
|
||||
drop(guard_b);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_with_clients_and_quorum() {
|
||||
// Same 3-node setup as multi-node simulation; use explicit quorum via with_clients_and_quorum
|
||||
let manager1 = Arc::new(GlobalLockManager::new());
|
||||
let manager2 = Arc::new(GlobalLockManager::new());
|
||||
let manager3 = Arc::new(GlobalLockManager::new());
|
||||
|
||||
let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
|
||||
let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
|
||||
let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));
|
||||
|
||||
let clients = vec![client1, client2, client3];
|
||||
|
||||
// Create NamespaceLock with explicit quorum=2 (same as with_clients default for 3 nodes)
|
||||
let lock = NamespaceLock::with_clients_and_quorum("multi-node".to_string(), clients, 2);
|
||||
assert_eq!(lock.namespace(), "multi-node");
|
||||
|
||||
let resource = create_test_object_key("test-bucket", "test-object");
|
||||
|
||||
// Owner A acquires write lock successfully
|
||||
let mut guard_a = lock
|
||||
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("Owner A should acquire write lock");
|
||||
|
||||
match &guard_a {
|
||||
NamespaceLockGuard::Standard(_) => {}
|
||||
NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
|
||||
}
|
||||
|
||||
// Owner B tries to acquire while A holds it - should fail (quorum not reached)
|
||||
let result_b = lock
|
||||
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
|
||||
.await;
|
||||
|
||||
assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");
|
||||
|
||||
if let Err(err) = result_b {
|
||||
let err_str = err.to_string().to_lowercase();
|
||||
assert!(
|
||||
err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
|
||||
"Error should be timeout or quorum related, got: {}",
|
||||
err
|
||||
);
|
||||
}
|
||||
|
||||
// Release owner A's lock
|
||||
assert!(guard_a.release(), "Should release guard_a successfully");
|
||||
assert!(guard_a.is_released(), "Guard A should be marked as released");
|
||||
|
||||
// Owner B should now acquire the lock
|
||||
let guard_b = lock
|
||||
.get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("Owner B should acquire write lock after A releases");
|
||||
|
||||
match &guard_b {
|
||||
NamespaceLockGuard::Standard(_) => {}
|
||||
NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
|
||||
}
|
||||
|
||||
// Health check: 3 nodes, Healthy
|
||||
let health = lock.get_health().await;
|
||||
assert_eq!(health.node_id, "multi-node");
|
||||
assert_eq!(health.total_nodes, 3);
|
||||
assert_eq!(health.connected_nodes, 3);
|
||||
assert_eq!(health.status, crate::types::HealthStatus::Healthy);
|
||||
|
||||
drop(guard_b);
|
||||
}
|
||||
Reference in New Issue
Block a user