// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::*; use crate::client::{ClientFactory, local::LocalClient}; use crate::types::LockType; use crate::{GlobalLockManager, LockError, LockInfo, LockResponse, LockStats}; use std::sync::Arc; use std::time::Duration; #[derive(Debug, Default)] struct FailingClient; #[async_trait::async_trait] impl crate::client::LockClient for FailingClient { async fn acquire_lock(&self, _request: &LockRequest) -> crate::Result { Err(LockError::internal("simulated offline client")) } async fn release(&self, _lock_id: &LockId) -> crate::Result { Ok(false) } async fn refresh(&self, _lock_id: &LockId) -> crate::Result { Ok(false) } async fn force_release(&self, _lock_id: &LockId) -> crate::Result { Ok(false) } async fn check_status(&self, _lock_id: &LockId) -> crate::Result> { Ok(None) } async fn get_stats(&self) -> crate::Result { Ok(LockStats::default()) } async fn close(&self) -> crate::Result<()> { Ok(()) } async fn is_online(&self) -> bool { false } async fn is_local(&self) -> bool { false } } fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey { ObjectKey { bucket: Arc::from(bucket), object: Arc::from(object), version: None, } } #[tokio::test] async fn test_namespace_lock_new() { let client = ClientFactory::create_local(); let lock = NamespaceLock::new("test-namespace".to_string(), client); assert_eq!(lock.namespace(), "test-namespace"); } #[tokio::test] async fn test_namespace_lock_with_client() { let client = ClientFactory::create_local(); let lock = NamespaceLock::with_client(client); assert_eq!(lock.namespace(), "default"); } #[tokio::test] async fn test_namespace_lock_with_local_manager() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("local-ns".to_string(), manager); assert_eq!(lock.namespace(), "local-ns"); } #[tokio::test] async fn test_namespace_lock_with_clients() { let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()]; let lock = NamespaceLock::with_clients("multi-client".to_string(), clients); assert_eq!(lock.namespace(), "multi-client"); } #[tokio::test] async fn test_lock_client_default_batch_acquire_and_release() { let manager = Arc::new(GlobalLockManager::new()); let client = LocalClient::with_manager(manager); let requests = vec![ LockRequest::new(create_test_object_key("bucket", "object-a"), LockType::Exclusive, "owner-a") .with_acquire_timeout(Duration::from_secs(1)), LockRequest::new(create_test_object_key("bucket", "object-b"), LockType::Exclusive, "owner-a") .with_acquire_timeout(Duration::from_secs(1)), ]; let responses = client.acquire_locks_batch(&requests).await.unwrap(); assert_eq!(responses.len(), requests.len()); assert!(responses.iter().all(|response| response.success)); let lock_ids = responses .iter() .map(|response| { response .lock_info .as_ref() .expect("successful batch acquire should return lock info") .id .clone() }) .collect::>(); let released = client.release_locks_batch(&lock_ids).await.unwrap(); assert_eq!(released, vec![true, true]); } #[tokio::test] async fn test_namespace_lock_get_resource_key() { let client = ClientFactory::create_local(); let lock = NamespaceLock::new("test-ns".to_string(), client); let resource = create_test_object_key("bucket", "object"); let key = lock.get_resource_key(&resource); assert!(key.contains("test-ns")); assert!(key.contains("bucket")); assert!(key.contains("object")); } #[tokio::test] async fn test_namespace_lock_acquire_guard_local() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("test-local".to_string(), manager); let resource = create_test_object_key("bucket", "object"); let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner1") .with_acquire_timeout(Duration::from_secs(5)) .with_ttl(Duration::from_secs(30)); let guard_opt = lock.acquire_guard(&request).await.unwrap(); assert!(guard_opt.is_some()); if let Some(NamespaceLockGuard::Fast(guard)) = guard_opt { assert_eq!(guard.key(), &resource); assert!(!guard.is_released()); // Test release let mut guard = guard; assert!(guard.release()); assert!(guard.is_released()); } else { panic!("Expected Fast guard"); } } #[tokio::test] async fn test_namespace_lock_get_write_lock_local() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("test-write".to_string(), manager); let resource = create_test_object_key("bucket", "object"); let guard = lock .get_write_lock(resource.clone(), "owner1", Duration::from_secs(5)) .await .unwrap(); match guard { NamespaceLockGuard::Fast(guard) => { assert_eq!(guard.key(), &resource); assert!(!guard.is_released()); } NamespaceLockGuard::Standard(_) => { panic!("Expected Fast guard for local lock"); } } } #[tokio::test] async fn test_namespace_lock_get_read_lock_local() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("test-read".to_string(), manager); let resource = create_test_object_key("bucket", "object"); let guard = lock .get_read_lock(resource.clone(), "owner1", Duration::from_secs(5)) .await .unwrap(); match guard { NamespaceLockGuard::Fast(guard) => { assert_eq!(guard.key(), &resource); assert!(!guard.is_released()); } NamespaceLockGuard::Standard(_) => { panic!("Expected Fast guard for local lock"); } } } #[tokio::test] async fn test_namespace_lock_guard_release() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("test-release".to_string(), manager); let resource = create_test_object_key("bucket", "object"); let mut guard = lock.get_write_lock(resource, "owner1", Duration::from_secs(5)).await.unwrap(); assert!(!guard.is_released()); assert!(guard.release()); assert!(guard.is_released()); } #[tokio::test] async fn test_namespace_lock_wrapper() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("wrapper-test".to_string(), manager); let resource = create_test_object_key("bucket", "object"); let wrapper = NamespaceLockWrapper::new(lock, resource.clone(), "owner1".to_string()); let guard = wrapper.get_write_lock(Duration::from_secs(5)).await.unwrap(); match guard { NamespaceLockGuard::Fast(guard) => { assert_eq!(guard.key(), &resource); } _ => panic!("Expected Fast guard"), } } #[tokio::test] async fn test_namespace_lock_get_health_local() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("health-test".to_string(), manager); let health = lock.get_health().await; assert_eq!(health.node_id, "health-test"); assert_eq!(health.status, crate::types::HealthStatus::Healthy); assert_eq!(health.connected_nodes, 1); assert_eq!(health.total_nodes, 1); } #[tokio::test] async fn test_namespace_lock_get_stats_local() { let manager = Arc::new(GlobalLockManager::new()); let lock = NamespaceLock::with_local_manager("stats-test".to_string(), manager); let stats = lock.get_stats().await; // Local locks don't expose detailed stats, so defaults should be 0 assert_eq!(stats.successful_acquires, 0); assert_eq!(stats.failed_acquires, 0); } #[tokio::test] async fn test_namespace_lock_default() { let lock = NamespaceLock::default(); assert_eq!(lock.namespace(), "default"); } #[tokio::test] async fn test_namespace_lock_guard_lock_id() { let client = ClientFactory::create_local(); let lock = NamespaceLock::new("test-id".to_string(), client); let resource = create_test_object_key("bucket", "object"); let request = LockRequest::new(resource, LockType::Exclusive, "owner1") .with_acquire_timeout(Duration::from_secs(5)) .with_ttl(Duration::from_secs(30)); if let Some(NamespaceLockGuard::Standard(guard)) = lock.acquire_guard(&request).await.unwrap() { // lock_id() returns &LockId, not Option, so we just check it's not empty let lock_id = guard.lock_id(); assert!(!lock_id.uuid.is_empty()); } } #[tokio::test] async fn test_namespace_lock_distributed_multi_node_simulation() { // Simulate a 3-node distributed environment where each node has its own lock backend let manager1 = Arc::new(GlobalLockManager::new()); let manager2 = Arc::new(GlobalLockManager::new()); let manager3 = Arc::new(GlobalLockManager::new()); // Create 3 clients, each bound to its own manager (simulating independent nodes) let client1: Arc = Arc::new(LocalClient::with_manager(manager1)); let client2: Arc = Arc::new(LocalClient::with_manager(manager2)); let client3: Arc = Arc::new(LocalClient::with_manager(manager3)); let clients = vec![client1, client2, client3]; // Create NamespaceLock with 3 clients (quorum will be 2) let lock = NamespaceLock::with_clients("multi-node".to_string(), clients); assert_eq!(lock.namespace(), "multi-node"); let resource = create_test_object_key("test-bucket", "test-object"); // Test 1: Owner A acquires write lock successfully let mut guard_a = lock .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5)) .await .expect("Owner A should acquire write lock"); // Verify it's a Standard guard (DistributedLock path) match &guard_a { NamespaceLockGuard::Standard(_) => { // Expected for distributed lock } NamespaceLockGuard::Fast(_) => { panic!("Expected Standard guard for distributed lock"); } } // Test 2: Owner B tries to acquire write lock while A holds it - should fail // Since all 3 backends are holding locks from owner-a, owner-b cannot acquire on any backend // This means 0 successes < quorum(2), so acquisition should fail let result_b = lock .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100)) .await; assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it"); // Verify the error is a timeout or quorum failure (since quorum cannot be reached) if let Err(err) = result_b { // The error should indicate timeout or quorum failure let err_str = err.to_string().to_lowercase(); assert!( err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"), "Error should be timeout or quorum related, got: {}", err ); } // Test 3: Release owner A's lock assert!(guard_a.release(), "Should release guard_a successfully"); assert!(guard_a.is_released(), "Guard A should be marked as released"); // Test 4: Owner B should now be able to acquire the lock let guard_b = lock .get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5)) .await .expect("Owner B should acquire write lock after A releases"); match &guard_b { NamespaceLockGuard::Standard(_) => { // Expected for distributed lock } NamespaceLockGuard::Fast(_) => { panic!("Expected Standard guard for distributed lock"); } } // Test 5: Verify health check shows 3 nodes let health = lock.get_health().await; assert_eq!(health.node_id, "multi-node"); assert_eq!(health.total_nodes, 3); assert_eq!(health.connected_nodes, 3); assert_eq!(health.status, crate::types::HealthStatus::Healthy); // Cleanup drop(guard_b); } #[tokio::test] async fn test_namespace_lock_distributed_with_clients_and_quorum() { // Same 3-node setup as multi-node simulation; use explicit quorum via with_clients_and_quorum let manager1 = Arc::new(GlobalLockManager::new()); let manager2 = Arc::new(GlobalLockManager::new()); let manager3 = Arc::new(GlobalLockManager::new()); let client1: Arc = Arc::new(LocalClient::with_manager(manager1)); let client2: Arc = Arc::new(LocalClient::with_manager(manager2)); let client3: Arc = Arc::new(LocalClient::with_manager(manager3)); let clients = vec![client1, client2, client3]; // Create NamespaceLock with explicit quorum=2 (same as with_clients default for 3 nodes) let lock = NamespaceLock::with_clients_and_quorum("multi-node".to_string(), clients, 2); assert_eq!(lock.namespace(), "multi-node"); let resource = create_test_object_key("test-bucket", "test-object"); // Owner A acquires write lock successfully let mut guard_a = lock .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5)) .await .expect("Owner A should acquire write lock"); match &guard_a { NamespaceLockGuard::Standard(_) => {} NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"), } // Owner B tries to acquire while A holds it - should fail (quorum not reached) let result_b = lock .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100)) .await; assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it"); if let Err(err) = result_b { let err_str = err.to_string().to_lowercase(); assert!( err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"), "Error should be timeout or quorum related, got: {}", err ); } // Release owner A's lock assert!(guard_a.release(), "Should release guard_a successfully"); assert!(guard_a.is_released(), "Guard A should be marked as released"); // Owner B should now acquire the lock let guard_b = lock .get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5)) .await .expect("Owner B should acquire write lock after A releases"); match &guard_b { NamespaceLockGuard::Standard(_) => {} NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"), } // Health check: 3 nodes, Healthy let health = lock.get_health().await; assert_eq!(health.node_id, "multi-node"); assert_eq!(health.total_nodes, 3); assert_eq!(health.connected_nodes, 3); assert_eq!(health.status, crate::types::HealthStatus::Healthy); drop(guard_b); } #[tokio::test] async fn test_namespace_lock_distributed_read_lock_succeeds_with_two_nodes_one_offline() { let manager = Arc::new(GlobalLockManager::new()); let client_ok: Arc = Arc::new(LocalClient::with_manager(manager)); let client_offline: Arc = Arc::new(FailingClient); let lock = NamespaceLock::with_clients_and_quorum("two-node".to_string(), vec![client_ok, client_offline], 2); let resource = create_test_object_key("bucket", "object"); let guard = lock .get_read_lock(resource, "owner-a", Duration::from_millis(100)) .await .expect("read lock should succeed with one healthy node in a two-node cluster"); match guard { NamespaceLockGuard::Standard(_) => {} NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"), } } #[tokio::test] async fn test_namespace_lock_distributed_write_lock_fails_with_two_nodes_one_offline() { let manager = Arc::new(GlobalLockManager::new()); let client_ok: Arc = Arc::new(LocalClient::with_manager(manager)); let client_offline: Arc = Arc::new(FailingClient); let lock = NamespaceLock::with_clients_and_quorum("two-node".to_string(), vec![client_ok, client_offline], 2); let resource = create_test_object_key("bucket", "object"); let err = lock .get_write_lock(resource, "owner-a", Duration::from_millis(100)) .await .expect_err("write lock should fail with one healthy node in a two-node cluster"); let err_str = err.to_string().to_lowercase(); assert!( err_str.contains("quorum") || err_str.contains("not reached"), "expected quorum error, got: {err}" ); } #[tokio::test] async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_nodes() { let manager1 = Arc::new(GlobalLockManager::new()); let manager2 = Arc::new(GlobalLockManager::new()); let client1: Arc = Arc::new(LocalClient::with_manager(manager1.clone())); let client2: Arc = Arc::new(LocalClient::with_manager(manager2.clone())); let client3: Arc = Arc::new(FailingClient); let resource = create_test_object_key("bucket", "object"); let distributed_lock = NamespaceLock::with_clients_and_quorum("three-node".to_string(), vec![client1, client2, client3], 3); let err = distributed_lock .get_write_lock(resource.clone(), "owner-a", Duration::from_millis(100)) .await .expect_err("write lock should fail when quorum requires all three nodes"); let err_str = err.to_string().to_lowercase(); assert!( err_str.contains("quorum") || err_str.contains("not reached"), "expected quorum error, got: {err}" ); let local_lock_1 = NamespaceLock::with_local_manager("node-1".to_string(), manager1); let local_lock_2 = NamespaceLock::with_local_manager("node-2".to_string(), manager2); let guard1 = local_lock_1 .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100)) .await .expect("quorum rollback should release node 1"); let guard2 = local_lock_2 .get_write_lock(resource, "owner-b", Duration::from_millis(100)) .await .expect("quorum rollback should release node 2"); drop(guard1); drop(guard2); } #[tokio::test] async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() { let manager1 = Arc::new(GlobalLockManager::new()); let manager2 = Arc::new(GlobalLockManager::new()); let client1: Arc = Arc::new(LocalClient::with_manager(manager1)); let client2: Arc = Arc::new(LocalClient::with_manager(manager2)); let client3: Arc = Arc::new(FailingClient); let client4: Arc = Arc::new(FailingClient); let lock = NamespaceLock::with_clients("four-node".to_string(), vec![client1, client2, client3, client4]); let resource = create_test_object_key("bucket", "object"); let mut read_guard = lock .get_read_lock(resource.clone(), "owner-a", Duration::from_millis(100)) .await .expect("read lock should succeed with two healthy nodes in a four-node cluster"); match &read_guard { NamespaceLockGuard::Standard(_) => {} NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"), } assert!(read_guard.release(), "read guard should release cleanly"); let err = lock .get_write_lock(resource, "owner-a", Duration::from_millis(100)) .await .expect_err("write lock should fail because four-node cluster requires quorum of 3"); let err_str = err.to_string().to_lowercase(); assert!( err_str.contains("quorum") || err_str.contains("not reached"), "expected quorum error, got: {err}" ); }