rustfs/crates/lock/src/namespace/tests.rs

// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use super::*;
use crate::client::{ClientFactory, local::LocalClient};
use crate::types::LockType;
use crate::{GlobalLockManager, LockError, LockInfo, LockResponse, LockStats};
use std::sync::Arc;
use std::time::Duration;

#[derive(Debug, Default)]
struct FailingClient;

#[async_trait::async_trait]
impl crate::client::LockClient for FailingClient {
    async fn acquire_lock(&self, _request: &LockRequest) -> crate::Result<LockResponse> {
        Err(LockError::internal("simulated offline client"))
    }

    async fn release(&self, _lock_id: &LockId) -> crate::Result<bool> {
        Ok(false)
    }

    async fn refresh(&self, _lock_id: &LockId) -> crate::Result<bool> {
        Ok(false)
    }

    async fn force_release(&self, _lock_id: &LockId) -> crate::Result<bool> {
        Ok(false)
    }

    async fn check_status(&self, _lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
        Ok(None)
    }

    async fn get_stats(&self) -> crate::Result<LockStats> {
        Ok(LockStats::default())
    }

    async fn close(&self) -> crate::Result<()> {
        Ok(())
    }

    async fn is_online(&self) -> bool {
        false
    }

    async fn is_local(&self) -> bool {
        false
    }
}

fn create_test_object_key(bucket: &str, object: &str) -> ObjectKey {
    ObjectKey {
        bucket: Arc::from(bucket),
        object: Arc::from(object),
        version: None,
    }
}

#[tokio::test]
async fn test_namespace_lock_new() {
    let client = ClientFactory::create_local();
    let lock = NamespaceLock::new("test-namespace".to_string(), client);
    assert_eq!(lock.namespace(), "test-namespace");
}

#[tokio::test]
async fn test_namespace_lock_with_client() {
    let client = ClientFactory::create_local();
    let lock = NamespaceLock::with_client(client);
    assert_eq!(lock.namespace(), "default");
}

#[tokio::test]
async fn test_namespace_lock_with_local_manager() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("local-ns".to_string(), manager);
    assert_eq!(lock.namespace(), "local-ns");
}

#[tokio::test]
async fn test_namespace_lock_with_clients() {
    let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
    let lock = NamespaceLock::with_clients("multi-client".to_string(), clients);
    assert_eq!(lock.namespace(), "multi-client");
}

#[tokio::test]
async fn test_lock_client_default_batch_acquire_and_release() {
    let manager = Arc::new(GlobalLockManager::new());
    let client = LocalClient::with_manager(manager);
    let requests = vec![
        LockRequest::new(create_test_object_key("bucket", "object-a"), LockType::Exclusive, "owner-a")
            .with_acquire_timeout(Duration::from_secs(1)),
        LockRequest::new(create_test_object_key("bucket", "object-b"), LockType::Exclusive, "owner-a")
            .with_acquire_timeout(Duration::from_secs(1)),
    ];

    let responses = client.acquire_locks_batch(&requests).await.unwrap();
    assert_eq!(responses.len(), requests.len());
    assert!(responses.iter().all(|response| response.success));

    let lock_ids = responses
        .iter()
        .map(|response| {
            response
                .lock_info
                .as_ref()
                .expect("successful batch acquire should return lock info")
                .id
                .clone()
        })
        .collect::<Vec<_>>();
    let released = client.release_locks_batch(&lock_ids).await.unwrap();

    assert_eq!(released, vec![true, true]);
}

#[tokio::test]
async fn test_namespace_lock_get_resource_key() {
    let client = ClientFactory::create_local();
    let lock = NamespaceLock::new("test-ns".to_string(), client);
    let resource = create_test_object_key("bucket", "object");
    let key = lock.get_resource_key(&resource);
    assert!(key.contains("test-ns"));
    assert!(key.contains("bucket"));
    assert!(key.contains("object"));
}

#[tokio::test]
async fn test_namespace_lock_acquire_guard_local() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("test-local".to_string(), manager);
    let resource = create_test_object_key("bucket", "object");

    let request = LockRequest::new(resource.clone(), LockType::Exclusive, "owner1")
        .with_acquire_timeout(Duration::from_secs(5))
        .with_ttl(Duration::from_secs(30));

    let guard_opt = lock.acquire_guard(&request).await.unwrap();
    assert!(guard_opt.is_some());

    if let Some(NamespaceLockGuard::Fast(guard)) = guard_opt {
        assert_eq!(guard.key(), &resource);
        assert!(!guard.is_released());

        // Test release
        let mut guard = guard;
        assert!(guard.release());
        assert!(guard.is_released());
    } else {
        panic!("Expected Fast guard");
    }
}

#[tokio::test]
async fn test_namespace_lock_get_write_lock_local() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("test-write".to_string(), manager);
    let resource = create_test_object_key("bucket", "object");

    let guard = lock
        .get_write_lock(resource.clone(), "owner1", Duration::from_secs(5))
        .await
        .unwrap();

    match guard {
        NamespaceLockGuard::Fast(guard) => {
            assert_eq!(guard.key(), &resource);
            assert!(!guard.is_released());
        }
        NamespaceLockGuard::Standard(_) => {
            panic!("Expected Fast guard for local lock");
        }
    }
}

#[tokio::test]
async fn test_namespace_lock_get_read_lock_local() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("test-read".to_string(), manager);
    let resource = create_test_object_key("bucket", "object");

    let guard = lock
        .get_read_lock(resource.clone(), "owner1", Duration::from_secs(5))
        .await
        .unwrap();

    match guard {
        NamespaceLockGuard::Fast(guard) => {
            assert_eq!(guard.key(), &resource);
            assert!(!guard.is_released());
        }
        NamespaceLockGuard::Standard(_) => {
            panic!("Expected Fast guard for local lock");
        }
    }
}

#[tokio::test]
async fn test_namespace_lock_guard_release() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("test-release".to_string(), manager);
    let resource = create_test_object_key("bucket", "object");

    let mut guard = lock.get_write_lock(resource, "owner1", Duration::from_secs(5)).await.unwrap();

    assert!(!guard.is_released());
    assert!(guard.release());
    assert!(guard.is_released());
}

#[tokio::test]
async fn test_namespace_lock_wrapper() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("wrapper-test".to_string(), manager);
    let resource = create_test_object_key("bucket", "object");
    let wrapper = NamespaceLockWrapper::new(lock, resource.clone(), "owner1".to_string());

    let guard = wrapper.get_write_lock(Duration::from_secs(5)).await.unwrap();

    match guard {
        NamespaceLockGuard::Fast(guard) => {
            assert_eq!(guard.key(), &resource);
        }
        _ => panic!("Expected Fast guard"),
    }
}

#[tokio::test]
async fn test_namespace_lock_get_health_local() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("health-test".to_string(), manager);

    let health = lock.get_health().await;
    assert_eq!(health.node_id, "health-test");
    assert_eq!(health.status, crate::types::HealthStatus::Healthy);
    assert_eq!(health.connected_nodes, 1);
    assert_eq!(health.total_nodes, 1);
}

#[tokio::test]
async fn test_namespace_lock_get_stats_local() {
    let manager = Arc::new(GlobalLockManager::new());
    let lock = NamespaceLock::with_local_manager("stats-test".to_string(), manager);

    let stats = lock.get_stats().await;
    // Local locks don't expose detailed stats, so defaults should be 0
    assert_eq!(stats.successful_acquires, 0);
    assert_eq!(stats.failed_acquires, 0);
}

#[tokio::test]
async fn test_namespace_lock_default() {
    let lock = NamespaceLock::default();
    assert_eq!(lock.namespace(), "default");
}

#[tokio::test]
async fn test_namespace_lock_guard_lock_id() {
    let client = ClientFactory::create_local();
    let lock = NamespaceLock::new("test-id".to_string(), client);
    let resource = create_test_object_key("bucket", "object");

    let request = LockRequest::new(resource, LockType::Exclusive, "owner1")
        .with_acquire_timeout(Duration::from_secs(5))
        .with_ttl(Duration::from_secs(30));

    if let Some(NamespaceLockGuard::Standard(guard)) = lock.acquire_guard(&request).await.unwrap() {
        // lock_id() returns &LockId, not Option, so we just check it's not empty
        let lock_id = guard.lock_id();
        assert!(!lock_id.uuid.is_empty());
    }
}

#[tokio::test]
async fn test_namespace_lock_distributed_multi_node_simulation() {
    // Simulate a 3-node distributed environment where each node has its own lock backend
    let manager1 = Arc::new(GlobalLockManager::new());
    let manager2 = Arc::new(GlobalLockManager::new());
    let manager3 = Arc::new(GlobalLockManager::new());

    // Create 3 clients, each bound to its own manager (simulating independent nodes)
    let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
    let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
    let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));

    let clients = vec![client1, client2, client3];

    // Create NamespaceLock with 3 clients (quorum will be 2)
    let lock = NamespaceLock::with_clients("multi-node".to_string(), clients);
    assert_eq!(lock.namespace(), "multi-node");

    let resource = create_test_object_key("test-bucket", "test-object");

    // Test 1: Owner A acquires write lock successfully
    let mut guard_a = lock
        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
        .await
        .expect("Owner A should acquire write lock");

    // Verify it's a Standard guard (DistributedLock path)
    match &guard_a {
        NamespaceLockGuard::Standard(_) => {
            // Expected for distributed lock
        }
        NamespaceLockGuard::Fast(_) => {
            panic!("Expected Standard guard for distributed lock");
        }
    }

    // Test 2: Owner B tries to acquire write lock while A holds it - should fail
    // Since all 3 backends are holding locks from owner-a, owner-b cannot acquire on any backend
    // This means 0 successes < quorum(2), so acquisition should fail
    let result_b = lock
        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
        .await;

    assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");

    // Verify the error is a timeout or quorum failure (since quorum cannot be reached)
    if let Err(err) = result_b {
        // The error should indicate timeout or quorum failure
        let err_str = err.to_string().to_lowercase();
        assert!(
            err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
            "Error should be timeout or quorum related, got: {}",
            err
        );
    }

    // Test 3: Release owner A's lock
    assert!(guard_a.release(), "Should release guard_a successfully");
    assert!(guard_a.is_released(), "Guard A should be marked as released");

    // Test 4: Owner B should now be able to acquire the lock
    let guard_b = lock
        .get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
        .await
        .expect("Owner B should acquire write lock after A releases");

    match &guard_b {
        NamespaceLockGuard::Standard(_) => {
            // Expected for distributed lock
        }
        NamespaceLockGuard::Fast(_) => {
            panic!("Expected Standard guard for distributed lock");
        }
    }

    // Test 5: Verify health check shows 3 nodes
    let health = lock.get_health().await;
    assert_eq!(health.node_id, "multi-node");
    assert_eq!(health.total_nodes, 3);
    assert_eq!(health.connected_nodes, 3);
    assert_eq!(health.status, crate::types::HealthStatus::Healthy);

    // Cleanup
    drop(guard_b);
}

#[tokio::test]
async fn test_namespace_lock_distributed_with_clients_and_quorum() {
    // Same 3-node setup as multi-node simulation; use explicit quorum via with_clients_and_quorum
    let manager1 = Arc::new(GlobalLockManager::new());
    let manager2 = Arc::new(GlobalLockManager::new());
    let manager3 = Arc::new(GlobalLockManager::new());

    let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
    let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
    let client3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager3));

    let clients = vec![client1, client2, client3];

    // Create NamespaceLock with explicit quorum=2 (same as with_clients default for 3 nodes)
    let lock = NamespaceLock::with_clients_and_quorum("multi-node".to_string(), clients, 2);
    assert_eq!(lock.namespace(), "multi-node");

    let resource = create_test_object_key("test-bucket", "test-object");

    // Owner A acquires write lock successfully
    let mut guard_a = lock
        .get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
        .await
        .expect("Owner A should acquire write lock");

    match &guard_a {
        NamespaceLockGuard::Standard(_) => {}
        NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
    }

    // Owner B tries to acquire while A holds it - should fail (quorum not reached)
    let result_b = lock
        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
        .await;

    assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");

    if let Err(err) = result_b {
        let err_str = err.to_string().to_lowercase();
        assert!(
            err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
            "Error should be timeout or quorum related, got: {}",
            err
        );
    }

    // Release owner A's lock
    assert!(guard_a.release(), "Should release guard_a successfully");
    assert!(guard_a.is_released(), "Guard A should be marked as released");

    // Owner B should now acquire the lock
    let guard_b = lock
        .get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
        .await
        .expect("Owner B should acquire write lock after A releases");

    match &guard_b {
        NamespaceLockGuard::Standard(_) => {}
        NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
    }

    // Health check: 3 nodes, Healthy
    let health = lock.get_health().await;
    assert_eq!(health.node_id, "multi-node");
    assert_eq!(health.total_nodes, 3);
    assert_eq!(health.connected_nodes, 3);
    assert_eq!(health.status, crate::types::HealthStatus::Healthy);

    drop(guard_b);
}

#[tokio::test]
async fn test_namespace_lock_distributed_read_lock_succeeds_with_two_nodes_one_offline() {
    let manager = Arc::new(GlobalLockManager::new());
    let client_ok: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
    let client_offline: Arc<dyn LockClient> = Arc::new(FailingClient);

    let lock = NamespaceLock::with_clients_and_quorum("two-node".to_string(), vec![client_ok, client_offline], 2);
    let resource = create_test_object_key("bucket", "object");

    let guard = lock
        .get_read_lock(resource, "owner-a", Duration::from_millis(100))
        .await
        .expect("read lock should succeed with one healthy node in a two-node cluster");

    match guard {
        NamespaceLockGuard::Standard(_) => {}
        NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
    }
}

#[tokio::test]
async fn test_namespace_lock_distributed_write_lock_fails_with_two_nodes_one_offline() {
    let manager = Arc::new(GlobalLockManager::new());
    let client_ok: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
    let client_offline: Arc<dyn LockClient> = Arc::new(FailingClient);

    let lock = NamespaceLock::with_clients_and_quorum("two-node".to_string(), vec![client_ok, client_offline], 2);
    let resource = create_test_object_key("bucket", "object");

    let err = lock
        .get_write_lock(resource, "owner-a", Duration::from_millis(100))
        .await
        .expect_err("write lock should fail with one healthy node in a two-node cluster");

    let err_str = err.to_string().to_lowercase();
    assert!(
        err_str.contains("quorum") || err_str.contains("not reached"),
        "expected quorum error, got: {err}"
    );
}

#[tokio::test]
async fn test_namespace_lock_distributed_quorum_failure_rolls_back_successful_nodes() {
    let manager1 = Arc::new(GlobalLockManager::new());
    let manager2 = Arc::new(GlobalLockManager::new());

    let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1.clone()));
    let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2.clone()));
    let client3: Arc<dyn LockClient> = Arc::new(FailingClient);

    let resource = create_test_object_key("bucket", "object");

    let distributed_lock = NamespaceLock::with_clients_and_quorum("three-node".to_string(), vec![client1, client2, client3], 3);
    let err = distributed_lock
        .get_write_lock(resource.clone(), "owner-a", Duration::from_millis(100))
        .await
        .expect_err("write lock should fail when quorum requires all three nodes");

    let err_str = err.to_string().to_lowercase();
    assert!(
        err_str.contains("quorum") || err_str.contains("not reached"),
        "expected quorum error, got: {err}"
    );

    let local_lock_1 = NamespaceLock::with_local_manager("node-1".to_string(), manager1);
    let local_lock_2 = NamespaceLock::with_local_manager("node-2".to_string(), manager2);

    let guard1 = local_lock_1
        .get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
        .await
        .expect("quorum rollback should release node 1");
    let guard2 = local_lock_2
        .get_write_lock(resource, "owner-b", Duration::from_millis(100))
        .await
        .expect("quorum rollback should release node 2");

    drop(guard1);
    drop(guard2);
}

#[tokio::test]
async fn test_namespace_lock_distributed_even_node_read_write_quorum_split() {
    let manager1 = Arc::new(GlobalLockManager::new());
    let manager2 = Arc::new(GlobalLockManager::new());

    let client1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager1));
    let client2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager2));
    let client3: Arc<dyn LockClient> = Arc::new(FailingClient);
    let client4: Arc<dyn LockClient> = Arc::new(FailingClient);

    let lock = NamespaceLock::with_clients("four-node".to_string(), vec![client1, client2, client3, client4]);
    let resource = create_test_object_key("bucket", "object");

    let mut read_guard = lock
        .get_read_lock(resource.clone(), "owner-a", Duration::from_millis(100))
        .await
        .expect("read lock should succeed with two healthy nodes in a four-node cluster");

    match &read_guard {
        NamespaceLockGuard::Standard(_) => {}
        NamespaceLockGuard::Fast(_) => panic!("Expected Standard guard for distributed lock"),
    }
    assert!(read_guard.release(), "read guard should release cleanly");

    let err = lock
        .get_write_lock(resource, "owner-a", Duration::from_millis(100))
        .await
        .expect_err("write lock should fail because four-node cluster requires quorum of 3");

    let err_str = err.to_string().to_lowercase();
    assert!(
        err_str.contains("quorum") || err_str.contains("not reached"),
        "expected quorum error, got: {err}"
    );
}