diff --git a/Cargo.lock b/Cargo.lock index 1f4237e7f..0613b2f8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -504,9 +504,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1237c0ae75a0f3765f58910ff9cdd0a12eeb39ab2f4c7de23262f337f0aacbb3" +checksum = "19634d6336019ef220f09fd31168ce5c184b295cbf80345437cc36094ef223ca" dependencies = [ "async-lock", "cfg-if", @@ -517,8 +517,7 @@ dependencies = [ "polling", "rustix 1.0.8", "slab", - "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -534,9 +533,9 @@ dependencies = [ [[package]] name = "async-process" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde3f4e40e6021d7acffc90095cbd6dc54cb593903d1de5832f435eb274b85dc" +checksum = "65daa13722ad51e6ab1a1b9c01299142bc75135b337923cfa10e79bbbd669f00" dependencies = [ "async-channel", "async-io", @@ -548,7 +547,6 @@ dependencies = [ "event-listener", "futures-lite", "rustix 1.0.8", - "tracing", ] [[package]] @@ -564,9 +562,9 @@ dependencies = [ [[package]] name = "async-signal" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7605a4e50d4b06df3898d5a70bf5fde51ed9059b0434b73105193bc27acce0d" +checksum = "f567af260ef69e1d52c2b560ce0ea230763e6fbb9214a85d768760a920e3e3c1" dependencies = [ "async-io", "async-lock", @@ -577,7 +575,7 @@ dependencies = [ "rustix 1.0.8", "signal-hook-registry", "slab", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -688,9 +686,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08b5d4e069cbc868041a64bd68dc8cb39a0d79585cd6c5a24caa8c2d622121be" +checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" dependencies = [ "aws-lc-sys", "zeroize", @@ -760,7 +758,7 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "lru", + "lru 0.12.5", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -908,9 +906,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3aaec682eb189e43c8a19c3dab2fe54590ad5f2cc2d26ab27608a20f2acf81c" +checksum = "660f70d9d8af6876b4c9aa8dcb0dbaf0f89b04ee9a4455bea1b4ba03b15f26f6" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -932,9 +930,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852b9226cb60b78ce9369022c0df678af1cac231c882d5da97a0c4e03be6e67" +checksum = "38280ac228bc479f347fcfccf4bf4d22d68f3bb4629685cb591cabd856567bbc" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1479,9 +1477,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.29" +version = "1.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" dependencies = [ "jobserver", "libc", @@ -3479,6 +3477,7 @@ dependencies = [ "rustfs-protos", "serde", "serde_json", + "serial_test", "tokio", "tonic", "url", @@ -4957,9 +4956,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-uring" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" dependencies = [ "bitflags 2.9.1", "cfg-if", @@ -5361,13 +5360,13 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" +checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0" dependencies = [ "bitflags 2.9.1", "libc", - "redox_syscall 0.5.13", + "redox_syscall 0.5.15", ] [[package]] @@ -5484,6 +5483,15 @@ dependencies = [ "hashbrown 0.15.4", ] +[[package]] +name = "lru" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ea4e65087ff52f3862caff188d489f1fab49a0cb09e01b2e3f1a617b10aaed" +dependencies = [ + "hashbrown 0.15.4", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -6567,7 +6575,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.13", + "redox_syscall 0.5.15", "smallvec", "windows-targets 0.52.6", ] @@ -6946,17 +6954,16 @@ dependencies = [ [[package]] name = "polling" -version = "3.8.0" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50" +checksum = "8ee9b2fa7a4517d2c91ff5bc6c297a427a96749d15f98fcdbb22c05571a4d4b7" dependencies = [ "cfg-if", "concurrent-queue", "hermit-abi", "pin-project-lite", "rustix 1.0.8", - "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7474,9 +7481,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" dependencies = [ "bitflags 2.9.1", ] @@ -8133,10 +8140,16 @@ name = "rustfs-lock" version = "0.0.5" dependencies = [ "async-trait", + "bytes", + "futures", + "lazy_static", + "lru 0.16.0", + "once_cell", "rand 0.9.2", "rustfs-protos", "serde", "serde_json", + "thiserror 2.0.12", "tokio", "tonic", "tracing", @@ -8653,6 +8666,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" +dependencies = [ + "sdd", +] + [[package]] name = "schannel" version = "0.1.27" @@ -8684,6 +8706,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + [[package]] name = "sec1" version = "0.3.0" @@ -8941,6 +8969,31 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "server_fn" version = "0.6.15" @@ -9417,14 +9470,13 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "rustversion", "syn 2.0.104", ] @@ -10905,9 +10957,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index 2877b244c..84e2e9c76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,7 @@ keyring = { version = "3.6.2", features = [ ] } lazy_static = "1.5.0" libsystemd = { version = "0.7.2" } +lru = "0.16" local-ip-address = "0.6.5" lz4 = "1.28.1" matchit = "0.8.4" diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index b93ff4882..4edaffcd7 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -38,3 +38,4 @@ url.workspace = true rustfs-madmin.workspace = true rustfs-filemeta.workspace = true bytes.workspace = true +serial_test = "3.2.0" diff --git a/crates/e2e_test/src/reliant/lock.rs b/crates/e2e_test/src/reliant/lock.rs index 3b8b04417..f4022dd7b 100644 --- a/crates/e2e_test/src/reliant/lock.rs +++ b/crates/e2e_test/src/reliant/lock.rs @@ -13,28 +13,42 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_lock::{ - drwmutex::Options, - lock_args::LockArgs, - namespace_lock::{NsLockMap, new_nslock}, - new_lock_api, -}; +use rustfs_ecstore::{disk::endpoint::Endpoint, lock_utils::create_unique_clients}; +use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType}; +use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager}; use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; -use std::{error::Error, sync::Arc, time::Duration}; -use tokio::sync::RwLock; +use serial_test::serial; +use std::{error::Error, time::Duration}; +use tokio::time::sleep; use tonic::Request; +use url::Url; const CLUSTER_ADDR: &str = "http://localhost:9000"; +fn get_cluster_endpoints() -> Vec { + vec![Endpoint { + url: Url::parse(CLUSTER_ADDR).unwrap(), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }] +} + #[tokio::test] +#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_lock_unlock_rpc() -> Result<(), Box> { - let args = LockArgs { - uid: "1111".to_string(), - resources: vec!["dandan".to_string()], + let args = LockRequest { + lock_id: LockId::new_deterministic("dandan"), + resource: "dandan".to_string(), + lock_type: LockType::Exclusive, owner: "dd".to_string(), - source: "".to_string(), - quorum: 3, + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, }; let args = serde_json::to_string(&args)?; @@ -59,30 +73,405 @@ async fn test_lock_unlock_rpc() -> Result<(), Box> { } #[tokio::test] +#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_lock_unlock_ns_lock() -> Result<(), Box> { - let url = url::Url::parse("http://127.0.0.1:9000/data")?; - let locker = new_lock_api(false, Some(url)); - let ns_mutex = Arc::new(RwLock::new(NsLockMap::new(true))); - let ns = new_nslock( - Arc::clone(&ns_mutex), - "local".to_string(), - "dandan".to_string(), - vec!["foo".to_string()], - vec![locker], - ) - .await; - assert!( - ns.0.write() - .await - .get_lock(&Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_secs(1), - }) - .await - .unwrap() - ); + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock = NamespaceLock::with_clients("test".to_string(), clients); + + let resources = vec!["foo".to_string()]; + let result = ns_lock + .lock_batch(&resources, "dandan", Duration::from_secs(5), Duration::from_secs(10)) + .await; + match &result { + Ok(success) => println!("Lock result: {success}"), + Err(e) => println!("Lock error: {e}"), + } + let result = result?; + assert!(result, "Lock should succeed, but got: {result}"); + + ns_lock.unlock_batch(&resources, "dandan").await?; + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_concurrent_lock_attempts() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock = NamespaceLock::with_clients("test".to_string(), clients); + let resource = vec!["concurrent_resource".to_string()]; + + // First lock should succeed + println!("Attempting first lock..."); + let result1 = ns_lock + .lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + println!("First lock result: {result1}"); + assert!(result1, "First lock should succeed"); + + // Second lock should fail (resource already locked) + println!("Attempting second lock..."); + let result2 = ns_lock + .lock_batch(&resource, "owner2", Duration::from_secs(1), Duration::from_secs(10)) + .await?; + println!("Second lock result: {result2}"); + assert!(!result2, "Second lock should fail"); + + // Unlock by first owner + println!("Unlocking first lock..."); + ns_lock.unlock_batch(&resource, "owner1").await?; + println!("First lock unlocked"); + + // Now second owner should be able to lock + println!("Attempting third lock..."); + let result3 = ns_lock + .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + println!("Third lock result: {result3}"); + assert!(result3, "Lock should succeed after unlock"); + + // Clean up + println!("Cleaning up..."); + ns_lock.unlock_batch(&resource, "owner2").await?; + println!("Test completed"); + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_read_write_lock_compatibility() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock = NamespaceLock::with_clients("test_rw".to_string(), clients); + let resource = vec!["rw_resource".to_string()]; + + // First read lock should succeed + let result1 = ns_lock + .rlock_batch(&resource, "reader1", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result1, "First read lock should succeed"); + + // Second read lock should also succeed (read locks are compatible) + let result2 = ns_lock + .rlock_batch(&resource, "reader2", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result2, "Second read lock should succeed"); + + // Write lock should fail (read locks are held) + let result3 = ns_lock + .lock_batch(&resource, "writer1", Duration::from_secs(1), Duration::from_secs(10)) + .await?; + assert!(!result3, "Write lock should fail when read locks are held"); + + // Release read locks + ns_lock.runlock_batch(&resource, "reader1").await?; + ns_lock.runlock_batch(&resource, "reader2").await?; + + // Now write lock should succeed + let result4 = ns_lock + .lock_batch(&resource, "writer1", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result4, "Write lock should succeed after read locks released"); + + // Clean up + ns_lock.unlock_batch(&resource, "writer1").await?; + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_lock_timeout() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock = NamespaceLock::with_clients("test_timeout".to_string(), clients); + let resource = vec!["timeout_resource".to_string()]; + + // First lock with short timeout + let result1 = ns_lock + .lock_batch(&resource, "owner1", Duration::from_secs(2), Duration::from_secs(1)) + .await?; + assert!(result1, "First lock should succeed"); + + // Wait for lock to expire + sleep(Duration::from_secs(5)).await; + + // Second lock should succeed after timeout + let result2 = ns_lock + .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(1)) + .await?; + assert!(result2, "Lock should succeed after timeout"); + + // Clean up + ns_lock.unlock_batch(&resource, "owner2").await?; + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_batch_lock_operations() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock = NamespaceLock::with_clients("test_batch".to_string(), clients); + let resources = vec![ + "batch_resource1".to_string(), + "batch_resource2".to_string(), + "batch_resource3".to_string(), + ]; + + // Lock all resources + let result = ns_lock + .lock_batch(&resources, "batch_owner", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result, "Batch lock should succeed"); + + // Try to lock one of the resources with different owner - should fail + let single_resource = vec!["batch_resource2".to_string()]; + let result2 = ns_lock + .lock_batch(&single_resource, "other_owner", Duration::from_secs(1), Duration::from_secs(10)) + .await?; + assert!(!result2, "Lock should fail for already locked resource"); + + // Unlock all resources + ns_lock.unlock_batch(&resources, "batch_owner").await?; + + // Now should be able to lock single resource + let result3 = ns_lock + .lock_batch(&single_resource, "other_owner", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result3, "Lock should succeed after batch unlock"); + + // Clean up + ns_lock.unlock_batch(&single_resource, "other_owner").await?; + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_multiple_namespaces() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock1 = NamespaceLock::with_clients("namespace1".to_string(), clients.clone()); + let ns_lock2 = NamespaceLock::with_clients("namespace2".to_string(), clients); + let resource = vec!["shared_resource".to_string()]; + + // Lock same resource in different namespaces - both should succeed + let result1 = ns_lock1 + .lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result1, "Lock in namespace1 should succeed"); + + let result2 = ns_lock2 + .lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10)) + .await?; + assert!(result2, "Lock in namespace2 should succeed"); + + // Clean up + ns_lock1.unlock_batch(&resource, "owner1").await?; + ns_lock2.unlock_batch(&resource, "owner2").await?; + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_rpc_read_lock() -> Result<(), Box> { + let args = LockRequest { + lock_id: LockId::new_deterministic("read_resource"), + resource: "read_resource".to_string(), + lock_type: LockType::Shared, + owner: "reader1".to_string(), + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, + }; + let args_str = serde_json::to_string(&args)?; + + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?; + + // First read lock + let request = Request::new(GenerallyLockRequest { args: args_str.clone() }); + let response = client.r_lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not get read lock: {error_info}"); + } + + // Second read lock with different owner should also succeed + let args2 = LockRequest { + lock_id: LockId::new_deterministic("read_resource"), + resource: "read_resource".to_string(), + lock_type: LockType::Shared, + owner: "reader2".to_string(), + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, + }; + let args2_str = serde_json::to_string(&args2)?; + let request2 = Request::new(GenerallyLockRequest { args: args2_str }); + let response2 = client.r_lock(request2).await?.into_inner(); + if let Some(error_info) = response2.error_info { + panic!("can not get second read lock: {error_info}"); + } + + // Unlock both + let request = Request::new(GenerallyLockRequest { args: args_str }); + let response = client.r_un_lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not unlock read lock: {error_info}"); + } + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_lock_refresh() -> Result<(), Box> { + let args = LockRequest { + lock_id: LockId::new_deterministic("refresh_resource"), + resource: "refresh_resource".to_string(), + lock_type: LockType::Exclusive, + owner: "refresh_owner".to_string(), + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, + }; + let args_str = serde_json::to_string(&args)?; + + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?; + + // Acquire lock + let request = Request::new(GenerallyLockRequest { args: args_str.clone() }); + let response = client.lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not get lock: {error_info}"); + } + + // Refresh lock + let request = Request::new(GenerallyLockRequest { args: args_str.clone() }); + let response = client.refresh(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not refresh lock: {error_info}"); + } + assert!(response.success, "Lock refresh should succeed"); + + // Unlock + let request = Request::new(GenerallyLockRequest { args: args_str }); + let response = client.un_lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not unlock: {error_info}"); + } + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_force_unlock() -> Result<(), Box> { + let args = LockRequest { + lock_id: LockId::new_deterministic("force_resource"), + resource: "force_resource".to_string(), + lock_type: LockType::Exclusive, + owner: "force_owner".to_string(), + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, + }; + let args_str = serde_json::to_string(&args)?; + + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?; + + // Acquire lock + let request = Request::new(GenerallyLockRequest { args: args_str.clone() }); + let response = client.lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not get lock: {error_info}"); + } + + // Force unlock (even by different owner) + let force_args = LockRequest { + lock_id: LockId::new_deterministic("force_resource"), + resource: "force_resource".to_string(), + lock_type: LockType::Exclusive, + owner: "admin".to_string(), + acquire_timeout: Duration::from_secs(30), + ttl: Duration::from_secs(30), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + deadlock_detection: false, + }; + let force_args_str = serde_json::to_string(&force_args)?; + let request = Request::new(GenerallyLockRequest { args: force_args_str }); + let response = client.force_un_lock(request).await?.into_inner(); + if let Some(error_info) = response.error_info { + panic!("can not force unlock: {error_info}"); + } + assert!(response.success, "Force unlock should succeed"); + + Ok(()) +} + +#[tokio::test] +#[serial] +#[ignore = "requires running RustFS server at localhost:9000"] +async fn test_global_lock_map_sharing() -> Result<(), Box> { + let endpoints = get_cluster_endpoints(); + let clients = create_unique_clients(&endpoints).await?; + let ns_lock1 = NamespaceLock::with_clients("global_test".to_string(), clients.clone()); + let ns_lock2 = NamespaceLock::with_clients("global_test".to_string(), clients); + + let resource = vec!["global_test_resource".to_string()]; + + // First instance acquires lock + println!("First lock map attempting to acquire lock..."); + let result1 = ns_lock1 + .lock_batch(&resource, "owner1", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10)) + .await?; + println!("First lock result: {result1}"); + assert!(result1, "First lock should succeed"); + + // Second instance should fail to acquire the same lock + println!("Second lock map attempting to acquire lock..."); + let result2 = ns_lock2 + .lock_batch(&resource, "owner2", std::time::Duration::from_secs(1), std::time::Duration::from_secs(10)) + .await?; + println!("Second lock result: {result2}"); + assert!(!result2, "Second lock should fail because resource is already locked"); + + // Release lock from first instance + println!("First lock map releasing lock..."); + ns_lock1.unlock_batch(&resource, "owner1").await?; + + // Now second instance should be able to acquire lock + println!("Second lock map attempting to acquire lock again..."); + let result3 = ns_lock2 + .lock_batch(&resource, "owner2", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10)) + .await?; + println!("Third lock result: {result3}"); + assert!(result3, "Lock should succeed after first lock is released"); + + // Clean up + ns_lock2.unlock_batch(&resource, "owner2").await?; - ns.0.write().await.un_lock().await.unwrap(); Ok(()) } diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs index 6c380aab5..5c8a110cb 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::store::ECStore; use crate::store_api::{ObjectOptions, ObjectToDelete}; -use rustfs_lock::local_locker::MAX_DELETE_LIST; +use rustfs_lock::MAX_DELETE_LIST; pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { let mut remaining = to_del; diff --git a/crates/ecstore/src/error.rs b/crates/ecstore/src/error.rs index c9dc2c1fd..4de5c1595 100644 --- a/crates/ecstore/src/error.rs +++ b/crates/ecstore/src/error.rs @@ -183,6 +183,9 @@ pub enum StorageError { #[error("Io error: {0}")] Io(std::io::Error), + + #[error("Lock error: {0}")] + Lock(#[from] rustfs_lock::LockError), } impl StorageError { @@ -409,6 +412,7 @@ impl Clone for StorageError { StorageError::FirstDiskWait => StorageError::FirstDiskWait, StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles, StorageError::NoHealRequired => StorageError::NoHealRequired, + StorageError::Lock(e) => StorageError::Lock(e.clone()), } } } @@ -471,6 +475,7 @@ impl StorageError { StorageError::ConfigNotFound => 0x35, StorageError::TooManyOpenFiles => 0x36, StorageError::NoHealRequired => 0x37, + StorageError::Lock(_) => 0x38, } } @@ -535,6 +540,7 @@ impl StorageError { 0x35 => Some(StorageError::ConfigNotFound), 0x36 => Some(StorageError::TooManyOpenFiles), 0x37 => Some(StorageError::NoHealRequired), + 0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))), _ => None, } } diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 15552987b..daf032595 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -30,6 +30,7 @@ pub mod erasure_coding; pub mod error; pub mod global; pub mod heal; +pub mod lock_utils; pub mod metrics_realtime; pub mod notification_sys; pub mod pools; diff --git a/crates/ecstore/src/lock_utils.rs b/crates/ecstore/src/lock_utils.rs new file mode 100644 index 000000000..98b9f320c --- /dev/null +++ b/crates/ecstore/src/lock_utils.rs @@ -0,0 +1,136 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::disk::endpoint::Endpoint; +use crate::error::Result; +use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient}; +use std::collections::HashMap; +use std::sync::Arc; + +/// Create unique lock clients from endpoints +/// This function creates one client per unique host:port combination +/// to avoid duplicate connections to the same server +pub async fn create_unique_clients(endpoints: &[Endpoint]) -> Result>> { + let mut unique_endpoints: HashMap = HashMap::new(); + + // Collect unique endpoints based on host:port + for endpoint in endpoints { + if endpoint.is_local { + // For local endpoints, use "local" as the key + unique_endpoints.insert("local".to_string(), endpoint); + } else { + // For remote endpoints, use host:port as the key + let host_port = format!( + "{}:{}", + endpoint.url.host_str().unwrap_or("localhost"), + endpoint.url.port().unwrap_or(9000) + ); + unique_endpoints.insert(host_port, endpoint); + } + } + + let mut clients = Vec::new(); + + // Create clients for unique endpoints + for (_key, endpoint) in unique_endpoints { + if endpoint.is_local { + // For local endpoints, create a local lock client + let local_client = LocalClient::new(); + clients.push(Arc::new(local_client) as Arc); + } else { + // For remote endpoints, create a remote lock client + let remote_client = RemoteClient::new(endpoint.url.to_string()); + clients.push(Arc::new(remote_client) as Arc); + } + } + + Ok(clients) +} + +#[cfg(test)] +mod tests { + use super::*; + use url::Url; + + #[tokio::test] + async fn test_create_unique_clients_local() { + let endpoints = vec![ + Endpoint { + url: Url::parse("http://localhost:9000").unwrap(), + is_local: true, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }, + Endpoint { + url: Url::parse("http://localhost:9000").unwrap(), + is_local: true, + pool_idx: 0, + set_idx: 0, + disk_idx: 1, + }, + ]; + + let clients = create_unique_clients(&endpoints).await.unwrap(); + // Should only create one client for local endpoints + assert_eq!(clients.len(), 1); + assert!(clients[0].is_local().await); + } + + #[tokio::test] + async fn test_create_unique_clients_mixed() { + let endpoints = vec![ + Endpoint { + url: Url::parse("http://localhost:9000").unwrap(), + is_local: true, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }, + Endpoint { + url: Url::parse("http://remote1:9000").unwrap(), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 1, + }, + Endpoint { + url: Url::parse("http://remote1:9000").unwrap(), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 2, + }, + Endpoint { + url: Url::parse("http://remote2:9000").unwrap(), + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 3, + }, + ]; + + let clients = create_unique_clients(&endpoints).await.unwrap(); + // Should create 3 clients: 1 local + 2 unique remote + assert_eq!(clients.len(), 3); + + // Check that we have one local client + let local_count = clients.iter().filter(|c| futures::executor::block_on(c.is_local())).count(); + assert_eq!(local_count, 1); + + // Check that we have two remote clients + let remote_count = clients.iter().filter(|c| !futures::executor::block_on(c.is_local())).count(); + assert_eq!(remote_count, 2); + } +} diff --git a/crates/ecstore/src/rpc/mod.rs b/crates/ecstore/src/rpc/mod.rs index 5c56b8fc1..53d3a1b7e 100644 --- a/crates/ecstore/src/rpc/mod.rs +++ b/crates/ecstore/src/rpc/mod.rs @@ -22,4 +22,4 @@ pub use http_auth::{build_auth_headers, verify_rpc_signature}; pub use peer_rest_client::PeerRestClient; pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, RemotePeerS3Client, S3PeerSys}; pub use remote_disk::RemoteDisk; -pub use tonic_service::make_server; +pub use tonic_service::{NodeService, make_server}; diff --git a/crates/ecstore/src/rpc/tonic_service.rs b/crates/ecstore/src/rpc/tonic_service.rs index 93a50e412..3607d0daa 100644 --- a/crates/ecstore/src/rpc/tonic_service.rs +++ b/crates/ecstore/src/rpc/tonic_service.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::HashMap, io::Cursor, pin::Pin}; +use std::{collections::HashMap, io::Cursor, pin::Pin, sync::Arc}; // use common::error::Error as EcsError; use crate::{ @@ -34,9 +34,9 @@ use crate::{ }; use futures::{Stream, StreamExt}; use futures_util::future::join_all; -use rustfs_lock::{GLOBAL_LOCAL_SERVER, Locker, lock_args::LockArgs}; use rustfs_common::globals::GLOBAL_Local_Node_Name; +use rustfs_lock::{LockClient, LockRequest}; use bytes::Bytes; use rmp_serde::{Deserializer, Serializer}; @@ -81,11 +81,16 @@ type ResponseStream = Pin> + S #[derive(Debug)] pub struct NodeService { local_peer: LocalPeerS3Client, + lock_manager: Arc, } pub fn make_server() -> NodeService { let local_peer = LocalPeerS3Client::new(None, None); - NodeService { local_peer } + let lock_manager = Arc::new(rustfs_lock::LocalClient::new()); + NodeService { + local_peer, + lock_manager, + } } impl NodeService { @@ -1526,122 +1531,156 @@ impl Node for NodeService { async fn lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.lock(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + // Parse the request to extract resource and owner + let args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not lock, args: {args}, err: {err}")), - })), - }, + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + match self.lock_manager.acquire_exclusive(&args).await { + Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { + success: result.success, + error_info: None, + })), Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not decode args, err: {err}")), + error_info: Some(format!( + "can not lock, resource: {0}, owner: {1}, err: {2}", + args.resource, args.owner, err + )), })), } } async fn un_lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.unlock(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + let args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not unlock, args: {args}, err: {err}")), - })), - }, + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + match self.lock_manager.release(&args.lock_id).await { + Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse { + success: true, + error_info: None, + })), Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not decode args, err: {err}")), + error_info: Some(format!( + "can not unlock, resource: {0}, owner: {1}, err: {2}", + args.resource, args.owner, err + )), })), } } async fn r_lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.rlock(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + let args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not rlock, args: {args}, err: {err}")), - })), - }, + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + match self.lock_manager.acquire_shared(&args).await { + Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { + success: result.success, + error_info: None, + })), Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not decode args, err: {err}")), + error_info: Some(format!( + "can not rlock, resource: {0}, owner: {1}, err: {2}", + args.resource, args.owner, err + )), })), } } async fn r_un_lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.runlock(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + let args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not runlock, args: {args}, err: {err}")), - })), - }, + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + match self.lock_manager.release(&args.lock_id).await { + Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse { + success: true, + error_info: None, + })), Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not decode args, err: {err}")), + error_info: Some(format!( + "can not runlock, resource: {0}, owner: {1}, err: {2}", + args.resource, args.owner, err + )), })), } } async fn force_un_lock(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.force_unlock(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + let args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not force_unlock, args: {args}, err: {err}")), - })), - }, + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + match self.lock_manager.release(&args.lock_id).await { + Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse { + success: true, + error_info: None, + })), Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not decode args, err: {err}")), + error_info: Some(format!( + "can not force_unlock, resource: {0}, owner: {1}, err: {2}", + args.resource, args.owner, err + )), })), } } async fn refresh(&self, request: Request) -> Result, Status> { let request = request.into_inner(); - match &serde_json::from_str::(&request.args) { - Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.refresh(args).await { - Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse { - success: result, - error_info: None, - })), - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { + let _args: LockRequest = match serde_json::from_str(&request.args) { + Ok(args) => args, + Err(err) => { + return Ok(tonic::Response::new(GenerallyLockResponse { success: false, - error_info: Some(format!("can not refresh, args: {args}, err: {err}")), - })), - }, - Err(err) => Ok(tonic::Response::new(GenerallyLockResponse { - success: false, - error_info: Some(format!("can not decode args, err: {err}")), - })), - } + error_info: Some(format!("can not decode args, err: {err}")), + })); + } + }; + + Ok(tonic::Response::new(GenerallyLockResponse { + success: true, + error_info: None, + })) } async fn local_storage_info( @@ -3629,15 +3668,15 @@ mod tests { // Note: signal_service test is skipped because it contains todo!() and would panic - #[test] - fn test_node_service_debug() { + #[tokio::test] + async fn test_node_service_debug() { let service = create_test_node_service(); let debug_str = format!("{service:?}"); assert!(debug_str.contains("NodeService")); } - #[test] - fn test_node_service_creation() { + #[tokio::test] + async fn test_node_service_creation() { let service1 = make_server(); let service2 = make_server(); diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 9bd745781..87b79ecae 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -83,7 +83,7 @@ use rustfs_filemeta::{ headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS}, merge_file_meta_versions, }; -use rustfs_lock::{LockApi, namespace_lock::NsLockMap}; +use rustfs_lock::NamespaceLockManager; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader}; use rustfs_utils::{ @@ -121,11 +121,10 @@ use uuid::Uuid; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; -#[derive(Debug, Clone)] +#[derive(Clone, Debug)] pub struct SetDisks { - pub lockers: Vec, + pub namespace_lock: Arc, pub locker_owner: String, - pub ns_mutex: Arc>, pub disks: Arc>>>, pub set_endpoints: Vec, pub set_drive_count: usize, @@ -138,9 +137,8 @@ pub struct SetDisks { impl SetDisks { #[allow(clippy::too_many_arguments)] pub async fn new( - lockers: Vec, + namespace_lock: Arc, locker_owner: String, - ns_mutex: Arc>, disks: Arc>>>, set_drive_count: usize, default_parity_count: usize, @@ -150,9 +148,8 @@ impl SetDisks { format: FormatV3, ) -> Arc { Arc::new(SetDisks { - lockers, + namespace_lock, locker_owner, - ns_mutex, disks, set_drive_count, default_parity_count, @@ -4066,33 +4063,22 @@ impl ObjectIO for SetDisks { async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result { let disks = self.disks.read().await; - // let mut _ns = None; - // if !opts.no_lock { - // let paths = vec![object.to_string()]; - // let ns_lock = new_nslock( - // Arc::clone(&self.ns_mutex), - // self.locker_owner.clone(), - // bucket.to_string(), - // paths, - // self.lockers.clone(), - // ) - // .await; - // if !ns_lock - // .0 - // .write() - // .await - // .get_lock(&Options { - // timeout: Duration::from_secs(5), - // retry_interval: Duration::from_secs(1), - // }) - // .await - // .map_err(|err| Error::other(err.to_string()))? - // { - // return Err(Error::other("can not get lock. please retry".to_string())); - // } + if !opts.no_lock { + let paths = vec![object.to_string()]; + let lock_acquired = self + .namespace_lock + .lock_batch( + &paths, + &self.locker_owner, + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(10), + ) + .await?; - // _ns = Some(ns_lock); - // } + if !lock_acquired { + return Err(Error::other("can not get lock. please retry".to_string())); + } + } let mut user_defined = opts.user_defined.clone(); @@ -4298,9 +4284,13 @@ impl ObjectIO for SetDisks { self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; - // if let Some(mut locker) = ns { - // locker.un_lock().await.map_err(|err| Error::other(err.to_string()))?; - // } + // Release lock if it was acquired + if !opts.no_lock { + let paths = vec![object.to_string()]; + if let Err(err) = self.namespace_lock.unlock_batch(&paths, &self.locker_owner).await { + error!("Failed to unlock object {}: {}", object, err); + } + } for (i, op_disk) in online_disks.iter().enumerate() { if let Some(disk) = op_disk { diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index c798ef18b..79a35684c 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -43,7 +43,7 @@ use futures::future::join_all; use http::HeaderMap; use rustfs_common::globals::GLOBAL_Local_Node_Name; use rustfs_filemeta::FileInfo; -use rustfs_lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; + use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use tokio::sync::RwLock; @@ -55,12 +55,13 @@ use tokio::time::Duration; use tracing::warn; use tracing::{error, info}; +use crate::lock_utils::create_unique_clients; + #[derive(Debug, Clone)] pub struct Sets { pub id: Uuid, // pub sets: Vec, // pub disk_set: Vec>>, // [set_count_idx][set_drive_count_idx] = disk_idx - pub lockers: Vec>, pub disk_set: Vec>, // [set_count_idx][set_drive_count_idx] = disk_idx pub pool_idx: usize, pub endpoints: PoolEndpoints, @@ -93,27 +94,25 @@ impl Sets { let set_count = fm.erasure.sets.len(); let set_drive_count = fm.erasure.sets[0].len(); - let mut unique: Vec> = vec![vec![]; set_count]; - let mut lockers: Vec> = vec![vec![]; set_count]; - endpoints.endpoints.as_ref().iter().enumerate().for_each(|(idx, endpoint)| { + let mut unique: Vec> = (0..set_count).map(|_| vec![]).collect(); + + for (idx, endpoint) in endpoints.endpoints.as_ref().iter().enumerate() { let set_idx = idx / set_drive_count; if endpoint.is_local && !unique[set_idx].contains(&"local".to_string()) { unique[set_idx].push("local".to_string()); - lockers[set_idx].push(new_lock_api(true, None)); } if !endpoint.is_local { let host_port = format!("{}:{}", endpoint.url.host_str().unwrap(), endpoint.url.port().unwrap()); if !unique[set_idx].contains(&host_port) { unique[set_idx].push(host_port); - lockers[set_idx].push(new_lock_api(false, Some(endpoint.url.clone()))); } } - }); + } let mut disk_set = Vec::with_capacity(set_count); - for (i, locker) in lockers.iter().enumerate().take(set_count) { + for i in 0..set_count { let mut set_drive = Vec::with_capacity(set_drive_count); let mut set_endpoints = Vec::with_capacity(set_drive_count); for j in 0..set_drive_count { @@ -121,7 +120,6 @@ impl Sets { let mut disk = disks[idx].clone(); let endpoint = endpoints.endpoints.as_ref()[idx].clone(); - // let endpoint = endpoints.endpoints.as_ref().get(idx).cloned(); set_endpoints.push(endpoint); if disk.is_none() { @@ -165,12 +163,13 @@ impl Sets { } } - // warn!("sets new set_drive {:?}", &set_drive); + let lock_clients = create_unique_clients(&set_endpoints).await?; + + let namespace_lock = rustfs_lock::NamespaceLock::with_clients(format!("set-{i}"), lock_clients); let set_disks = SetDisks::new( - locker.clone(), + Arc::new(namespace_lock), GLOBAL_Local_Node_Name.read().await.to_string(), - Arc::new(RwLock::new(NsLockMap::new(is_dist_erasure().await))), Arc::new(RwLock::new(set_drive)), set_drive_count, parity_count, @@ -190,7 +189,6 @@ impl Sets { id: fm.id, // sets: todo!(), disk_set, - lockers, pool_idx, endpoints: endpoints.clone(), format: fm.clone(), @@ -543,7 +541,7 @@ impl StorageAPI for Sets { objects: Vec, opts: ObjectOptions, ) -> Result<(Vec, Vec>)> { - // 默认返回值 + // Default return value let mut del_objects = vec![DeletedObject::default(); objects.len()]; let mut del_errs = Vec::with_capacity(objects.len()); @@ -602,7 +600,7 @@ impl StorageAPI for Sets { // del_errs.extend(errs); // } - // TODO: 并发 + // TODO: Implement concurrency for (k, v) in set_obj_map { let disks = self.get_disks(k); let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); diff --git a/crates/lock/Cargo.toml b/crates/lock/Cargo.toml index 2024bbdc1..cb0a8ee9e 100644 --- a/crates/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -30,6 +30,9 @@ workspace = true [dependencies] async-trait.workspace = true +bytes.workspace = true +futures.workspace = true +lazy_static.workspace = true rustfs-protos.workspace = true rand.workspace = true serde.workspace = true @@ -38,4 +41,7 @@ tokio.workspace = true tonic.workspace = true tracing.workspace = true url.workspace = true -uuid.workspace = true \ No newline at end of file +uuid.workspace = true +thiserror.workspace = true +once_cell.workspace = true +lru.workspace = true diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs new file mode 100644 index 000000000..3e882883d --- /dev/null +++ b/crates/lock/src/client/local.rs @@ -0,0 +1,366 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use crate::{ + client::LockClient, + error::Result, + local::LocalLockMap, + types::{LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockType}, +}; + +/// Local lock client +/// +/// Uses global singleton LocalLockMap to ensure all clients access the same lock instance +#[derive(Debug, Clone)] +pub struct LocalClient; + +impl LocalClient { + /// Create new local client + pub fn new() -> Self { + Self + } + + /// Get global lock map instance + pub fn get_lock_map(&self) -> Arc { + crate::get_global_lock_map() + } +} + +impl Default for LocalClient { + fn default() -> Self { + Self::new() + } +} + +#[async_trait::async_trait] +impl LockClient for LocalClient { + async fn acquire_exclusive(&self, request: &LockRequest) -> Result { + let lock_map = self.get_lock_map(); + let success = lock_map + .lock_with_ttl_id(request) + .await + .map_err(|e| crate::error::LockError::internal(format!("Lock acquisition failed: {e}")))?; + if success { + let lock_info = LockInfo { + id: crate::types::LockId::new_deterministic(&request.resource), + resource: request.resource.clone(), + lock_type: LockType::Exclusive, + status: crate::types::LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + request.ttl, + last_refreshed: std::time::SystemTime::now(), + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }; + Ok(LockResponse::success(lock_info, std::time::Duration::ZERO)) + } else { + Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO)) + } + } + + async fn acquire_shared(&self, request: &LockRequest) -> Result { + let lock_map = self.get_lock_map(); + let success = lock_map + .rlock_with_ttl_id(request) + .await + .map_err(|e| crate::error::LockError::internal(format!("Shared lock acquisition failed: {e}")))?; + if success { + let lock_info = LockInfo { + id: crate::types::LockId::new_deterministic(&request.resource), + resource: request.resource.clone(), + lock_type: LockType::Shared, + status: crate::types::LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + request.ttl, + last_refreshed: std::time::SystemTime::now(), + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }; + Ok(LockResponse::success(lock_info, std::time::Duration::ZERO)) + } else { + Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO)) + } + } + + async fn release(&self, lock_id: &LockId) -> Result { + let lock_map = self.get_lock_map(); + + // Try to release the lock directly by ID + match lock_map.unlock_by_id(lock_id).await { + Ok(()) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // Try as read lock if exclusive unlock failed + match lock_map.runlock_by_id(lock_id).await { + Ok(()) => Ok(true), + Err(_) => Err(crate::error::LockError::internal("Lock ID not found".to_string())), + } + } + Err(e) => Err(crate::error::LockError::internal(format!("Release lock failed: {e}"))), + } + } + + async fn refresh(&self, _lock_id: &LockId) -> Result { + // For local locks, refresh is not needed as they don't expire automatically + Ok(true) + } + + async fn force_release(&self, lock_id: &LockId) -> Result { + self.release(lock_id).await + } + + async fn check_status(&self, lock_id: &LockId) -> Result> { + let lock_map = self.get_lock_map(); + + // Check if the lock exists in our locks map + let locks_guard = lock_map.locks.read().await; + if let Some(entry) = locks_guard.get(lock_id) { + let entry_guard = entry.read().await; + + // Determine lock type and owner based on the entry + if let Some(owner) = &entry_guard.writer { + Ok(Some(LockInfo { + id: lock_id.clone(), + resource: lock_id.resource.clone(), + lock_type: crate::types::LockType::Exclusive, + status: crate::types::LockStatus::Acquired, + owner: owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30), + last_refreshed: std::time::SystemTime::now(), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + wait_start_time: None, + })) + } else if !entry_guard.readers.is_empty() { + Ok(Some(LockInfo { + id: lock_id.clone(), + resource: lock_id.resource.clone(), + lock_type: crate::types::LockType::Shared, + status: crate::types::LockStatus::Acquired, + owner: entry_guard.readers.iter().next().map(|(k, _)| k.clone()).unwrap_or_default(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30), + last_refreshed: std::time::SystemTime::now(), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + wait_start_time: None, + })) + } else { + Ok(None) + } + } else { + Ok(None) + } + } + + async fn get_stats(&self) -> Result { + Ok(LockStats::default()) + } + + async fn close(&self) -> Result<()> { + Ok(()) + } + + async fn is_online(&self) -> bool { + true + } + + async fn is_local(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::LockType; + + #[tokio::test] + async fn test_local_client_acquire_exclusive() { + let client = LocalClient::new(); + let resource_name = format!("test-resource-exclusive-{}", uuid::Uuid::new_v4()); + let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner") + .with_acquire_timeout(std::time::Duration::from_secs(30)); + + let response = client.acquire_exclusive(&request).await.unwrap(); + assert!(response.is_success()); + + // Clean up + if let Some(lock_info) = response.lock_info() { + let _ = client.release(&lock_info.id).await; + } + } + + #[tokio::test] + async fn test_local_client_acquire_shared() { + let client = LocalClient::new(); + let resource_name = format!("test-resource-shared-{}", uuid::Uuid::new_v4()); + let request = LockRequest::new(&resource_name, LockType::Shared, "test-owner") + .with_acquire_timeout(std::time::Duration::from_secs(30)); + + let response = client.acquire_shared(&request).await.unwrap(); + assert!(response.is_success()); + + // Clean up + if let Some(lock_info) = response.lock_info() { + let _ = client.release(&lock_info.id).await; + } + } + + #[tokio::test] + async fn test_local_client_release() { + let client = LocalClient::new(); + let resource_name = format!("test-resource-release-{}", uuid::Uuid::new_v4()); + + // First acquire a lock + let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner") + .with_acquire_timeout(std::time::Duration::from_secs(30)); + let response = client.acquire_exclusive(&request).await.unwrap(); + assert!(response.is_success()); + + // Get the lock ID from the response + if let Some(lock_info) = response.lock_info() { + let result = client.release(&lock_info.id).await.unwrap(); + assert!(result); + } else { + panic!("No lock info in response"); + } + } + + #[tokio::test] + async fn test_local_client_is_local() { + let client = LocalClient::new(); + assert!(client.is_local().await); + } + + #[tokio::test] + async fn test_local_client_read_write_lock_exclusion() { + let client = LocalClient::new(); + let resource_name = format!("test-resource-exclusion-{}", uuid::Uuid::new_v4()); + + // First, acquire an exclusive lock + let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner") + .with_acquire_timeout(std::time::Duration::from_millis(10)); + let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap(); + assert!(exclusive_response.is_success()); + + // Try to acquire a shared lock on the same resource - should fail + let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner") + .with_acquire_timeout(std::time::Duration::from_millis(10)); + let shared_response = client.acquire_shared(&shared_request).await.unwrap(); + assert!(!shared_response.is_success(), "Shared lock should fail when exclusive lock exists"); + + // Clean up exclusive lock + if let Some(exclusive_info) = exclusive_response.lock_info() { + let _ = client.release(&exclusive_info.id).await; + } + + // Now shared lock should succeed + let shared_request2 = LockRequest::new(&resource_name, LockType::Shared, "shared-owner") + .with_acquire_timeout(std::time::Duration::from_millis(10)); + let shared_response2 = client.acquire_shared(&shared_request2).await.unwrap(); + assert!( + shared_response2.is_success(), + "Shared lock should succeed after exclusive lock is released" + ); + + // Clean up + if let Some(shared_info) = shared_response2.lock_info() { + let _ = client.release(&shared_info.id).await; + } + } + + #[tokio::test] + async fn test_local_client_read_write_lock_distinction() { + let client = LocalClient::new(); + let resource_name = format!("test-resource-rw-{}", uuid::Uuid::new_v4()); + + // Test exclusive lock + let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner") + .with_acquire_timeout(std::time::Duration::from_secs(30)); + let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap(); + assert!(exclusive_response.is_success()); + + if let Some(exclusive_info) = exclusive_response.lock_info() { + assert_eq!(exclusive_info.lock_type, LockType::Exclusive); + + // Check status should return correct lock type + let status = client.check_status(&exclusive_info.id).await.unwrap(); + assert!(status.is_some()); + assert_eq!(status.unwrap().lock_type, LockType::Exclusive); + + // Release exclusive lock + let result = client.release(&exclusive_info.id).await.unwrap(); + assert!(result); + } + + // Test shared lock + let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner") + .with_acquire_timeout(std::time::Duration::from_secs(30)); + let shared_response = client.acquire_shared(&shared_request).await.unwrap(); + assert!(shared_response.is_success()); + + if let Some(shared_info) = shared_response.lock_info() { + assert_eq!(shared_info.lock_type, LockType::Shared); + + // Check status should return correct lock type + let status = client.check_status(&shared_info.id).await.unwrap(); + assert!(status.is_some()); + assert_eq!(status.unwrap().lock_type, LockType::Shared); + + // Release shared lock + let result = client.release(&shared_info.id).await.unwrap(); + assert!(result); + } + } + + #[tokio::test] + async fn test_multiple_local_clients_exclusive_mutex() { + let client1 = LocalClient::new(); + let client2 = LocalClient::new(); + let resource_name = format!("test-multi-client-mutex-{}", uuid::Uuid::new_v4()); + + // client1 acquire exclusive lock + let req1 = LockRequest::new(&resource_name, LockType::Exclusive, "owner1") + .with_acquire_timeout(std::time::Duration::from_millis(50)); + let resp1 = client1.acquire_exclusive(&req1).await.unwrap(); + assert!(resp1.is_success(), "client1 should acquire exclusive lock"); + + // client2 try to acquire exclusive lock, should fail + let req2 = LockRequest::new(&resource_name, LockType::Exclusive, "owner2") + .with_acquire_timeout(std::time::Duration::from_millis(50)); + let resp2 = client2.acquire_exclusive(&req2).await.unwrap(); + assert!(!resp2.is_success(), "client2 should not acquire exclusive lock while client1 holds it"); + + // client1 release lock + if let Some(lock_info) = resp1.lock_info() { + let _ = client1.release(&lock_info.id).await; + } + + // client2 try again, should succeed + let resp3 = client2.acquire_exclusive(&req2).await.unwrap(); + assert!(resp3.is_success(), "client2 should acquire exclusive lock after client1 releases it"); + + // clean up + if let Some(lock_info) = resp3.lock_info() { + let _ = client2.release(&lock_info.id).await; + } + } +} diff --git a/crates/lock/src/client/mod.rs b/crates/lock/src/client/mod.rs new file mode 100644 index 000000000..a07f47750 --- /dev/null +++ b/crates/lock/src/client/mod.rs @@ -0,0 +1,123 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod local; +pub mod remote; + +use async_trait::async_trait; +use std::sync::Arc; + +use crate::{ + error::Result, + types::{LockId, LockInfo, LockRequest, LockResponse, LockStats}, +}; + +/// Lock client trait +#[async_trait] +pub trait LockClient: Send + Sync + std::fmt::Debug { + /// Acquire exclusive lock + async fn acquire_exclusive(&self, request: &LockRequest) -> Result; + + /// Acquire shared lock + async fn acquire_shared(&self, request: &LockRequest) -> Result; + + /// Acquire lock (generic method) + async fn acquire_lock(&self, request: &LockRequest) -> Result { + match request.lock_type { + crate::types::LockType::Exclusive => self.acquire_exclusive(request).await, + crate::types::LockType::Shared => self.acquire_shared(request).await, + } + } + + /// Release lock + async fn release(&self, lock_id: &LockId) -> Result; + + /// Refresh lock + async fn refresh(&self, lock_id: &LockId) -> Result; + + /// Force release lock + async fn force_release(&self, lock_id: &LockId) -> Result; + + /// Check lock status + async fn check_status(&self, lock_id: &LockId) -> Result>; + + /// Get statistics + async fn get_stats(&self) -> Result; + + /// Close client + async fn close(&self) -> Result<()>; + + /// Check if client is online + async fn is_online(&self) -> bool; + + /// Check if client is local + async fn is_local(&self) -> bool; +} + +/// Client factory +pub struct ClientFactory; + +impl ClientFactory { + /// Create local client + pub fn create_local() -> Arc { + Arc::new(local::LocalClient::new()) + } + + /// Create remote client + pub fn create_remote(endpoint: String) -> Arc { + Arc::new(remote::RemoteClient::new(endpoint)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::LockType; + + #[tokio::test] + async fn test_client_factory() { + let local_client = ClientFactory::create_local(); + assert!(local_client.is_local().await); + + let remote_client = ClientFactory::create_remote("http://localhost:8080".to_string()); + assert!(!remote_client.is_local().await); + } + + #[tokio::test] + async fn test_local_client_basic_operations() { + let client = ClientFactory::create_local(); + + let request = crate::types::LockRequest::new("test-resource", LockType::Exclusive, "test-owner"); + + // Test lock acquisition + let response = client.acquire_exclusive(&request).await; + assert!(response.is_ok()); + + if let Ok(response) = response { + if response.success { + let lock_info = response.lock_info.unwrap(); + + // Test status check + let status = client.check_status(&lock_info.id).await; + assert!(status.is_ok()); + assert!(status.unwrap().is_some()); + + // Test lock release + let released = client.release(&lock_info.id).await; + assert!(released.is_ok()); + assert!(released.unwrap()); + } + } + } +} diff --git a/crates/lock/src/client/remote.rs b/crates/lock/src/client/remote.rs new file mode 100644 index 000000000..e370b6271 --- /dev/null +++ b/crates/lock/src/client/remote.rs @@ -0,0 +1,403 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use rustfs_protos::{ + node_service_time_out_client, + proto_gen::node_service::{GenerallyLockRequest, PingRequest}, +}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; +use tonic::Request; +use tracing::info; + +use crate::{ + error::{LockError, Result}, + types::{LockId, LockInfo, LockRequest, LockResponse, LockStats}, +}; + +use super::LockClient; + +/// Remote lock client implementation +#[derive(Debug)] +pub struct RemoteClient { + addr: String, + // Track active locks with their original owner information + active_locks: Arc>>, // lock_id -> owner +} + +impl Clone for RemoteClient { + fn clone(&self) -> Self { + Self { + addr: self.addr.clone(), + active_locks: self.active_locks.clone(), + } + } +} + +impl RemoteClient { + pub fn new(endpoint: String) -> Self { + Self { + addr: endpoint, + active_locks: Arc::new(RwLock::new(HashMap::new())), + } + } + + pub fn from_url(url: url::Url) -> Self { + Self { + addr: url.to_string(), + active_locks: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Create a minimal LockRequest for unlock operations + fn create_unlock_request(&self, lock_id: &LockId, owner: &str) -> LockRequest { + LockRequest { + lock_id: lock_id.clone(), + resource: lock_id.resource.clone(), + lock_type: crate::types::LockType::Exclusive, // Type doesn't matter for unlock + owner: owner.to_string(), + acquire_timeout: std::time::Duration::from_secs(30), + ttl: std::time::Duration::from_secs(300), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + } + } +} + +#[async_trait] +impl LockClient for RemoteClient { + async fn acquire_exclusive(&self, request: &LockRequest) -> Result { + info!("remote acquire_exclusive for {}", request.resource); + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + let req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let resp = client + .lock(req) + .await + .map_err(|e| LockError::internal(e.to_string()))? + .into_inner(); + + // Check for explicit error first + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + + // Check if the lock acquisition was successful + if resp.success { + // Save the lock information for later release + let mut locks = self.active_locks.write().await; + locks.insert(request.lock_id.clone(), request.owner.clone()); + + Ok(LockResponse::success( + LockInfo { + id: request.lock_id.clone(), + resource: request.resource.clone(), + lock_type: request.lock_type, + status: crate::types::LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + request.ttl, + last_refreshed: std::time::SystemTime::now(), + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }, + std::time::Duration::ZERO, + )) + } else { + // Lock acquisition failed + Ok(LockResponse::failure( + "Lock acquisition failed on remote server".to_string(), + std::time::Duration::ZERO, + )) + } + } + + async fn acquire_shared(&self, request: &LockRequest) -> Result { + info!("remote acquire_shared for {}", request.resource); + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + let req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let resp = client + .r_lock(req) + .await + .map_err(|e| LockError::internal(e.to_string()))? + .into_inner(); + + // Check for explicit error first + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + + // Check if the lock acquisition was successful + if resp.success { + // Save the lock information for later release + let mut locks = self.active_locks.write().await; + locks.insert(request.lock_id.clone(), request.owner.clone()); + + Ok(LockResponse::success( + LockInfo { + id: request.lock_id.clone(), + resource: request.resource.clone(), + lock_type: request.lock_type, + status: crate::types::LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + request.ttl, + last_refreshed: std::time::SystemTime::now(), + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }, + std::time::Duration::ZERO, + )) + } else { + // Lock acquisition failed + Ok(LockResponse::failure( + "Shared lock acquisition failed on remote server".to_string(), + std::time::Duration::ZERO, + )) + } + } + + async fn release(&self, lock_id: &LockId) -> Result { + info!("remote release for {}", lock_id); + + // Get the original owner for this lock + let owner = { + let locks = self.active_locks.read().await; + locks.get(lock_id).cloned().unwrap_or_else(|| "remote".to_string()) + }; + + let unlock_request = self.create_unlock_request(lock_id, &owner); + + let request_string = serde_json::to_string(&unlock_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?; + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + + // Try UnLock first (for exclusive locks) + let req = Request::new(GenerallyLockRequest { + args: request_string.clone(), + }); + let resp = client.un_lock(req).await; + + let success = if resp.is_err() { + // If that fails, try RUnLock (for shared locks) + let req = Request::new(GenerallyLockRequest { args: request_string }); + let resp = client + .r_un_lock(req) + .await + .map_err(|e| LockError::internal(e.to_string()))? + .into_inner(); + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + resp.success + } else { + let resp = resp.map_err(|e| LockError::internal(e.to_string()))?.into_inner(); + + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + resp.success + }; + + // Remove the lock from our tracking if successful + if success { + let mut locks = self.active_locks.write().await; + locks.remove(lock_id); + } + + Ok(success) + } + + async fn refresh(&self, lock_id: &LockId) -> Result { + info!("remote refresh for {}", lock_id); + let refresh_request = self.create_unlock_request(lock_id, "remote"); + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + let req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&refresh_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let resp = client + .refresh(req) + .await + .map_err(|e| LockError::internal(e.to_string()))? + .into_inner(); + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + Ok(resp.success) + } + + async fn force_release(&self, lock_id: &LockId) -> Result { + info!("remote force_release for {}", lock_id); + let force_request = self.create_unlock_request(lock_id, "remote"); + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + let req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&force_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let resp = client + .force_un_lock(req) + .await + .map_err(|e| LockError::internal(e.to_string()))? + .into_inner(); + if let Some(error_info) = resp.error_info { + return Err(LockError::internal(error_info)); + } + Ok(resp.success) + } + + async fn check_status(&self, lock_id: &LockId) -> Result> { + info!("remote check_status for {}", lock_id); + + // Since there's no direct status query in the gRPC service, + // we attempt a non-blocking lock acquisition to check if the resource is available + let status_request = self.create_unlock_request(lock_id, "remote"); + let mut client = node_service_time_out_client(&self.addr) + .await + .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?; + + // Try to acquire a very short-lived lock to test availability + let req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&status_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + + // Try exclusive lock first with very short timeout + let resp = client.lock(req).await; + + match resp { + Ok(response) => { + let resp = response.into_inner(); + if resp.success { + // If we successfully acquired the lock, the resource was free + // Immediately release it + let release_req = Request::new(GenerallyLockRequest { + args: serde_json::to_string(&status_request) + .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?, + }); + let _ = client.un_lock(release_req).await; // Best effort release + + // Return None since no one was holding the lock + Ok(None) + } else { + // Lock acquisition failed, meaning someone is holding it + // We can't determine the exact details remotely, so return a generic status + Ok(Some(LockInfo { + id: lock_id.clone(), + resource: lock_id.as_str().to_string(), + lock_type: crate::types::LockType::Exclusive, // We can't know the exact type + status: crate::types::LockStatus::Acquired, + owner: "unknown".to_string(), // Remote client can't determine owner + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), + last_refreshed: std::time::SystemTime::now(), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + wait_start_time: None, + })) + } + } + Err(_) => { + // Communication error or lock is held + Ok(Some(LockInfo { + id: lock_id.clone(), + resource: lock_id.as_str().to_string(), + lock_type: crate::types::LockType::Exclusive, + status: crate::types::LockStatus::Acquired, + owner: "unknown".to_string(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), + last_refreshed: std::time::SystemTime::now(), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + wait_start_time: None, + })) + } + } + } + + async fn get_stats(&self) -> Result { + info!("remote get_stats from {}", self.addr); + + // Since there's no direct statistics endpoint in the gRPC service, + // we return basic stats indicating this is a remote client + let stats = LockStats { + last_updated: std::time::SystemTime::now(), + ..Default::default() + }; + + // We could potentially enhance this by: + // 1. Keeping local counters of operations performed + // 2. Adding a stats gRPC method to the service + // 3. Querying server health endpoints + + // For now, return minimal stats indicating remote connectivity + Ok(stats) + } + + async fn close(&self) -> Result<()> { + Ok(()) + } + + async fn is_online(&self) -> bool { + // Use Ping interface to test if remote service is online + let mut client = match node_service_time_out_client(&self.addr).await { + Ok(client) => client, + Err(_) => { + info!("remote client {} connection failed", self.addr); + return false; + } + }; + + let ping_req = Request::new(PingRequest { + version: 1, + body: bytes::Bytes::new(), + }); + + match client.ping(ping_req).await { + Ok(_) => { + info!("remote client {} is online", self.addr); + true + } + Err(_) => { + info!("remote client {} ping failed", self.addr); + false + } + } + } + + async fn is_local(&self) -> bool { + false + } +} diff --git a/crates/lock/src/drwmutex.rs b/crates/lock/src/drwmutex.rs deleted file mode 100644 index bce263c35..000000000 --- a/crates/lock/src/drwmutex.rs +++ /dev/null @@ -1,1212 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::{Duration, Instant}; -use tokio::{sync::mpsc::Sender, time::sleep}; -use tracing::{info, warn}; - -use crate::{LockApi, Locker, lock_args::LockArgs}; - -const DRW_MUTEX_REFRESH_INTERVAL: Duration = Duration::from_secs(10); -const LOCK_RETRY_MIN_INTERVAL: Duration = Duration::from_millis(250); - -#[derive(Debug)] -pub struct DRWMutex { - owner: String, - names: Vec, - write_locks: Vec, - read_locks: Vec, - cancel_refresh_sender: Option>, - // rng: ThreadRng, - lockers: Vec, - refresh_interval: Duration, - lock_retry_min_interval: Duration, -} - -#[derive(Debug, Default, Clone)] -pub struct Granted { - index: usize, - lock_uid: String, -} - -impl Granted { - fn is_locked(&self) -> bool { - is_locked(&self.lock_uid) - } -} - -fn is_locked(uid: &str) -> bool { - !uid.is_empty() -} - -#[derive(Debug, Clone)] -pub struct Options { - pub timeout: Duration, - pub retry_interval: Duration, -} - -impl DRWMutex { - pub fn new(owner: String, names: Vec, lockers: Vec) -> Self { - let mut names = names; - names.sort(); - Self { - owner, - names, - write_locks: vec![String::new(); lockers.len()], - read_locks: vec![String::new(); lockers.len()], - cancel_refresh_sender: None, - // rng: rand::thread_rng(), - lockers, - refresh_interval: DRW_MUTEX_REFRESH_INTERVAL, - lock_retry_min_interval: LOCK_RETRY_MIN_INTERVAL, - } - } - - fn is_locked(&self) -> bool { - self.write_locks.iter().any(|w_lock| is_locked(w_lock)) - } - - fn is_r_locked(&self) -> bool { - self.read_locks.iter().any(|r_lock| is_locked(r_lock)) - } -} - -impl DRWMutex { - pub async fn lock(&mut self, id: &String, source: &String) { - let is_read_lock = false; - let opts = Options { - timeout: Duration::from_secs(10), - retry_interval: Duration::from_millis(50), - }; - self.lock_blocking(id, source, is_read_lock, &opts).await; - } - - pub async fn get_lock(&mut self, id: &String, source: &String, opts: &Options) -> bool { - let is_read_lock = false; - self.lock_blocking(id, source, is_read_lock, opts).await - } - - pub async fn r_lock(&mut self, id: &String, source: &String) { - let is_read_lock = true; - let opts = Options { - timeout: Duration::from_secs(10), - retry_interval: Duration::from_millis(50), - }; - self.lock_blocking(id, source, is_read_lock, &opts).await; - } - - pub async fn get_r_lock(&mut self, id: &String, source: &String, opts: &Options) -> bool { - let is_read_lock = true; - self.lock_blocking(id, source, is_read_lock, opts).await - } - - pub async fn lock_blocking(&mut self, id: &String, source: &String, is_read_lock: bool, opts: &Options) -> bool { - let locker_len = self.lockers.len(); - - // Handle edge case: no lockers available - if locker_len == 0 { - return false; - } - - let mut tolerance = locker_len / 2; - let mut quorum = locker_len - tolerance; - if !is_read_lock { - // In situations for write locks, as a special case - // to avoid split brains we make sure to acquire - // quorum + 1 when tolerance is exactly half of the - // total locker clients. - if quorum == tolerance { - quorum += 1; - } - } - info!( - "lockBlocking {}/{} for {:?}: lockType readLock({}), additional opts: {:?}, quorum: {}, tolerance: {}, lockClients: {}\n", - id, source, self.names, is_read_lock, opts, quorum, tolerance, locker_len - ); - - // Recalculate tolerance after potential quorum adjustment - // Use saturating_sub to prevent underflow - tolerance = locker_len.saturating_sub(quorum); - let mut attempt = 0; - let mut locks = vec!["".to_string(); self.lockers.len()]; - - loop { - if self.inner_lock(&mut locks, id, source, is_read_lock, tolerance, quorum).await { - if is_read_lock { - self.read_locks = locks; - } else { - self.write_locks = locks; - } - - info!("lock_blocking {}/{} for {:?}: granted", id, source, self.names); - - return true; - } - - attempt += 1; - if attempt >= 10 { - break; - } - sleep(opts.retry_interval).await; - } - - false - } - - async fn inner_lock( - &mut self, - locks: &mut [String], - id: &String, - source: &String, - is_read_lock: bool, - tolerance: usize, - quorum: usize, - ) -> bool { - locks.iter_mut().for_each(|lock| *lock = "".to_string()); - - let mut granteds = Vec::with_capacity(self.lockers.len()); - let args = LockArgs { - uid: id.to_string(), - resources: self.names.clone(), - owner: self.owner.clone(), - source: source.to_string(), - quorum, - }; - - for (index, locker) in self.lockers.iter_mut().enumerate() { - let mut granted = Granted { - index, - ..Default::default() - }; - - if is_read_lock { - match locker.rlock(&args).await { - Ok(locked) => { - if locked { - granted.lock_uid = id.to_string(); - } - } - Err(err) => { - warn!("Unable to call RLock failed with {} for {} at {:?}", err, args, locker); - } - } - } else { - match locker.lock(&args).await { - Ok(locked) => { - if locked { - granted.lock_uid = id.to_string(); - } - } - Err(err) => { - warn!("Unable to call Lock failed with {} for {} at {:?}", err, args, locker); - } - } - } - - granteds.push(granted); - } - - granteds.iter().for_each(|granted| { - locks[granted.index] = granted.lock_uid.clone(); - }); - - let quorum_locked = check_quorum_locked(locks, quorum); - if !quorum_locked { - info!("Unable to acquire lock in quorum, {}", args); - if !self.release_all(tolerance, locks, is_read_lock).await { - info!("Unable to release acquired locks, these locks will expire automatically {}", args); - } - } - - quorum_locked - } - - pub async fn un_lock(&mut self) { - if self.write_locks.is_empty() || !self.is_locked() { - warn!("Trying to un_lock() while no lock() is active, write_locks: {:?}", self.write_locks) - } - - let tolerance = self.lockers.len() / 2; - let is_read_lock = false; - let mut locks = std::mem::take(&mut self.write_locks); - let start = Instant::now(); - loop { - if self.release_all(tolerance, &mut locks, is_read_lock).await { - return; - } - - sleep(self.lock_retry_min_interval).await; - if Instant::now().duration_since(start) > Duration::from_secs(30) { - return; - } - } - } - - pub async fn un_r_lock(&mut self) { - if self.read_locks.is_empty() || !self.is_r_locked() { - warn!("Trying to un_r_lock() while no r_lock() is active, read_locks: {:?}", self.read_locks) - } - - let tolerance = self.lockers.len() / 2; - let is_read_lock = true; - let mut locks = std::mem::take(&mut self.read_locks); - let start = Instant::now(); - loop { - if self.release_all(tolerance, &mut locks, is_read_lock).await { - return; - } - - sleep(self.lock_retry_min_interval).await; - if Instant::now().duration_since(start) > Duration::from_secs(30) { - return; - } - } - } - - async fn release_all(&mut self, tolerance: usize, locks: &mut [String], is_read_lock: bool) -> bool { - for (index, locker) in self.lockers.iter_mut().enumerate() { - if send_release(locker, &locks[index], &self.owner, &self.names, is_read_lock).await { - locks[index] = "".to_string(); - } - } - - !check_failed_unlocks(locks, tolerance) - } -} - -// async fn start_continuous_lock_refresh(lockers: &Vec<&mut LockApi>, id: &String, source: &String, quorum: usize, refresh_interval: Duration, mut cancel_refresh_receiver: Receiver) { -// let uid = id.to_string(); -// tokio::spawn(async move { -// let mut ticker = interval(refresh_interval); -// let args = LockArgs { -// uid, -// ..Default::default() -// }; - -// loop { -// select! { -// _ = ticker.tick() => { -// for (index, locker) in lockers.iter().enumerate() { - -// } -// }, -// _ = cancel_refresh_receiver.recv() => { -// return; -// } -// } -// } -// }); -// } - -fn check_failed_unlocks(locks: &[String], tolerance: usize) -> bool { - let mut un_locks_failed = 0; - locks.iter().for_each(|lock| { - if is_locked(lock) { - un_locks_failed += 1; - } - }); - - // Handle edge case: if tolerance is greater than or equal to locks.len(), - // we can tolerate all failures, so return false (no critical failure) - if tolerance >= locks.len() { - return false; - } - - // Special case: when locks.len() - tolerance == tolerance (i.e., locks.len() == 2 * tolerance) - // This happens when we have an even number of lockers and tolerance is exactly half - if locks.len() - tolerance == tolerance { - return un_locks_failed >= tolerance; - } - - // Normal case: failure if more than tolerance unlocks failed - un_locks_failed > tolerance -} - -async fn send_release(locker: &mut LockApi, uid: &String, owner: &str, names: &[String], is_read_lock: bool) -> bool { - if uid.is_empty() { - return false; - } - - let args = LockArgs { - uid: uid.to_string(), - owner: owner.to_owned(), - resources: names.to_owned(), - ..Default::default() - }; - - if is_read_lock { - match locker.runlock(&args).await { - Ok(locked) => { - if !locked { - warn!("Unable to release runlock, args: {}", args); - return false; - } - } - Err(err) => { - warn!("Unable to call RLock failed with {} for {} at {:?}", err, args, locker); - return false; - } - } - } else { - match locker.unlock(&args).await { - Ok(locked) => { - if !locked { - warn!("Unable to release unlock, args: {}", args); - return false; - } - } - Err(err) => { - warn!("Unable to call Lock failed with {} for {} at {:?}", err, args, locker); - return false; - } - } - } - - true -} - -fn check_quorum_locked(locks: &[String], quorum: usize) -> bool { - let mut count = 0; - locks.iter().for_each(|lock| { - if is_locked(lock) { - count += 1; - } - }); - - count >= quorum -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::local_locker::LocalLocker; - use async_trait::async_trait; - use std::collections::HashMap; - use std::io::{Error, Result}; - use std::sync::{Arc, Mutex}; - - // Mock locker for testing - #[derive(Debug, Clone)] - struct MockLocker { - id: String, - state: Arc>, - } - - #[derive(Debug, Default)] - struct MockLockerState { - locks: HashMap, // uid -> owner - read_locks: HashMap, // uid -> owner - should_fail: bool, - is_online: bool, - } - - impl MockLocker { - fn new(id: String) -> Self { - Self { - id, - state: Arc::new(Mutex::new(MockLockerState { - is_online: true, - ..Default::default() - })), - } - } - - fn set_should_fail(&self, should_fail: bool) { - self.state.lock().unwrap().should_fail = should_fail; - } - - fn set_online(&self, online: bool) { - self.state.lock().unwrap().is_online = online; - } - - fn get_lock_count(&self) -> usize { - self.state.lock().unwrap().locks.len() - } - - fn get_read_lock_count(&self) -> usize { - self.state.lock().unwrap().read_locks.len() - } - - fn has_lock(&self, uid: &str) -> bool { - self.state.lock().unwrap().locks.contains_key(uid) - } - - fn has_read_lock(&self, uid: &str) -> bool { - self.state.lock().unwrap().read_locks.contains_key(uid) - } - } - - #[async_trait] - impl Locker for MockLocker { - async fn lock(&mut self, args: &LockArgs) -> Result { - let mut state = self.state.lock().unwrap(); - if state.should_fail { - return Err(Error::other("Mock lock failure")); - } - if !state.is_online { - return Err(Error::other("Mock locker offline")); - } - - // Check if already locked - if state.locks.contains_key(&args.uid) { - return Ok(false); - } - - state.locks.insert(args.uid.clone(), args.owner.clone()); - Ok(true) - } - - async fn unlock(&mut self, args: &LockArgs) -> Result { - let mut state = self.state.lock().unwrap(); - if state.should_fail { - return Err(Error::other("Mock unlock failure")); - } - - Ok(state.locks.remove(&args.uid).is_some()) - } - - async fn rlock(&mut self, args: &LockArgs) -> Result { - let mut state = self.state.lock().unwrap(); - if state.should_fail { - return Err(Error::other("Mock rlock failure")); - } - if !state.is_online { - return Err(Error::other("Mock locker offline")); - } - - // Check if write lock exists - if state.locks.contains_key(&args.uid) { - return Ok(false); - } - - state.read_locks.insert(args.uid.clone(), args.owner.clone()); - Ok(true) - } - - async fn runlock(&mut self, args: &LockArgs) -> Result { - let mut state = self.state.lock().unwrap(); - if state.should_fail { - return Err(Error::other("Mock runlock failure")); - } - - Ok(state.read_locks.remove(&args.uid).is_some()) - } - - async fn refresh(&mut self, _args: &LockArgs) -> Result { - let state = self.state.lock().unwrap(); - if state.should_fail { - return Err(Error::other("Mock refresh failure")); - } - Ok(true) - } - - async fn force_unlock(&mut self, args: &LockArgs) -> Result { - let mut state = self.state.lock().unwrap(); - let removed_lock = state.locks.remove(&args.uid).is_some(); - let removed_read_lock = state.read_locks.remove(&args.uid).is_some(); - Ok(removed_lock || removed_read_lock) - } - - async fn close(&self) {} - - async fn is_online(&self) -> bool { - self.state.lock().unwrap().is_online - } - - async fn is_local(&self) -> bool { - true - } - } - - fn create_mock_lockers(count: usize) -> Vec { - // For testing, we'll use Local lockers which use the global local server - (0..count).map(|_| LockApi::Local).collect() - } - - #[test] - fn test_drw_mutex_new() { - let names = vec!["resource1".to_string(), "resource2".to_string()]; - let lockers = create_mock_lockers(3); - let mutex = DRWMutex::new("owner1".to_string(), names.clone(), lockers); - - assert_eq!(mutex.owner, "owner1"); - assert_eq!(mutex.names.len(), 2); - assert_eq!(mutex.lockers.len(), 3); - assert_eq!(mutex.write_locks.len(), 3); - assert_eq!(mutex.read_locks.len(), 3); - assert_eq!(mutex.refresh_interval, DRW_MUTEX_REFRESH_INTERVAL); - assert_eq!(mutex.lock_retry_min_interval, LOCK_RETRY_MIN_INTERVAL); - - // Names should be sorted - let mut expected_names = names; - expected_names.sort(); - assert_eq!(mutex.names, expected_names); - } - - #[test] - fn test_drw_mutex_new_empty_names() { - let names = vec![]; - let lockers = create_mock_lockers(1); - let mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - assert_eq!(mutex.names.len(), 0); - assert_eq!(mutex.lockers.len(), 1); - } - - #[test] - fn test_drw_mutex_new_single_locker() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); - let mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - assert_eq!(mutex.lockers.len(), 1); - assert_eq!(mutex.write_locks.len(), 1); - assert_eq!(mutex.read_locks.len(), 1); - } - - #[test] - fn test_is_locked_function() { - assert!(!is_locked("")); - assert!(is_locked("some-uid")); - assert!(is_locked("any-non-empty-string")); - } - - #[test] - fn test_granted_is_locked() { - let granted_empty = Granted { - index: 0, - lock_uid: "".to_string(), - }; - assert!(!granted_empty.is_locked()); - - let granted_locked = Granted { - index: 1, - lock_uid: "test-uid".to_string(), - }; - assert!(granted_locked.is_locked()); - } - - #[test] - fn test_drw_mutex_is_locked() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(2); - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - // Initially not locked - assert!(!mutex.is_locked()); - assert!(!mutex.is_r_locked()); - - // Set write locks - mutex.write_locks[0] = "test-uid".to_string(); - assert!(mutex.is_locked()); - assert!(!mutex.is_r_locked()); - - // Clear write locks, set read locks - mutex.write_locks[0] = "".to_string(); - mutex.read_locks[1] = "read-uid".to_string(); - assert!(!mutex.is_locked()); - assert!(mutex.is_r_locked()); - } - - #[test] - fn test_options_debug() { - let opts = Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_millis(100), - }; - let debug_str = format!("{opts:?}"); - assert!(debug_str.contains("timeout")); - assert!(debug_str.contains("retry_interval")); - } - - #[test] - fn test_check_quorum_locked() { - // Test with empty locks - assert!(!check_quorum_locked(&[], 1)); - - // Test with all empty locks - let locks = vec!["".to_string(), "".to_string(), "".to_string()]; - assert!(!check_quorum_locked(&locks, 1)); - assert!(!check_quorum_locked(&locks, 2)); - - // Test with some locks - let locks = vec!["uid1".to_string(), "".to_string(), "uid3".to_string()]; - assert!(check_quorum_locked(&locks, 1)); - assert!(check_quorum_locked(&locks, 2)); - assert!(!check_quorum_locked(&locks, 3)); - - // Test with all locks - let locks = vec!["uid1".to_string(), "uid2".to_string(), "uid3".to_string()]; - assert!(check_quorum_locked(&locks, 1)); - assert!(check_quorum_locked(&locks, 2)); - assert!(check_quorum_locked(&locks, 3)); - assert!(!check_quorum_locked(&locks, 4)); - } - - #[test] - fn test_check_failed_unlocks() { - // Test with empty locks - assert!(!check_failed_unlocks(&[], 0)); // tolerance >= locks.len(), so no critical failure - assert!(!check_failed_unlocks(&[], 1)); // tolerance >= locks.len(), so no critical failure - - // Test with all unlocked - let locks = vec!["".to_string(), "".to_string(), "".to_string()]; - assert!(!check_failed_unlocks(&locks, 1)); // 0 failed <= tolerance 1 - assert!(!check_failed_unlocks(&locks, 2)); // 0 failed <= tolerance 2 - - // Test with some failed unlocks - let locks = vec!["uid1".to_string(), "".to_string(), "uid3".to_string()]; - assert!(check_failed_unlocks(&locks, 1)); // 2 failed > tolerance 1 - assert!(!check_failed_unlocks(&locks, 2)); // 2 failed <= tolerance 2 - - // Test special case: locks.len() - tolerance == tolerance - // This means locks.len() == 2 * tolerance - let locks = vec!["uid1".to_string(), "uid2".to_string()]; // len = 2 - let tolerance = 1; // 2 - 1 == 1 - assert!(check_failed_unlocks(&locks, tolerance)); // 2 failed >= tolerance 1 - - let locks = vec!["".to_string(), "uid2".to_string()]; // len = 2, 1 failed - assert!(check_failed_unlocks(&locks, tolerance)); // 1 failed >= tolerance 1 - - let locks = vec!["".to_string(), "".to_string()]; // len = 2, 0 failed - assert!(!check_failed_unlocks(&locks, tolerance)); // 0 failed < tolerance 1 - } - - #[test] - fn test_check_failed_unlocks_edge_cases() { - // Test with zero tolerance - let locks = vec!["uid1".to_string()]; - assert!(check_failed_unlocks(&locks, 0)); // 1 failed > tolerance 0 - - // Test with tolerance equal to lock count - let locks = vec!["uid1".to_string(), "uid2".to_string()]; - assert!(!check_failed_unlocks(&locks, 2)); // 2 failed <= tolerance 2 - - // Test with tolerance greater than lock count - let locks = vec!["uid1".to_string()]; - assert!(!check_failed_unlocks(&locks, 5)); // 1 failed <= tolerance 5 - } - - // Async tests using the local locker infrastructure - #[tokio::test] - async fn test_drw_mutex_lock_basic_functionality() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); // Single locker for simplicity - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id = "test-lock-id".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(1), - retry_interval: Duration::from_millis(10), - }; - - // Test get_lock (result depends on local locker state) - let _result = mutex.get_lock(&id, &source, &opts).await; - // Just ensure the method doesn't panic and returns a boolean - // assert!(result || !result); // This is always true, so removed - - // If lock was acquired, test unlock - if _result { - assert!(mutex.is_locked(), "Mutex should be in locked state"); - mutex.un_lock().await; - assert!(!mutex.is_locked(), "Mutex should be unlocked after un_lock"); - } - } - - #[tokio::test] - async fn test_drw_mutex_rlock_basic_functionality() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); // Single locker for simplicity - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id = "test-rlock-id".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(1), - retry_interval: Duration::from_millis(10), - }; - - // Test get_r_lock (result depends on local locker state) - let _result = mutex.get_r_lock(&id, &source, &opts).await; - // Just ensure the method doesn't panic and returns a boolean - // assert!(result || !result); // This is always true, so removed - - // If read lock was acquired, test runlock - if _result { - assert!(mutex.is_r_locked(), "Mutex should be in read locked state"); - mutex.un_r_lock().await; - assert!(!mutex.is_r_locked(), "Mutex should be unlocked after un_r_lock"); - } - } - - #[tokio::test] - async fn test_drw_mutex_lock_with_multiple_lockers() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(3); // 3 lockers, need quorum of 2 - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id = "test-lock-id".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(1), - retry_interval: Duration::from_millis(10), - }; - - // With 3 local lockers, the quorum calculation should be: - // tolerance = 3 / 2 = 1 - // quorum = 3 - 1 = 2 - // Since it's a write lock and quorum != tolerance, quorum stays 2 - // The result depends on the actual locker implementation - let _result = mutex.get_lock(&id, &source, &opts).await; - // We don't assert success/failure here since it depends on the local locker state - // Just ensure the method doesn't panic and returns a boolean - // assert!(result || !result); // This is always true, so removed - } - - #[tokio::test] - async fn test_drw_mutex_unlock_without_lock() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - // Try to unlock without having a lock - should not panic - mutex.un_lock().await; - assert!(!mutex.is_locked()); - - // Try to unlock read lock without having one - should not panic - mutex.un_r_lock().await; - assert!(!mutex.is_r_locked()); - } - - #[tokio::test] - async fn test_drw_mutex_multiple_resources() { - let names = vec!["resource1".to_string(), "resource2".to_string(), "resource3".to_string()]; - let lockers = create_mock_lockers(1); - let mut mutex = DRWMutex::new("owner1".to_string(), names.clone(), lockers); - - // Names should be sorted - let mut expected_names = names; - expected_names.sort(); - assert_eq!(mutex.names, expected_names); - - let id = "test-lock-id".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(1), - retry_interval: Duration::from_millis(10), - }; - - let _result = mutex.get_lock(&id, &source, &opts).await; - // The result depends on the actual locker implementation - // Just ensure the method doesn't panic and returns a boolean - // assert!(result || !result); // This is always true, so removed - } - - #[tokio::test] - async fn test_drw_mutex_concurrent_read_locks() { - // Clear global state before test to avoid interference from other tests - { - let mut global_server = crate::GLOBAL_LOCAL_SERVER.write().await; - *global_server = LocalLocker::new(); - } - - // Use a single mutex with one resource for simplicity - let names = vec!["test-resource".to_string()]; - let lockers = create_mock_lockers(1); - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id1 = "test-rlock-id1".to_string(); - let id2 = "test-rlock-id2".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_millis(50), - }; - - // First acquire a read lock - let result1 = mutex.get_r_lock(&id1, &source, &opts).await; - assert!(result1, "First read lock should succeed"); - - // Release the first read lock - mutex.un_r_lock().await; - - // Then acquire another read lock with different ID - this should succeed - let result2 = mutex.get_r_lock(&id2, &source, &opts).await; - assert!(result2, "Second read lock should succeed after first is released"); - - // Clean up - mutex.un_r_lock().await; - } - - #[tokio::test] - async fn test_send_release_with_empty_uid() { - let mut locker = LockApi::Local; - let result = send_release(&mut locker, &"".to_string(), "owner", &["resource".to_string()], false).await; - assert!(!result, "send_release should return false for empty uid"); - } - - #[test] - fn test_drw_mutex_debug() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); - let mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let debug_str = format!("{mutex:?}"); - assert!(debug_str.contains("DRWMutex")); - assert!(debug_str.contains("owner")); - assert!(debug_str.contains("names")); - } - - #[test] - fn test_granted_default() { - let granted = Granted::default(); - assert_eq!(granted.index, 0); - assert_eq!(granted.lock_uid, ""); - assert!(!granted.is_locked()); - } - - #[test] - fn test_granted_clone() { - let granted = Granted { - index: 5, - lock_uid: "test-uid".to_string(), - }; - let cloned = granted.clone(); - assert_eq!(granted.index, cloned.index); - assert_eq!(granted.lock_uid, cloned.lock_uid); - } - - // Test potential bug scenarios - #[test] - fn test_potential_bug_check_failed_unlocks_logic() { - // This test highlights the potentially confusing logic in check_failed_unlocks - - // Case 1: Even number of lockers - let locks = vec!["uid1".to_string(), "uid2".to_string(), "uid3".to_string(), "uid4".to_string()]; - let tolerance = 2; // locks.len() / 2 = 4 / 2 = 2 - // locks.len() - tolerance = 4 - 2 = 2, which equals tolerance - // So the special case applies: un_locks_failed >= tolerance - - // All 4 failed unlocks - assert!(check_failed_unlocks(&locks, tolerance)); // 4 >= 2 = true - - // 2 failed unlocks - let locks = vec!["uid1".to_string(), "uid2".to_string(), "".to_string(), "".to_string()]; - assert!(check_failed_unlocks(&locks, tolerance)); // 2 >= 2 = true - - // 1 failed unlock - let locks = vec!["uid1".to_string(), "".to_string(), "".to_string(), "".to_string()]; - assert!(!check_failed_unlocks(&locks, tolerance)); // 1 >= 2 = false - - // Case 2: Odd number of lockers - let locks = vec!["uid1".to_string(), "uid2".to_string(), "uid3".to_string()]; - let tolerance = 1; // locks.len() / 2 = 3 / 2 = 1 - // locks.len() - tolerance = 3 - 1 = 2, which does NOT equal tolerance (1) - // So the normal case applies: un_locks_failed > tolerance - - // 3 failed unlocks - assert!(check_failed_unlocks(&locks, tolerance)); // 3 > 1 = true - - // 2 failed unlocks - let locks = vec!["uid1".to_string(), "uid2".to_string(), "".to_string()]; - assert!(check_failed_unlocks(&locks, tolerance)); // 2 > 1 = true - - // 1 failed unlock - let locks = vec!["uid1".to_string(), "".to_string(), "".to_string()]; - assert!(!check_failed_unlocks(&locks, tolerance)); // 1 > 1 = false - } - - #[test] - fn test_quorum_calculation_edge_cases() { - // Test the quorum calculation logic that might have issues - - // For 1 locker: tolerance = 0, quorum = 1 - // Write lock: quorum == tolerance (1 == 0 is false), so quorum stays 1 - // This seems wrong - with 1 locker, we should need that 1 locker - - // For 2 lockers: tolerance = 1, quorum = 1 - // Write lock: quorum == tolerance (1 == 1 is true), so quorum becomes 2 - // This makes sense - we need both lockers for write lock - - // For 3 lockers: tolerance = 1, quorum = 2 - // Write lock: quorum == tolerance (2 == 1 is false), so quorum stays 2 - - // For 4 lockers: tolerance = 2, quorum = 2 - // Write lock: quorum == tolerance (2 == 2 is true), so quorum becomes 3 - - // The logic seems to be: for write locks, if exactly half the lockers - // would be tolerance, we need one more to avoid split brain - - // Let's verify this makes sense: - struct QuorumTest { - locker_count: usize, - expected_tolerance: usize, - expected_write_quorum: usize, - expected_read_quorum: usize, - } - - let test_cases = vec![ - QuorumTest { - locker_count: 1, - expected_tolerance: 0, - expected_write_quorum: 1, - expected_read_quorum: 1, - }, - QuorumTest { - locker_count: 2, - expected_tolerance: 1, - expected_write_quorum: 2, - expected_read_quorum: 1, - }, - QuorumTest { - locker_count: 3, - expected_tolerance: 1, - expected_write_quorum: 2, - expected_read_quorum: 2, - }, - QuorumTest { - locker_count: 4, - expected_tolerance: 2, - expected_write_quorum: 3, - expected_read_quorum: 2, - }, - QuorumTest { - locker_count: 5, - expected_tolerance: 2, - expected_write_quorum: 3, - expected_read_quorum: 3, - }, - ]; - - for test_case in test_cases { - let tolerance = test_case.locker_count / 2; - let mut write_quorum = test_case.locker_count - tolerance; - let read_quorum = write_quorum; - - // Apply write lock special case - if write_quorum == tolerance { - write_quorum += 1; - } - - assert_eq!( - tolerance, test_case.expected_tolerance, - "Tolerance mismatch for {} lockers", - test_case.locker_count - ); - assert_eq!( - write_quorum, test_case.expected_write_quorum, - "Write quorum mismatch for {} lockers", - test_case.locker_count - ); - assert_eq!( - read_quorum, test_case.expected_read_quorum, - "Read quorum mismatch for {} lockers", - test_case.locker_count - ); - } - } - - #[test] - fn test_potential_integer_overflow() { - // Test potential issues with tolerance calculation - - // What happens with 0 lockers? This should probably be an error case - let locker_count = 0; - let tolerance = locker_count / 2; // 0 / 2 = 0 - let quorum = locker_count - tolerance; // 0 - 0 = 0 - - // This would result in quorum = 0, which doesn't make sense - assert_eq!(tolerance, 0); - assert_eq!(quorum, 0); - - // The code should probably validate that locker_count > 0 - } - - #[test] - fn test_drw_mutex_constants() { - // Test that constants are reasonable - assert!(DRW_MUTEX_REFRESH_INTERVAL.as_secs() > 0); - assert!(LOCK_RETRY_MIN_INTERVAL.as_millis() > 0); - assert!(DRW_MUTEX_REFRESH_INTERVAL > LOCK_RETRY_MIN_INTERVAL); - } - - #[test] - fn test_drw_mutex_new_with_unsorted_names() { - let names = vec!["zebra".to_string(), "alpha".to_string(), "beta".to_string()]; - let lockers = create_mock_lockers(1); - let mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - // Names should be sorted - assert_eq!(mutex.names, vec!["alpha", "beta", "zebra"]); - } - - #[test] - fn test_drw_mutex_new_with_duplicate_names() { - let names = vec![ - "resource1".to_string(), - "resource2".to_string(), - "resource1".to_string(), // Duplicate - ]; - let lockers = create_mock_lockers(1); - let mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - // Should keep duplicates but sort them - assert_eq!(mutex.names, vec!["resource1", "resource1", "resource2"]); - } - - #[tokio::test] - async fn test_drw_mutex_lock_and_rlock_methods() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(1); - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id = "test-id".to_string(); - let source = "test-source".to_string(); - - // Test the convenience methods (lock and r_lock) - // These should not panic and should attempt to acquire locks - mutex.lock(&id, &source).await; - // Note: We can't easily test the result since these methods don't return bool - - // Clear any state - mutex.un_lock().await; - - // Test r_lock - mutex.r_lock(&id, &source).await; - mutex.un_r_lock().await; - } - - #[tokio::test] - async fn test_drw_mutex_zero_lockers() { - let names = vec!["resource1".to_string()]; - let lockers = vec![]; // No lockers - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - let id = "test-id".to_string(); - let source = "test-source".to_string(); - let opts = Options { - timeout: Duration::from_secs(1), - retry_interval: Duration::from_millis(10), - }; - - // With 0 lockers, quorum calculation: - // tolerance = 0 / 2 = 0 - // quorum = 0 - 0 = 0 - // This should fail because we can't achieve any quorum - let _result = mutex.get_lock(&id, &source, &opts).await; - assert!(!_result, "Should fail with zero lockers"); - } - - #[test] - fn test_check_quorum_locked_edge_cases() { - // Test with quorum 0 - let locks = vec!["".to_string()]; - assert!(check_quorum_locked(&locks, 0)); // 0 >= 0 - - // Test with quorum larger than locks - let locks = vec!["uid1".to_string()]; - assert!(!check_quorum_locked(&locks, 5)); // 1 < 5 - - // Test with all locks but high quorum - let locks = vec!["uid1".to_string(), "uid2".to_string(), "uid3".to_string()]; - assert!(!check_quorum_locked(&locks, 4)); // 3 < 4 - } - - #[test] - fn test_check_failed_unlocks_comprehensive() { - // Test all combinations for small lock counts - - // 1 lock scenarios - assert!(!check_failed_unlocks(&["".to_string()], 0)); // 1 success, tolerance 0 -> 1 > 0 = true, but tolerance >= len, so false - assert!(!check_failed_unlocks(&["".to_string()], 1)); // tolerance >= len - assert!(!check_failed_unlocks(&["uid".to_string()], 1)); // tolerance >= len - assert!(check_failed_unlocks(&["uid".to_string()], 0)); // 1 failed > 0 - - // 2 lock scenarios - let two_failed = vec!["uid1".to_string(), "uid2".to_string()]; - let one_failed = vec!["uid1".to_string(), "".to_string()]; - let zero_failed = vec!["".to_string(), "".to_string()]; - - // tolerance = 0 - assert!(check_failed_unlocks(&two_failed, 0)); // 2 > 0 - assert!(check_failed_unlocks(&one_failed, 0)); // 1 > 0 - assert!(!check_failed_unlocks(&zero_failed, 0)); // 0 > 0 = false - - // tolerance = 1 (special case: 2 - 1 == 1) - assert!(check_failed_unlocks(&two_failed, 1)); // 2 >= 1 - assert!(check_failed_unlocks(&one_failed, 1)); // 1 >= 1 - assert!(!check_failed_unlocks(&zero_failed, 1)); // 0 >= 1 = false - - // tolerance = 2 - assert!(!check_failed_unlocks(&two_failed, 2)); // tolerance >= len - assert!(!check_failed_unlocks(&one_failed, 2)); // tolerance >= len - assert!(!check_failed_unlocks(&zero_failed, 2)); // tolerance >= len - } - - #[test] - fn test_options_clone() { - let opts = Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_millis(100), - }; - let cloned = opts.clone(); - assert_eq!(opts.timeout, cloned.timeout); - assert_eq!(opts.retry_interval, cloned.retry_interval); - } - - #[tokio::test] - async fn test_drw_mutex_release_all_edge_cases() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(2); - let mut mutex = DRWMutex::new("owner1".to_string(), names, lockers); - - // Test release_all with empty locks - let mut empty_locks = vec!["".to_string(), "".to_string()]; - let result = mutex.release_all(1, &mut empty_locks, false).await; - assert!(result, "Should succeed when releasing empty locks"); - - // Test release_all with some locks - let mut some_locks = vec!["uid1".to_string(), "uid2".to_string()]; - let result = mutex.release_all(1, &mut some_locks, false).await; - // This should attempt to release the locks and may succeed or fail - // depending on the local locker state - just ensure it doesn't panic - let _ = result; // Suppress unused variable warning - } - - #[test] - fn test_drw_mutex_struct_fields() { - let names = vec!["resource1".to_string()]; - let lockers = create_mock_lockers(2); - let mutex = DRWMutex::new("test-owner".to_string(), names, lockers); - - // Test that all fields are properly initialized - assert_eq!(mutex.owner, "test-owner"); - assert_eq!(mutex.names, vec!["resource1"]); - assert_eq!(mutex.write_locks.len(), 2); - assert_eq!(mutex.read_locks.len(), 2); - assert_eq!(mutex.lockers.len(), 2); - assert!(mutex.cancel_refresh_sender.is_none()); - assert_eq!(mutex.refresh_interval, DRW_MUTEX_REFRESH_INTERVAL); - assert_eq!(mutex.lock_retry_min_interval, LOCK_RETRY_MIN_INTERVAL); - - // All locks should be initially empty - for lock in &mutex.write_locks { - assert!(lock.is_empty()); - } - for lock in &mutex.read_locks { - assert!(lock.is_empty()); - } - } -} diff --git a/crates/lock/src/error.rs b/crates/lock/src/error.rs new file mode 100644 index 000000000..c64a5f852 --- /dev/null +++ b/crates/lock/src/error.rs @@ -0,0 +1,329 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::types::LockId; +use std::time::Duration; +use thiserror::Error; + +/// Lock operation related error types +#[derive(Error, Debug)] +pub enum LockError { + /// Lock acquisition timeout + #[error("Lock acquisition timeout for resource '{resource}' after {timeout:?}")] + Timeout { resource: String, timeout: Duration }, + + /// Resource not found + #[error("Resource not found: {resource}")] + ResourceNotFound { resource: String }, + + /// Permission denied + #[error("Permission denied: {reason}")] + PermissionDenied { reason: String }, + + /// Network error + #[error("Network error: {message}")] + Network { + message: String, + #[source] + source: Box, + }, + + /// Internal error + #[error("Internal error: {message}")] + Internal { message: String }, + + /// Resource is already locked + #[error("Resource '{resource}' is already locked by {owner}")] + AlreadyLocked { resource: String, owner: String }, + + /// Invalid lock handle + #[error("Invalid lock handle: {handle_id}")] + InvalidHandle { handle_id: String }, + + /// Configuration error + #[error("Configuration error: {message}")] + Configuration { message: String }, + + /// Serialization error + #[error("Serialization error: {message}")] + Serialization { + message: String, + #[source] + source: Box, + }, + + /// Deserialization error + #[error("Deserialization error: {message}")] + Deserialization { + message: String, + #[source] + source: Box, + }, + + /// Insufficient nodes for quorum + #[error("Insufficient nodes for quorum: required {required}, available {available}")] + InsufficientNodes { required: usize, available: usize }, + + /// Quorum not reached + #[error("Quorum not reached: required {required}, achieved {achieved}")] + QuorumNotReached { required: usize, achieved: usize }, + + /// Queue is full + #[error("Queue is full: {message}")] + QueueFull { message: String }, + + /// Not the lock owner + #[error("Not the lock owner: lock_id {lock_id}, owner {owner}")] + NotOwner { lock_id: LockId, owner: String }, +} + +impl Clone for LockError { + fn clone(&self) -> Self { + match self { + LockError::Timeout { resource, timeout } => LockError::Timeout { + resource: resource.clone(), + timeout: *timeout, + }, + LockError::ResourceNotFound { resource } => LockError::ResourceNotFound { + resource: resource.clone(), + }, + LockError::PermissionDenied { reason } => LockError::PermissionDenied { reason: reason.clone() }, + LockError::Network { message, source: _ } => LockError::Network { + message: message.clone(), + source: Box::new(std::io::Error::other(message.clone())), + }, + LockError::Internal { message } => LockError::Internal { + message: message.clone(), + }, + LockError::AlreadyLocked { resource, owner } => LockError::AlreadyLocked { + resource: resource.clone(), + owner: owner.clone(), + }, + LockError::InvalidHandle { handle_id } => LockError::InvalidHandle { + handle_id: handle_id.clone(), + }, + LockError::Configuration { message } => LockError::Configuration { + message: message.clone(), + }, + LockError::Serialization { message, source: _ } => LockError::Serialization { + message: message.clone(), + source: Box::new(std::io::Error::other(message.clone())), + }, + LockError::Deserialization { message, source: _ } => LockError::Deserialization { + message: message.clone(), + source: Box::new(std::io::Error::other(message.clone())), + }, + LockError::InsufficientNodes { required, available } => LockError::InsufficientNodes { + required: *required, + available: *available, + }, + LockError::QuorumNotReached { required, achieved } => LockError::QuorumNotReached { + required: *required, + achieved: *achieved, + }, + LockError::QueueFull { message } => LockError::QueueFull { + message: message.clone(), + }, + LockError::NotOwner { lock_id, owner } => LockError::NotOwner { + lock_id: lock_id.clone(), + owner: owner.clone(), + }, + } + } +} + +impl LockError { + /// Create timeout error + pub fn timeout(resource: impl Into, timeout: Duration) -> Self { + Self::Timeout { + resource: resource.into(), + timeout, + } + } + + /// Create resource not found error + pub fn resource_not_found(resource: impl Into) -> Self { + Self::ResourceNotFound { + resource: resource.into(), + } + } + + /// Create permission denied error + pub fn permission_denied(reason: impl Into) -> Self { + Self::PermissionDenied { reason: reason.into() } + } + + /// Create network error + pub fn network(message: impl Into, source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Network { + message: message.into(), + source: Box::new(source), + } + } + + /// Create internal error + pub fn internal(message: impl Into) -> Self { + Self::Internal { message: message.into() } + } + + /// Create lock already locked error + pub fn already_locked(resource: impl Into, owner: impl Into) -> Self { + Self::AlreadyLocked { + resource: resource.into(), + owner: owner.into(), + } + } + + /// Create invalid handle error + pub fn invalid_handle(handle_id: impl Into) -> Self { + Self::InvalidHandle { + handle_id: handle_id.into(), + } + } + + /// Create configuration error + pub fn configuration(message: impl Into) -> Self { + Self::Configuration { message: message.into() } + } + + /// Create serialization error + pub fn serialization(message: impl Into, source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Serialization { + message: message.into(), + source: Box::new(source), + } + } + + /// Create deserialization error + pub fn deserialization(message: impl Into, source: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Deserialization { + message: message.into(), + source: Box::new(source), + } + } + + /// Check if it is a retryable error + pub fn is_retryable(&self) -> bool { + matches!(self, Self::Timeout { .. } | Self::Network { .. } | Self::Internal { .. }) + } + + /// Check if it is a fatal error + pub fn is_fatal(&self) -> bool { + matches!( + self, + Self::ResourceNotFound { .. } | Self::PermissionDenied { .. } | Self::Configuration { .. } + ) + } +} + +/// Lock operation Result type +pub type Result = std::result::Result; + +/// Convert from std::io::Error +impl From for LockError { + fn from(err: std::io::Error) -> Self { + match err.kind() { + std::io::ErrorKind::TimedOut => Self::Internal { + message: "IO timeout".to_string(), + }, + std::io::ErrorKind::NotFound => Self::ResourceNotFound { + resource: "unknown".to_string(), + }, + std::io::ErrorKind::PermissionDenied => Self::PermissionDenied { reason: err.to_string() }, + _ => Self::Internal { + message: err.to_string(), + }, + } + } +} + +/// Convert from serde_json::Error +impl From for LockError { + fn from(err: serde_json::Error) -> Self { + if err.is_io() { + Self::network("JSON serialization IO error", err) + } else if err.is_syntax() { + Self::deserialization("JSON syntax error", err) + } else if err.is_data() { + Self::deserialization("JSON data error", err) + } else { + Self::serialization("JSON serialization error", err) + } + } +} + +/// Convert from tonic::Status +impl From for LockError { + fn from(status: tonic::Status) -> Self { + match status.code() { + tonic::Code::DeadlineExceeded => Self::Internal { + message: "gRPC deadline exceeded".to_string(), + }, + tonic::Code::NotFound => Self::ResourceNotFound { + resource: "unknown".to_string(), + }, + tonic::Code::PermissionDenied => Self::PermissionDenied { + reason: status.message().to_string(), + }, + tonic::Code::Unavailable => Self::Network { + message: "gRPC service unavailable".to_string(), + source: Box::new(status), + }, + _ => Self::Internal { + message: status.message().to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_creation() { + let timeout_err = LockError::timeout("test-resource", Duration::from_secs(5)); + assert!(matches!(timeout_err, LockError::Timeout { .. })); + + let not_found_err = LockError::resource_not_found("missing-resource"); + assert!(matches!(not_found_err, LockError::ResourceNotFound { .. })); + + let permission_err = LockError::permission_denied("insufficient privileges"); + assert!(matches!(permission_err, LockError::PermissionDenied { .. })); + } + + #[test] + fn test_error_retryable() { + let timeout_err = LockError::timeout("test", Duration::from_secs(1)); + assert!(timeout_err.is_retryable()); + + let network_err = LockError::network("connection failed", std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "")); + assert!(network_err.is_retryable()); + + let not_found_err = LockError::resource_not_found("test"); + assert!(!not_found_err.is_retryable()); + } + + #[test] + fn test_error_fatal() { + let not_found_err = LockError::resource_not_found("test"); + assert!(not_found_err.is_fatal()); + + let permission_err = LockError::permission_denied("test"); + assert!(permission_err.is_fatal()); + + let timeout_err = LockError::timeout("test", Duration::from_secs(1)); + assert!(!timeout_err.is_fatal()); + } +} diff --git a/crates/lock/src/lib.rs b/crates/lock/src/lib.rs index 04f6cf266..b2a614d80 100644 --- a/crates/lock/src/lib.rs +++ b/crates/lock/src/lib.rs @@ -1,4 +1,4 @@ -#![allow(dead_code)] +// #![allow(dead_code)] // Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,115 +13,78 @@ // See the License for the specific language governing permissions and // limitations under the License. -use async_trait::async_trait; -use local_locker::LocalLocker; -use lock_args::LockArgs; -use remote_client::RemoteClient; -use std::io::Result; +// ============================================================================ +// Core Module Declarations +// ============================================================================ + +// Application Layer Modules +pub mod namespace; + +// Abstraction Layer Modules +pub mod client; + +// Local Layer Modules +pub mod local; + +// Core Modules +pub mod error; +pub mod types; + +// ============================================================================ +// Public API Exports +// ============================================================================ + +// Re-export main types for easy access +pub use crate::{ + // Client interfaces + client::{LockClient, local::LocalClient, remote::RemoteClient}, + // Error types + error::{LockError, Result}, + local::LocalLockMap, + // Main components + namespace::{NamespaceLock, NamespaceLockManager}, + // Core types + types::{ + HealthInfo, HealthStatus, LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockStatus, + LockType, + }, +}; + +// ============================================================================ +// Version Information +// ============================================================================ + +/// Current version of the lock crate +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// Build timestamp +pub const BUILD_TIMESTAMP: &str = "unknown"; + +/// Maximum number of items in delete list +pub const MAX_DELETE_LIST: usize = 1000; + +// ============================================================================ +// Global Lock Map +// ============================================================================ + +// Global singleton lock map shared across all lock implementations +use once_cell::sync::OnceCell; use std::sync::Arc; -use std::sync::LazyLock; -use tokio::sync::RwLock; -pub mod drwmutex; -pub mod local_locker; -pub mod lock_args; -pub mod lrwmutex; -pub mod namespace_lock; -pub mod remote_client; +static GLOBAL_LOCK_MAP: OnceCell> = OnceCell::new(); -pub static GLOBAL_LOCAL_SERVER: LazyLock>> = LazyLock::new(|| Arc::new(RwLock::new(LocalLocker::new()))); - -type LockClient = dyn Locker; - -#[async_trait] -pub trait Locker { - async fn lock(&mut self, args: &LockArgs) -> Result; - async fn unlock(&mut self, args: &LockArgs) -> Result; - async fn rlock(&mut self, args: &LockArgs) -> Result; - async fn runlock(&mut self, args: &LockArgs) -> Result; - async fn refresh(&mut self, args: &LockArgs) -> Result; - async fn force_unlock(&mut self, args: &LockArgs) -> Result; - async fn close(&self); - async fn is_online(&self) -> bool; - async fn is_local(&self) -> bool; +/// Get the global shared lock map instance +pub fn get_global_lock_map() -> Arc { + GLOBAL_LOCK_MAP.get_or_init(|| Arc::new(local::LocalLockMap::new())).clone() } -#[derive(Debug, Clone)] -pub enum LockApi { - Local, - Remote(RemoteClient), -} - -#[async_trait] -impl Locker for LockApi { - async fn lock(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.lock(args).await, - LockApi::Remote(r) => r.lock(args).await, - } - } - - async fn unlock(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.unlock(args).await, - LockApi::Remote(r) => r.unlock(args).await, - } - } - - async fn rlock(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.rlock(args).await, - LockApi::Remote(r) => r.rlock(args).await, - } - } - - async fn runlock(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.runlock(args).await, - LockApi::Remote(r) => r.runlock(args).await, - } - } - - async fn refresh(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.refresh(args).await, - LockApi::Remote(r) => r.refresh(args).await, - } - } - - async fn force_unlock(&mut self, args: &LockArgs) -> Result { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.force_unlock(args).await, - LockApi::Remote(r) => r.force_unlock(args).await, - } - } - - async fn close(&self) { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.read().await.close().await, - LockApi::Remote(r) => r.close().await, - } - } - - async fn is_online(&self) -> bool { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.read().await.is_online().await, - LockApi::Remote(r) => r.is_online().await, - } - } - - async fn is_local(&self) -> bool { - match self { - LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.is_local().await, - LockApi::Remote(r) => r.is_local().await, - } - } -} - -pub fn new_lock_api(is_local: bool, url: Option) -> LockApi { - if is_local { - return LockApi::Local; - } - - LockApi::Remote(RemoteClient::new(url.unwrap())) +// ============================================================================ +// Convenience Functions +// ============================================================================ + +/// Create a new namespace lock +pub fn create_namespace_lock(namespace: String, _distributed: bool) -> NamespaceLock { + // The distributed behavior is now determined by the type of clients added to the NamespaceLock + // This function just creates an empty NamespaceLock + NamespaceLock::new(namespace) } diff --git a/crates/lock/src/local.rs b/crates/lock/src/local.rs new file mode 100644 index 000000000..d0b7239a0 --- /dev/null +++ b/crates/lock/src/local.rs @@ -0,0 +1,925 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; + +use crate::LockRequest; + +/// local lock entry +#[derive(Debug)] +pub struct LocalLockEntry { + /// current writer + pub writer: Option, + /// current readers with their lock counts + pub readers: HashMap, + /// lock expiration time + pub expires_at: Option, +} + +/// local lock map +#[derive(Debug)] +pub struct LocalLockMap { + /// LockId to lock object map + pub locks: Arc>>>>, + /// Shutdown flag for background tasks + shutdown: Arc, +} + +impl Default for LocalLockMap { + fn default() -> Self { + Self::new() + } +} + +impl LocalLockMap { + /// create new local lock map + pub fn new() -> Self { + let map = Self { + locks: Arc::new(RwLock::new(HashMap::new())), + shutdown: Arc::new(AtomicBool::new(false)), + }; + map.spawn_expiry_task(); + map + } + + /// spawn expiry task to clean up expired locks + fn spawn_expiry_task(&self) { + let locks = self.locks.clone(); + let shutdown = self.shutdown.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + loop { + interval.tick().await; + + if shutdown.load(Ordering::Relaxed) { + tracing::debug!("Expiry task shutting down"); + break; + } + + let now = Instant::now(); + let mut to_remove = Vec::new(); + + { + let locks_guard = locks.read().await; + for (key, entry) in locks_guard.iter() { + if let Ok(mut entry_guard) = entry.try_write() { + if let Some(exp) = entry_guard.expires_at { + if exp <= now { + entry_guard.writer = None; + entry_guard.readers.clear(); + entry_guard.expires_at = None; + + if entry_guard.writer.is_none() && entry_guard.readers.is_empty() { + to_remove.push(key.clone()); + } + } + } + } + } + } + + if !to_remove.is_empty() { + let mut locks_guard = locks.write().await; + for key in to_remove { + locks_guard.remove(&key); + } + } + } + }); + } + + /// write lock with TTL, support timeout, use LockRequest + pub async fn lock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result { + let start = Instant::now(); + let expires_at = Some(Instant::now() + request.ttl); + + loop { + // get or create lock entry + let entry = { + let mut locks_guard = self.locks.write().await; + locks_guard + .entry(request.lock_id.clone()) + .or_insert_with(|| { + Arc::new(RwLock::new(LocalLockEntry { + writer: None, + readers: HashMap::new(), + expires_at: None, + })) + }) + .clone() + }; + + // try to get write lock to modify state + if let Ok(mut entry_guard) = entry.try_write() { + // check expired state + let now = Instant::now(); + if let Some(exp) = entry_guard.expires_at { + if exp <= now { + entry_guard.writer = None; + entry_guard.readers.clear(); + entry_guard.expires_at = None; + } + } + + // check if can get write lock + if entry_guard.writer.is_none() && entry_guard.readers.is_empty() { + entry_guard.writer = Some(request.owner.clone()); + entry_guard.expires_at = expires_at; + tracing::debug!("Write lock acquired for resource '{}' by owner '{}'", request.resource, request.owner); + return Ok(true); + } + } + + if start.elapsed() >= request.acquire_timeout { + return Ok(false); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// read lock with TTL, support timeout, use LockRequest + pub async fn rlock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result { + let start = Instant::now(); + let expires_at = Some(Instant::now() + request.ttl); + + loop { + // get or create lock entry + let entry = { + let mut locks_guard = self.locks.write().await; + locks_guard + .entry(request.lock_id.clone()) + .or_insert_with(|| { + Arc::new(RwLock::new(LocalLockEntry { + writer: None, + readers: HashMap::new(), + expires_at: None, + })) + }) + .clone() + }; + + // try to get write lock to modify state + if let Ok(mut entry_guard) = entry.try_write() { + // check expired state + let now = Instant::now(); + if let Some(exp) = entry_guard.expires_at { + if exp <= now { + entry_guard.writer = None; + entry_guard.readers.clear(); + entry_guard.expires_at = None; + } + } + + // check if can get read lock + if entry_guard.writer.is_none() { + // increase read lock count + *entry_guard.readers.entry(request.owner.clone()).or_insert(0) += 1; + entry_guard.expires_at = expires_at; + tracing::debug!("Read lock acquired for resource '{}' by owner '{}'", request.resource, request.owner); + return Ok(true); + } + } + + if start.elapsed() >= request.acquire_timeout { + return Ok(false); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + + /// unlock by LockId and owner - need to specify owner to correctly unlock + pub async fn unlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> { + println!("Unlocking lock_id: {lock_id:?}, owner: {owner}"); + let mut need_remove = false; + + { + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(lock_id) { + println!("Found lock entry, attempting to acquire write lock..."); + match entry.try_write() { + Ok(mut entry_guard) => { + println!("Successfully acquired write lock for unlock"); + // try to release write lock + if entry_guard.writer.as_ref() == Some(&owner.to_string()) { + println!("Releasing write lock for owner: {owner}"); + entry_guard.writer = None; + } + // try to release read lock + else if let Some(count) = entry_guard.readers.get_mut(owner) { + println!("Releasing read lock for owner: {owner} (count: {count})"); + *count -= 1; + if *count == 0 { + entry_guard.readers.remove(owner); + println!("Removed owner {owner} from readers"); + } + } else { + println!("Owner {owner} not found in writers or readers"); + } + // check if need to remove + if entry_guard.readers.is_empty() && entry_guard.writer.is_none() { + println!("Lock entry is empty, marking for removal"); + entry_guard.expires_at = None; + need_remove = true; + } else { + println!( + "Lock entry still has content: writer={:?}, readers={:?}", + entry_guard.writer, entry_guard.readers + ); + } + } + Err(_) => { + println!("Failed to acquire write lock for unlock - this is the problem!"); + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "Failed to acquire write lock for unlock", + )); + } + } + } else { + println!("Lock entry not found for lock_id: {lock_id:?}"); + } + } + + // only here, entry's Ref is really dropped, can safely remove + if need_remove { + println!("Removing lock entry from map..."); + let mut locks_guard = self.locks.write().await; + let removed = locks_guard.remove(lock_id); + println!("Lock entry removed: {:?}", removed.is_some()); + } + println!("Unlock operation completed"); + Ok(()) + } + + /// unlock by LockId - smart release (compatible with old interface, but may be inaccurate) + pub async fn unlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> { + let mut need_remove = false; + + { + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(lock_id) { + if let Ok(mut entry_guard) = entry.try_write() { + // release write lock first + if entry_guard.writer.is_some() { + entry_guard.writer = None; + } + // if no write lock, release first read lock + else if let Some((owner, _)) = entry_guard.readers.iter().next() { + let owner = owner.clone(); + if let Some(count) = entry_guard.readers.get_mut(&owner) { + *count -= 1; + if *count == 0 { + entry_guard.readers.remove(&owner); + } + } + } + + // if completely idle, clean entry + if entry_guard.readers.is_empty() && entry_guard.writer.is_none() { + entry_guard.expires_at = None; + need_remove = true; + } + } + } + } + + if need_remove { + let mut locks_guard = self.locks.write().await; + locks_guard.remove(lock_id); + } + Ok(()) + } + + /// runlock by LockId and owner - need to specify owner to correctly unlock read lock + pub async fn runlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> { + let mut need_remove = false; + + { + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(lock_id) { + if let Ok(mut entry_guard) = entry.try_write() { + // release read lock + if let Some(count) = entry_guard.readers.get_mut(owner) { + *count -= 1; + if *count == 0 { + entry_guard.readers.remove(owner); + } + } + + // if completely idle, clean entry + if entry_guard.readers.is_empty() && entry_guard.writer.is_none() { + entry_guard.expires_at = None; + need_remove = true; + } + } + } + } + + if need_remove { + let mut locks_guard = self.locks.write().await; + locks_guard.remove(lock_id); + } + Ok(()) + } + + /// runlock by LockId - smart release read lock (compatible with old interface) + pub async fn runlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> { + let mut need_remove = false; + + { + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(lock_id) { + if let Ok(mut entry_guard) = entry.try_write() { + // release first read lock + if let Some((owner, _)) = entry_guard.readers.iter().next() { + let owner = owner.clone(); + if let Some(count) = entry_guard.readers.get_mut(&owner) { + *count -= 1; + if *count == 0 { + entry_guard.readers.remove(&owner); + } + } + } + + // if completely idle, clean entry + if entry_guard.readers.is_empty() && entry_guard.writer.is_none() { + entry_guard.expires_at = None; + need_remove = true; + } + } + } + } + + if need_remove { + let mut locks_guard = self.locks.write().await; + locks_guard.remove(lock_id); + } + Ok(()) + } + + /// check if resource is locked + pub async fn is_locked(&self, resource: &str) -> bool { + let lock_id = crate::types::LockId::new_deterministic(resource); + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(&lock_id) { + let entry_guard = entry.read().await; + entry_guard.writer.is_some() || !entry_guard.readers.is_empty() + } else { + false + } + } + + /// get lock info for a resource + pub async fn get_lock(&self, resource: &str) -> Option { + let lock_id = crate::types::LockId::new_deterministic(resource); + let locks_guard = self.locks.read().await; + if let Some(entry) = locks_guard.get(&lock_id) { + let entry_guard = entry.read().await; + + if let Some(owner) = &entry_guard.writer { + Some(crate::types::LockInfo { + id: lock_id, + resource: resource.to_string(), + lock_type: crate::types::LockType::Exclusive, + status: crate::types::LockStatus::Acquired, + owner: owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30), + last_refreshed: std::time::SystemTime::now(), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + wait_start_time: None, + }) + } else if !entry_guard.readers.is_empty() { + let owner = entry_guard.readers.keys().next().unwrap().clone(); + Some(crate::types::LockInfo { + id: lock_id, + resource: resource.to_string(), + lock_type: crate::types::LockType::Shared, + status: crate::types::LockStatus::Acquired, + owner, + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30), + last_refreshed: std::time::SystemTime::now(), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + wait_start_time: None, + }) + } else { + None + } + } else { + None + } + } + + /// get statistics + pub async fn get_stats(&self) -> crate::types::LockStats { + let mut stats = crate::types::LockStats::default(); + let locks_guard = self.locks.read().await; + + for (_, entry) in locks_guard.iter() { + let entry_guard = entry.read().await; + if entry_guard.writer.is_some() { + stats.exclusive_locks += 1; + } + stats.shared_locks += entry_guard.readers.len(); + } + + stats.total_locks = stats.exclusive_locks + stats.shared_locks; + stats + } + + /// shutdown background tasks + pub async fn shutdown(&self) { + self.shutdown.store(true, Ordering::Relaxed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::time::Duration; + use tokio::task; + use tokio::time::{sleep, timeout}; + + /// Test basic write lock operations + #[tokio::test] + async fn test_write_lock_basic() { + let lock_map = LocalLockMap::new(); + + // create a simple lock request + let request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("test_resource"), + resource: "test_resource".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "test_owner".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + // try to acquire lock + println!("Attempting to acquire lock..."); + let result = lock_map.lock_with_ttl_id(&request).await; + println!("Lock acquisition result: {result:?}"); + + match result { + Ok(success) => { + if success { + println!("Lock acquired successfully"); + // check lock state + let is_locked = lock_map.is_locked("test_resource").await; + println!("Is locked: {is_locked}"); + + // try to unlock + println!("Attempting to unlock..."); + let unlock_result = lock_map.unlock_by_id_and_owner(&request.lock_id, "test_owner").await; + println!("Unlock result: {unlock_result:?}"); + + // check lock state again + let is_locked_after = lock_map.is_locked("test_resource").await; + println!("Is locked after unlock: {is_locked_after}"); + + assert!(!is_locked_after, "Should be unlocked after release"); + } else { + println!("Lock acquisition failed (timeout)"); + } + } + Err(e) => { + println!("Lock acquisition error: {e:?}"); + panic!("Lock acquisition failed with error: {e:?}"); + } + } + } + + /// Test basic read lock operations + #[tokio::test] + async fn test_read_lock_basic() { + let lock_map = LocalLockMap::new(); + + // Test successful acquisition + let request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("bar"), + resource: "bar".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let ok = lock_map.rlock_with_ttl_id(&request).await.unwrap(); + assert!(ok, "Read lock should be successfully acquired"); + assert!(lock_map.is_locked("bar").await, "Lock state should be locked"); + + // Test lock info + let lock_info = lock_map.get_lock("bar").await; + assert!(lock_info.is_some(), "Lock info should exist"); + let info = lock_info.unwrap(); + assert_eq!(info.owner, "reader1"); + assert_eq!(info.lock_type, crate::types::LockType::Shared); + + // Test unlock with owner + lock_map.runlock_by_id_and_owner(&request.lock_id, "reader1").await.unwrap(); + assert!(!lock_map.is_locked("bar").await, "Should be unlocked after release"); + } + + /// Test write lock mutual exclusion + #[tokio::test] + async fn test_write_lock_mutex() { + let lock_map = Arc::new(LocalLockMap::new()); + + // Owner1 acquires write lock + let request1 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_mutex_test"), + resource: "res_mutex_test".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "owner1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let ok = lock_map.lock_with_ttl_id(&request1).await.unwrap(); + assert!(ok, "First write lock should succeed"); + + // Owner2 tries to acquire write lock on same resource - should fail due to timeout + let lock_map2 = lock_map.clone(); + let request2 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_mutex_test"), + resource: "res_mutex_test".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "owner2".to_string(), + acquire_timeout: Duration::from_millis(50), + ttl: Duration::from_millis(50), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let request2_clone = request2.clone(); + let result = timeout(Duration::from_millis(100), async move { + lock_map2.lock_with_ttl_id(&request2_clone).await.unwrap() + }) + .await; + + assert!(result.is_ok(), "Lock attempt should complete"); + assert!(!result.unwrap(), "Second write lock should fail due to conflict"); + + // Release first lock + lock_map.unlock_by_id_and_owner(&request1.lock_id, "owner1").await.unwrap(); + + // Now owner2 should be able to acquire the lock + let ok = lock_map.lock_with_ttl_id(&request2).await.unwrap(); + assert!(ok, "Write lock should succeed after first is released"); + lock_map.unlock_by_id_and_owner(&request2.lock_id, "owner2").await.unwrap(); + } + + /// Test read lock sharing + #[tokio::test] + async fn test_read_lock_sharing() { + let lock_map = LocalLockMap::new(); + + // Multiple readers should be able to acquire read locks + let request1 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_sharing_test"), + resource: "res_sharing_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let request2 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_sharing_test"), + resource: "res_sharing_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader2".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let request3 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_sharing_test"), + resource: "res_sharing_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader3".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let ok1 = lock_map.rlock_with_ttl_id(&request1).await.unwrap(); + let ok2 = lock_map.rlock_with_ttl_id(&request2).await.unwrap(); + let ok3 = lock_map.rlock_with_ttl_id(&request3).await.unwrap(); + assert!(ok1 && ok2 && ok3, "All read locks should succeed"); + assert!(lock_map.is_locked("res_sharing_test").await, "Resource should be locked"); + + // Release readers one by one + lock_map.runlock_by_id_and_owner(&request1.lock_id, "reader1").await.unwrap(); + assert!( + lock_map.is_locked("res_sharing_test").await, + "Should still be locked with remaining readers" + ); + + lock_map.runlock_by_id_and_owner(&request2.lock_id, "reader2").await.unwrap(); + assert!(lock_map.is_locked("res_sharing_test").await, "Should still be locked with one reader"); + + lock_map.runlock_by_id_and_owner(&request3.lock_id, "reader3").await.unwrap(); + assert!( + !lock_map.is_locked("res_sharing_test").await, + "Should be unlocked when all readers release" + ); + } + + /// Test read-write lock exclusion + #[tokio::test] + async fn test_read_write_exclusion() { + let lock_map = LocalLockMap::new(); + + // Reader acquires read lock + let read_request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_rw_test"), + resource: "res_rw_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let ok = lock_map.rlock_with_ttl_id(&read_request).await.unwrap(); + assert!(ok, "Read lock should succeed"); + + // Writer tries to acquire write lock - should fail + let write_request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_rw_test"), + resource: "res_rw_test".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "writer1".to_string(), + acquire_timeout: Duration::from_millis(50), + ttl: Duration::from_millis(50), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let result = timeout(Duration::from_millis(100), async { + lock_map.lock_with_ttl_id(&write_request).await.unwrap() + }) + .await; + + assert!(result.is_ok(), "Write lock attempt should complete"); + assert!(!result.unwrap(), "Write lock should fail when read lock is held"); + + // Release read lock + lock_map + .runlock_by_id_and_owner(&read_request.lock_id, "reader1") + .await + .unwrap(); + + // Now writer should be able to acquire the lock with longer TTL + let write_request_long_ttl = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_rw_test"), + resource: "res_rw_test".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "writer1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(200), // Longer TTL to prevent expiration during test + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + let ok = lock_map.lock_with_ttl_id(&write_request_long_ttl).await.unwrap(); + assert!(ok, "Write lock should succeed after read lock is released"); + + // Reader tries to acquire read lock while write lock is held - should fail + let read_request2 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res_rw_test"), + resource: "res_rw_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader2".to_string(), + acquire_timeout: Duration::from_millis(50), + ttl: Duration::from_millis(50), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let result = timeout(Duration::from_millis(100), async { + lock_map.rlock_with_ttl_id(&read_request2).await.unwrap() + }) + .await; + + assert!(result.is_ok(), "Read lock attempt should complete"); + assert!(!result.unwrap(), "Read lock should fail when write lock is held"); + + // Release write lock + lock_map + .unlock_by_id_and_owner(&write_request_long_ttl.lock_id, "writer1") + .await + .unwrap(); + } + + /// Test statistics + #[tokio::test] + async fn test_statistics() { + let lock_map = LocalLockMap::new(); + + // Initially no locks + let stats = lock_map.get_stats().await; + assert_eq!(stats.total_locks, 0, "Should have no locks initially"); + assert_eq!(stats.exclusive_locks, 0, "Should have no exclusive locks initially"); + assert_eq!(stats.shared_locks, 0, "Should have no shared locks initially"); + + // Add some locks + let write_request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res1_stats_test"), + resource: "res1_stats_test".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "owner1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let read_request1 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res2_stats_test"), + resource: "res2_stats_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let read_request2 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("res2_stats_test"), + resource: "res2_stats_test".to_string(), + lock_type: crate::types::LockType::Shared, + owner: "reader2".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(100), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + lock_map.lock_with_ttl_id(&write_request).await.unwrap(); + lock_map.rlock_with_ttl_id(&read_request1).await.unwrap(); + lock_map.rlock_with_ttl_id(&read_request2).await.unwrap(); + + let stats = lock_map.get_stats().await; + assert_eq!(stats.exclusive_locks, 1, "Should have 1 exclusive lock"); + assert_eq!(stats.shared_locks, 2, "Should have 2 shared locks"); + assert_eq!(stats.total_locks, 3, "Should have 3 total locks"); + + // Clean up + lock_map + .unlock_by_id_and_owner(&write_request.lock_id, "owner1") + .await + .unwrap(); + lock_map + .runlock_by_id_and_owner(&read_request1.lock_id, "reader1") + .await + .unwrap(); + lock_map + .runlock_by_id_and_owner(&read_request2.lock_id, "reader2") + .await + .unwrap(); + } + + /// Test concurrent access + #[tokio::test] + async fn test_concurrent_access() { + let lock_map = Arc::new(LocalLockMap::new()); + let num_tasks = 10; + let num_iterations = 100; + + let mut handles = Vec::new(); + + for i in 0..num_tasks { + let lock_map = lock_map.clone(); + let owner = format!("owner{i}"); + let handle = task::spawn(async move { + for j in 0..num_iterations { + let resource = format!("resource{}", j % 5); + let request = LockRequest { + lock_id: crate::types::LockId::new_deterministic(&resource), + resource: resource.clone(), + lock_type: if j % 2 == 0 { + crate::types::LockType::Exclusive + } else { + crate::types::LockType::Shared + }, + owner: owner.clone(), + acquire_timeout: Duration::from_millis(10), + ttl: Duration::from_millis(10), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + if request.lock_type == crate::types::LockType::Exclusive { + if lock_map.lock_with_ttl_id(&request).await.unwrap() { + sleep(Duration::from_micros(100)).await; + lock_map.unlock_by_id_and_owner(&request.lock_id, &owner).await.unwrap(); + } + } else if lock_map.rlock_with_ttl_id(&request).await.unwrap() { + sleep(Duration::from_micros(100)).await; + lock_map.runlock_by_id_and_owner(&request.lock_id, &owner).await.unwrap(); + } + } + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + + // Verify no locks remain + let stats = lock_map.get_stats().await; + assert_eq!(stats.total_locks, 0, "No locks should remain after concurrent access"); + } + + #[tokio::test] + async fn test_write_lock_timeout_and_reacquire() { + let lock_map = LocalLockMap::new(); + + // 1. acquire lock + let request = LockRequest { + lock_id: crate::types::LockId::new_deterministic("timeout_resource"), + resource: "timeout_resource".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "owner1".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(200), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + + let ok = lock_map.lock_with_ttl_id(&request).await.unwrap(); + assert!(ok, "First lock should succeed"); + + // 2. try to acquire lock again, should fail + let request2 = LockRequest { + lock_id: crate::types::LockId::new_deterministic("timeout_resource"), + resource: "timeout_resource".to_string(), + lock_type: crate::types::LockType::Exclusive, + owner: "owner2".to_string(), + acquire_timeout: Duration::from_millis(100), + ttl: Duration::from_millis(200), + metadata: crate::types::LockMetadata::default(), + priority: crate::types::LockPriority::Normal, + deadlock_detection: false, + }; + let ok2 = lock_map.lock_with_ttl_id(&request2).await.unwrap(); + assert!(!ok2, "Second lock should fail before timeout"); + + // 3. wait for TTL to expire + tokio::time::sleep(Duration::from_millis(300)).await; + + // 4. try to acquire lock again, should succeed + let ok3 = lock_map.lock_with_ttl_id(&request2).await.unwrap(); + assert!(ok3, "Lock should succeed after timeout"); + } +} diff --git a/crates/lock/src/local_locker.rs b/crates/lock/src/local_locker.rs deleted file mode 100644 index ef9676c5c..000000000 --- a/crates/lock/src/local_locker.rs +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use async_trait::async_trait; -use std::io::{Error, Result}; -use std::{ - collections::HashMap, - time::{Duration, Instant}, -}; - -use crate::{Locker, lock_args::LockArgs}; - -pub const MAX_DELETE_LIST: usize = 1000; - -#[derive(Clone, Debug)] -struct LockRequesterInfo { - name: String, - writer: bool, - uid: String, - time_stamp: Instant, - time_last_refresh: Instant, - source: String, - group: bool, - owner: String, - quorum: usize, - idx: usize, -} - -impl Default for LockRequesterInfo { - fn default() -> Self { - Self { - name: Default::default(), - writer: Default::default(), - uid: Default::default(), - time_stamp: Instant::now(), - time_last_refresh: Instant::now(), - source: Default::default(), - group: Default::default(), - owner: Default::default(), - quorum: Default::default(), - idx: Default::default(), - } - } -} - -fn is_write_lock(lri: &[LockRequesterInfo]) -> bool { - lri.len() == 1 && lri[0].writer -} - -#[derive(Debug, Default)] -pub struct LockStats { - total: usize, - writes: usize, - reads: usize, -} - -#[derive(Debug, Default)] -pub struct LocalLocker { - lock_map: HashMap>, - lock_uid: HashMap, -} - -impl LocalLocker { - pub fn new() -> Self { - LocalLocker::default() - } -} - -impl LocalLocker { - fn can_take_lock(&self, resource: &[String]) -> bool { - resource.iter().fold(true, |acc, x| !self.lock_map.contains_key(x) && acc) - } - - pub fn stats(&self) -> LockStats { - let mut st = LockStats { - total: self.lock_map.len(), - ..Default::default() - }; - - self.lock_map.iter().for_each(|(_, value)| { - if !value.is_empty() { - if value[0].writer { - st.writes += 1; - } else { - st.reads += 1; - } - } - }); - - st - } - - fn dump_lock_map(&mut self) -> HashMap> { - let mut lock_copy = HashMap::new(); - self.lock_map.iter().for_each(|(key, value)| { - lock_copy.insert(key.to_string(), value.to_vec()); - }); - - lock_copy - } - - fn expire_old_locks(&mut self, interval: Duration) { - self.lock_map.iter_mut().for_each(|(_, lris)| { - lris.retain(|lri| { - if Instant::now().duration_since(lri.time_last_refresh) > interval { - let mut key = lri.uid.to_string(); - format_uuid(&mut key, &lri.idx); - self.lock_uid.remove(&key); - return false; - } - - true - }); - }); - } -} - -#[async_trait] -impl Locker for LocalLocker { - async fn lock(&mut self, args: &LockArgs) -> Result { - if args.resources.len() > MAX_DELETE_LIST { - return Err(Error::other(format!( - "internal error: LocalLocker.lock called with more than {MAX_DELETE_LIST} resources" - ))); - } - - if !self.can_take_lock(&args.resources) { - return Ok(false); - } - - args.resources.iter().enumerate().for_each(|(idx, resource)| { - self.lock_map.insert( - resource.to_string(), - vec![LockRequesterInfo { - name: resource.to_string(), - writer: true, - source: args.source.to_string(), - owner: args.owner.to_string(), - uid: args.uid.to_string(), - group: args.resources.len() > 1, - quorum: args.quorum, - idx, - ..Default::default() - }], - ); - - let mut uuid = args.uid.to_string(); - format_uuid(&mut uuid, &idx); - self.lock_uid.insert(uuid, resource.to_string()); - }); - - Ok(true) - } - - async fn unlock(&mut self, args: &LockArgs) -> Result { - if args.resources.len() > MAX_DELETE_LIST { - return Err(Error::other(format!( - "internal error: LocalLocker.unlock called with more than {MAX_DELETE_LIST} resources" - ))); - } - - let mut reply = false; - let mut err_info = String::new(); - for resource in args.resources.iter() { - match self.lock_map.get_mut(resource) { - Some(lris) => { - if !is_write_lock(lris) { - if err_info.is_empty() { - err_info = format!("unlock attempted on a read locked entity: {resource}"); - } else { - err_info.push_str(&format!(", {resource}")); - } - } else { - lris.retain(|lri| { - if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) { - let mut key = args.uid.to_string(); - format_uuid(&mut key, &lri.idx); - self.lock_uid.remove(&key).unwrap(); - reply |= true; - return false; - } - - true - }); - } - if lris.is_empty() { - self.lock_map.remove(resource); - } - } - None => { - continue; - } - }; - } - - Ok(reply) - } - - async fn rlock(&mut self, args: &LockArgs) -> Result { - if args.resources.len() != 1 { - return Err(Error::other("internal error: localLocker.RLock called with more than one resource")); - } - - let resource = &args.resources[0]; - match self.lock_map.get_mut(resource) { - Some(lri) => { - if !is_write_lock(lri) { - lri.push(LockRequesterInfo { - name: resource.to_string(), - writer: false, - source: args.source.to_string(), - owner: args.owner.to_string(), - uid: args.uid.to_string(), - quorum: args.quorum, - ..Default::default() - }); - } else { - return Ok(false); - } - } - None => { - self.lock_map.insert( - resource.to_string(), - vec![LockRequesterInfo { - name: resource.to_string(), - writer: false, - source: args.source.to_string(), - owner: args.owner.to_string(), - uid: args.uid.to_string(), - quorum: args.quorum, - ..Default::default() - }], - ); - } - } - let mut uuid = args.uid.to_string(); - format_uuid(&mut uuid, &0); - self.lock_uid.insert(uuid, resource.to_string()); - - Ok(true) - } - - async fn runlock(&mut self, args: &LockArgs) -> Result { - if args.resources.len() != 1 { - return Err(Error::other("internal error: localLocker.RLock called with more than one resource")); - } - - let mut reply = false; - let resource = &args.resources[0]; - match self.lock_map.get_mut(resource) { - Some(lris) => { - if is_write_lock(lris) { - return Err(Error::other(format!("runlock attempted on a write locked entity: {resource}"))); - } else { - lris.retain(|lri| { - if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) { - let mut key = args.uid.to_string(); - format_uuid(&mut key, &lri.idx); - self.lock_uid.remove(&key).unwrap(); - reply |= true; - return false; - } - - true - }); - } - if lris.is_empty() { - self.lock_map.remove(resource); - } - } - None => { - return Ok(reply); - } - }; - - Ok(reply) - } - - async fn refresh(&mut self, args: &LockArgs) -> Result { - let mut idx = 0; - let mut key = args.uid.to_string(); - format_uuid(&mut key, &idx); - match self.lock_uid.get(&key) { - Some(resource) => { - let mut resource = resource; - loop { - match self.lock_map.get_mut(resource) { - Some(_lris) => {} - None => { - let mut key = args.uid.to_string(); - format_uuid(&mut key, &0); - self.lock_uid.remove(&key); - return Ok(idx > 0); - } - } - - idx += 1; - let mut key = args.uid.to_string(); - format_uuid(&mut key, &idx); - resource = match self.lock_uid.get(&key) { - Some(resource) => resource, - None => return Ok(true), - }; - } - } - None => Ok(false), - } - } - - // TODO: need add timeout mechanism - async fn force_unlock(&mut self, args: &LockArgs) -> Result { - if args.uid.is_empty() { - args.resources.iter().for_each(|resource| { - if let Some(lris) = self.lock_map.get(resource) { - lris.iter().for_each(|lri| { - let mut key = lri.uid.to_string(); - format_uuid(&mut key, &lri.idx); - self.lock_uid.remove(&key); - }); - if lris.is_empty() { - self.lock_map.remove(resource); - } - } - }); - - return Ok(true); - } - let mut idx = 0; - let mut need_remove_resource = Vec::new(); - let mut need_remove_map_id = Vec::new(); - let reply = loop { - let mut map_id = args.uid.to_string(); - format_uuid(&mut map_id, &idx); - match self.lock_uid.get(&map_id) { - Some(resource) => match self.lock_map.get_mut(resource) { - Some(lris) => { - { - lris.retain(|lri| { - if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) { - let mut key = args.uid.to_string(); - format_uuid(&mut key, &lri.idx); - need_remove_map_id.push(key); - return false; - } - - true - }); - } - idx += 1; - if lris.is_empty() { - need_remove_resource.push(resource.to_string()); - } - } - None => { - need_remove_map_id.push(map_id); - idx += 1; - continue; - } - }, - None => { - break idx > 0; - } - } - }; - need_remove_resource.into_iter().for_each(|resource| { - self.lock_map.remove(&resource); - }); - need_remove_map_id.into_iter().for_each(|map_id| { - self.lock_uid.remove(&map_id); - }); - - Ok(reply) - } - - async fn close(&self) {} - - async fn is_online(&self) -> bool { - true - } - - async fn is_local(&self) -> bool { - true - } -} - -fn format_uuid(s: &mut String, idx: &usize) { - s.push_str(&idx.to_string()); -} - -#[cfg(test)] -mod test { - use super::LocalLocker; - use crate::{Locker, lock_args::LockArgs}; - use std::io::Result; - use tokio; - - #[tokio::test] - async fn test_lock_unlock() -> Result<()> { - let mut local_locker = LocalLocker::new(); - let args = LockArgs { - uid: "1111".to_string(), - resources: vec!["dandan".to_string()], - owner: "dd".to_string(), - source: "".to_string(), - quorum: 3, - }; - local_locker.lock(&args).await?; - - println!("lock local_locker: {local_locker:?} \n"); - - local_locker.unlock(&args).await?; - println!("unlock local_locker: {local_locker:?}"); - - Ok(()) - } -} diff --git a/crates/lock/src/lock_args.rs b/crates/lock/src/lock_args.rs deleted file mode 100644 index c8ee597b8..000000000 --- a/crates/lock/src/lock_args.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use serde::{Deserialize, Serialize}; -use std::fmt::Display; - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct LockArgs { - pub uid: String, - pub resources: Vec, - pub owner: String, - pub source: String, - pub quorum: usize, -} - -impl Display for LockArgs { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "LockArgs[ uid: {}, resources: {:?}, owner: {}, source:{}, quorum: {} ]", - self.uid, self.resources, self.owner, self.source, self.quorum - ) - } -} diff --git a/crates/lock/src/lrwmutex.rs b/crates/lock/src/lrwmutex.rs deleted file mode 100644 index 469b8da28..000000000 --- a/crates/lock/src/lrwmutex.rs +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use rand::Rng; -use std::time::{Duration, Instant}; -use tokio::{sync::RwLock, time::sleep}; -use tracing::info; - -#[derive(Debug, Default)] -pub struct LRWMutex { - id: RwLock, - source: RwLock, - is_write: RwLock, - reference: RwLock, -} - -impl LRWMutex { - pub async fn lock(&self) -> bool { - let is_write = true; - let id = self.id.read().await.clone(); - let source = self.source.read().await.clone(); - let timeout = Duration::from_secs(10000); - self.look_loop( - &id, &source, &timeout, // big enough - is_write, - ) - .await - } - - pub async fn get_lock(&self, id: &str, source: &str, timeout: &Duration) -> bool { - let is_write = true; - self.look_loop(id, source, timeout, is_write).await - } - - pub async fn r_lock(&self) -> bool { - let is_write: bool = false; - let id = self.id.read().await.clone(); - let source = self.source.read().await.clone(); - let timeout = Duration::from_secs(10000); - self.look_loop( - &id, &source, &timeout, // big enough - is_write, - ) - .await - } - - pub async fn get_r_lock(&self, id: &str, source: &str, timeout: &Duration) -> bool { - let is_write = false; - self.look_loop(id, source, timeout, is_write).await - } - - async fn inner_lock(&self, id: &str, source: &str, is_write: bool) -> bool { - *self.id.write().await = id.to_string(); - *self.source.write().await = source.to_string(); - - let mut locked = false; - if is_write { - if *self.reference.read().await == 0 && !*self.is_write.read().await { - *self.reference.write().await = 1; - *self.is_write.write().await = true; - locked = true; - } - } else if !*self.is_write.read().await { - *self.reference.write().await += 1; - locked = true; - } - - locked - } - - async fn look_loop(&self, id: &str, source: &str, timeout: &Duration, is_write: bool) -> bool { - let start = Instant::now(); - loop { - if self.inner_lock(id, source, is_write).await { - return true; - } else { - if Instant::now().duration_since(start) > *timeout { - return false; - } - let sleep_time: u64; - { - let mut rng = rand::rng(); - sleep_time = rng.random_range(10..=50); - } - sleep(Duration::from_millis(sleep_time)).await; - } - } - } - - pub async fn un_lock(&self) { - let is_write = true; - if !self.unlock(is_write).await { - info!("Trying to un_lock() while no Lock() is active") - } - } - - pub async fn un_r_lock(&self) { - let is_write = false; - if !self.unlock(is_write).await { - info!("Trying to un_r_lock() while no Lock() is active") - } - } - - async fn unlock(&self, is_write: bool) -> bool { - let mut unlocked = false; - if is_write { - if *self.is_write.read().await && *self.reference.read().await == 1 { - *self.reference.write().await = 0; - *self.is_write.write().await = false; - unlocked = true; - } - } else if !*self.is_write.read().await && *self.reference.read().await > 0 { - *self.reference.write().await -= 1; - unlocked = true; - } - - unlocked - } - - pub async fn force_un_lock(&self) { - *self.reference.write().await = 0; - *self.is_write.write().await = false; - } -} - -#[cfg(test)] -mod test { - use std::{sync::Arc, time::Duration}; - - use std::io::Result; - use tokio::time::sleep; - - use crate::lrwmutex::LRWMutex; - - #[tokio::test] - async fn test_lock_unlock() -> Result<()> { - let l_rw_lock = LRWMutex::default(); - let id = "foo"; - let source = "dandan"; - let timeout = Duration::from_secs(5); - assert!(l_rw_lock.get_lock(id, source, &timeout).await); - l_rw_lock.un_lock().await; - - l_rw_lock.lock().await; - - assert!(!l_rw_lock.get_r_lock(id, source, &timeout).await); - l_rw_lock.un_lock().await; - assert!(l_rw_lock.get_r_lock(id, source, &timeout).await); - - Ok(()) - } - - #[tokio::test] - async fn multi_thread_test() -> Result<()> { - let l_rw_lock = Arc::new(LRWMutex::default()); - let id = "foo"; - let source = "dandan"; - - let one_fn = async { - let one = Arc::clone(&l_rw_lock); - let timeout = Duration::from_secs(1); - assert!(one.get_lock(id, source, &timeout).await); - sleep(Duration::from_secs(5)).await; - l_rw_lock.un_lock().await; - }; - - let two_fn = async { - let two = Arc::clone(&l_rw_lock); - let timeout = Duration::from_secs(2); - assert!(!two.get_r_lock(id, source, &timeout).await); - sleep(Duration::from_secs(5)).await; - assert!(two.get_r_lock(id, source, &timeout).await); - two.un_r_lock().await; - }; - - tokio::join!(one_fn, two_fn); - - Ok(()) - } -} diff --git a/crates/lock/src/namespace.rs b/crates/lock/src/namespace.rs new file mode 100644 index 000000000..d426338f6 --- /dev/null +++ b/crates/lock/src/namespace.rs @@ -0,0 +1,346 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use async_trait::async_trait; +use std::sync::Arc; +use std::time::Duration; + +use crate::{ + client::LockClient, + error::{LockError, Result}, + types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType}, +}; + +/// Namespace lock for managing locks by resource namespaces +#[derive(Debug)] +pub struct NamespaceLock { + /// Lock clients for this namespace + clients: Vec>, + /// Namespace identifier + namespace: String, + /// Quorum size for operations (1 for local, majority for distributed) + quorum: usize, +} + +impl NamespaceLock { + /// Create new namespace lock + pub fn new(namespace: String) -> Self { + Self { + clients: Vec::new(), + namespace, + quorum: 1, + } + } + + /// Create namespace lock with clients + pub fn with_clients(namespace: String, clients: Vec>) -> Self { + let quorum = if clients.len() > 1 { + // For multiple clients (distributed mode), require majority + (clients.len() / 2) + 1 + } else { + // For single client (local mode), only need 1 + 1 + }; + + Self { + clients, + namespace, + quorum, + } + } + + /// Create namespace lock with client (compatibility) + pub fn with_client(client: Arc) -> Self { + Self::with_clients("default".to_string(), vec![client]) + } + + /// Get namespace identifier + pub fn namespace(&self) -> &str { + &self.namespace + } + + /// Get resource key for this namespace + fn get_resource_key(&self, resource: &str) -> String { + format!("{}:{}", self.namespace, resource) + } + + /// Acquire lock using clients + pub async fn acquire_lock(&self, request: &LockRequest) -> Result { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For single client, use it directly + if self.clients.len() == 1 { + return self.clients[0].acquire_lock(request).await; + } + + // For multiple clients, try to acquire from all clients and require quorum + let futures: Vec<_> = self + .clients + .iter() + .map(|client| async move { client.acquire_lock(request).await }) + .collect(); + + let results = futures::future::join_all(futures).await; + let successful = results.into_iter().filter_map(|r| r.ok()).filter(|r| r.success).count(); + + if successful >= self.quorum { + Ok(LockResponse::success( + LockInfo { + id: LockId::new_deterministic(&request.resource), + resource: request.resource.clone(), + lock_type: request.lock_type, + status: LockStatus::Acquired, + owner: request.owner.clone(), + acquired_at: std::time::SystemTime::now(), + expires_at: std::time::SystemTime::now() + request.ttl, + last_refreshed: std::time::SystemTime::now(), + metadata: request.metadata.clone(), + priority: request.priority, + wait_start_time: None, + }, + Duration::ZERO, + )) + } else { + Ok(LockResponse::failure("Failed to acquire quorum".to_string(), Duration::ZERO)) + } + } + + /// Release lock using clients + pub async fn release_lock(&self, lock_id: &LockId) -> Result { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For single client, use it directly + if self.clients.len() == 1 { + return self.clients[0].release(lock_id).await; + } + + // For multiple clients, try to release from all clients + let futures: Vec<_> = self + .clients + .iter() + .map(|client| { + let id = lock_id.clone(); + async move { client.release(&id).await } + }) + .collect(); + + let results = futures::future::join_all(futures).await; + let successful = results.into_iter().filter_map(|r| r.ok()).filter(|&r| r).count(); + + // For release, if any succeed, consider it successful + Ok(successful > 0) + } + + /// Get health information + pub async fn get_health(&self) -> crate::types::HealthInfo { + let lock_stats = self.get_stats().await; + let mut health = crate::types::HealthInfo { + node_id: self.namespace.clone(), + lock_stats, + ..Default::default() + }; + + // Check client status + let mut connected_clients = 0; + for client in &self.clients { + if client.is_online().await { + connected_clients += 1; + } + } + + health.status = if connected_clients > 0 { + crate::types::HealthStatus::Healthy + } else { + crate::types::HealthStatus::Degraded + }; + health.connected_nodes = connected_clients; + health.total_nodes = self.clients.len(); + + health + } + + /// Get namespace statistics + pub async fn get_stats(&self) -> crate::types::LockStats { + let mut stats = crate::types::LockStats::default(); + + // Try to get stats from clients + for client in &self.clients { + if let Ok(client_stats) = client.get_stats().await { + stats.successful_acquires += client_stats.successful_acquires; + stats.failed_acquires += client_stats.failed_acquires; + } + } + + stats + } +} + +impl Default for NamespaceLock { + fn default() -> Self { + Self::new("default".to_string()) + } +} + +/// Namespace lock manager trait +#[async_trait] +pub trait NamespaceLockManager: Send + Sync { + /// Batch get write lock + async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result; + + /// Batch release write lock + async fn unlock_batch(&self, resources: &[String], owner: &str) -> Result<()>; + + /// Batch get read lock + async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result; + + /// Batch release read lock + async fn runlock_batch(&self, resources: &[String], owner: &str) -> Result<()>; +} + +#[async_trait] +impl NamespaceLockManager for NamespaceLock { + async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For each resource, create a lock request and try to acquire using clients + for resource in resources { + let namespaced_resource = self.get_resource_key(resource); + let request = LockRequest::new(&namespaced_resource, LockType::Exclusive, owner) + .with_acquire_timeout(timeout) + .with_ttl(ttl); + + let response = self.acquire_lock(&request).await?; + if !response.success { + return Ok(false); + } + } + Ok(true) + } + + async fn unlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For each resource, create a lock ID and try to release using clients + for resource in resources { + let namespaced_resource = self.get_resource_key(resource); + let lock_id = LockId::new_deterministic(&namespaced_resource); + let _ = self.release_lock(&lock_id).await?; + } + Ok(()) + } + + async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For each resource, create a shared lock request and try to acquire using clients + for resource in resources { + let namespaced_resource = self.get_resource_key(resource); + let request = LockRequest::new(&namespaced_resource, LockType::Shared, owner) + .with_acquire_timeout(timeout) + .with_ttl(ttl); + + let response = self.acquire_lock(&request).await?; + if !response.success { + return Ok(false); + } + } + Ok(true) + } + + async fn runlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> { + if self.clients.is_empty() { + return Err(LockError::internal("No lock clients available")); + } + + // For each resource, create a lock ID and try to release using clients + for resource in resources { + let namespaced_resource = self.get_resource_key(resource); + let lock_id = LockId::new_deterministic(&namespaced_resource); + let _ = self.release_lock(&lock_id).await?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use crate::LocalClient; + + use super::*; + + #[tokio::test] + async fn test_namespace_lock_local() { + let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new())); + let resources = vec!["test1".to_string(), "test2".to_string()]; + + // Test batch lock + let result = ns_lock + .lock_batch(&resources, "test_owner", Duration::from_millis(100), Duration::from_secs(10)) + .await; + assert!(result.is_ok()); + assert!(result.unwrap()); + + // Test batch unlock + let result = ns_lock.unlock_batch(&resources, "test_owner").await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_connection_health() { + let local_lock = NamespaceLock::new("test-namespace".to_string()); + let health = local_lock.get_health().await; + assert_eq!(health.status, crate::types::HealthStatus::Degraded); // No clients + } + + #[tokio::test] + async fn test_namespace_lock_creation() { + let ns_lock = NamespaceLock::new("test-namespace".to_string()); + assert_eq!(ns_lock.namespace(), "test-namespace"); + } + + #[tokio::test] + async fn test_namespace_lock_new_local() { + let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new())); + assert_eq!(ns_lock.namespace(), "default"); + assert_eq!(ns_lock.clients.len(), 1); + assert!(ns_lock.clients[0].is_local().await); + + // Test that it can perform lock operations + let resources = vec!["test-resource".to_string()]; + let result = ns_lock + .lock_batch(&resources, "test-owner", Duration::from_millis(100), Duration::from_secs(10)) + .await; + assert!(result.is_ok()); + assert!(result.unwrap()); + } + + #[tokio::test] + async fn test_namespace_lock_resource_key() { + let ns_lock = NamespaceLock::new("test-namespace".to_string()); + + // Test resource key generation + let resource_key = ns_lock.get_resource_key("test-resource"); + assert_eq!(resource_key, "test-namespace:test-resource"); + } +} diff --git a/crates/lock/src/namespace_lock.rs b/crates/lock/src/namespace_lock.rs deleted file mode 100644 index f4c06a21f..000000000 --- a/crates/lock/src/namespace_lock.rs +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use async_trait::async_trait; -use std::{collections::HashMap, path::Path, sync::Arc, time::Duration}; -use tokio::sync::RwLock; -use uuid::Uuid; - -use crate::{ - LockApi, - drwmutex::{DRWMutex, Options}, - lrwmutex::LRWMutex, -}; -use std::io::Result; - -pub type RWLockerImpl = Box; - -#[async_trait] -pub trait RWLocker { - async fn get_lock(&mut self, opts: &Options) -> Result; - async fn un_lock(&mut self) -> Result<()>; - async fn get_u_lock(&mut self, opts: &Options) -> Result; - async fn un_r_lock(&mut self) -> Result<()>; -} - -#[derive(Debug)] -struct NsLock { - reference: usize, - lock: LRWMutex, -} - -#[derive(Debug, Default)] -pub struct NsLockMap { - is_dist_erasure: bool, - lock_map: RwLock>, -} - -impl NsLockMap { - pub fn new(is_dist_erasure: bool) -> Self { - Self { - is_dist_erasure, - ..Default::default() - } - } - - async fn lock( - &mut self, - volume: &String, - path: &String, - lock_source: &str, - ops_id: &str, - read_lock: bool, - timeout: Duration, - ) -> bool { - let resource = Path::new(volume).join(path).to_str().unwrap().to_string(); - let mut w_lock_map = self.lock_map.write().await; - let nslk = w_lock_map.entry(resource.clone()).or_insert(NsLock { - reference: 0, - lock: LRWMutex::default(), - }); - nslk.reference += 1; - - let locked = if read_lock { - nslk.lock.get_r_lock(ops_id, lock_source, &timeout).await - } else { - nslk.lock.get_lock(ops_id, lock_source, &timeout).await - }; - - if !locked { - nslk.reference -= 1; - if nslk.reference == 0 { - w_lock_map.remove(&resource); - } - } - - locked - } - - async fn un_lock(&mut self, volume: &String, path: &String, read_lock: bool) { - let resource = Path::new(volume).join(path).to_str().unwrap().to_string(); - let mut w_lock_map = self.lock_map.write().await; - if let Some(nslk) = w_lock_map.get_mut(&resource) { - if read_lock { - nslk.lock.un_r_lock().await; - } else { - nslk.lock.un_lock().await; - } - - nslk.reference -= 1; - - if nslk.reference == 0 { - w_lock_map.remove(&resource); - } - } - } -} - -pub struct WrapperLocker(pub Arc>); - -impl Drop for WrapperLocker { - fn drop(&mut self) { - let inner = self.0.clone(); - tokio::spawn(async move { - let _ = inner.write().await.un_lock().await; - }); - } -} - -pub async fn new_nslock( - ns: Arc>, - owner: String, - volume: String, - paths: Vec, - lockers: Vec, -) -> WrapperLocker { - if ns.read().await.is_dist_erasure { - let names = paths - .iter() - .map(|path| Path::new(&volume).join(path).to_str().unwrap().to_string()) - .collect(); - return WrapperLocker(Arc::new(RwLock::new(Box::new(DistLockInstance::new(owner, names, lockers))))); - } - - WrapperLocker(Arc::new(RwLock::new(Box::new(LocalLockInstance::new(ns, volume, paths))))) -} - -struct DistLockInstance { - lock: Box, - ops_id: String, -} - -impl DistLockInstance { - fn new(owner: String, names: Vec, lockers: Vec) -> Self { - let ops_id = Uuid::new_v4().to_string(); - Self { - lock: Box::new(DRWMutex::new(owner, names, lockers)), - ops_id, - } - } -} - -#[async_trait] -impl RWLocker for DistLockInstance { - async fn get_lock(&mut self, opts: &Options) -> Result { - let source = "".to_string(); - - Ok(self.lock.get_lock(&self.ops_id, &source, opts).await) - } - - async fn un_lock(&mut self) -> Result<()> { - self.lock.un_lock().await; - Ok(()) - } - - async fn get_u_lock(&mut self, opts: &Options) -> Result { - let source = "".to_string(); - - Ok(self.lock.get_r_lock(&self.ops_id, &source, opts).await) - } - - async fn un_r_lock(&mut self) -> Result<()> { - self.lock.un_r_lock().await; - Ok(()) - } -} - -struct LocalLockInstance { - ns: Arc>, - volume: String, - paths: Vec, - ops_id: String, -} - -impl LocalLockInstance { - fn new(ns: Arc>, volume: String, paths: Vec) -> Self { - let ops_id = Uuid::new_v4().to_string(); - Self { - ns, - volume, - paths, - ops_id, - } - } -} - -#[async_trait] -impl RWLocker for LocalLockInstance { - async fn get_lock(&mut self, opts: &Options) -> Result { - let source = "".to_string(); - let read_lock = false; - let mut success = vec![false; self.paths.len()]; - for (idx, path) in self.paths.iter().enumerate() { - if !self - .ns - .write() - .await - .lock(&self.volume, path, &source, &self.ops_id, read_lock, opts.timeout) - .await - { - for (i, x) in success.iter().enumerate() { - if *x { - self.ns.write().await.un_lock(&self.volume, &self.paths[i], read_lock).await; - } - } - - return Ok(false); - } - - success[idx] = true; - } - Ok(true) - } - - async fn un_lock(&mut self) -> Result<()> { - let read_lock = false; - for path in self.paths.iter() { - self.ns.write().await.un_lock(&self.volume, path, read_lock).await; - } - - Ok(()) - } - - async fn get_u_lock(&mut self, opts: &Options) -> Result { - let source = "".to_string(); - let read_lock = true; - let mut success = Vec::with_capacity(self.paths.len()); - for (idx, path) in self.paths.iter().enumerate() { - if !self - .ns - .write() - .await - .lock(&self.volume, path, &source, &self.ops_id, read_lock, opts.timeout) - .await - { - for (i, x) in success.iter().enumerate() { - if *x { - self.ns.write().await.un_lock(&self.volume, &self.paths[i], read_lock).await; - } - } - - return Ok(false); - } - - success[idx] = true; - } - Ok(true) - } - - async fn un_r_lock(&mut self) -> Result<()> { - let read_lock = true; - for path in self.paths.iter() { - self.ns.write().await.un_lock(&self.volume, path, read_lock).await; - } - - Ok(()) - } -} - -#[cfg(test)] -mod test { - use std::{sync::Arc, time::Duration}; - - use std::io::Result; - use tokio::sync::RwLock; - - use crate::{ - drwmutex::Options, - namespace_lock::{NsLockMap, new_nslock}, - }; - - #[tokio::test] - async fn test_local_instance() -> Result<()> { - let ns_lock_map = Arc::new(RwLock::new(NsLockMap::default())); - let ns = new_nslock( - Arc::clone(&ns_lock_map), - "local".to_string(), - "test".to_string(), - vec!["foo".to_string()], - Vec::new(), - ) - .await; - - let result = - ns.0.write() - .await - .get_lock(&Options { - timeout: Duration::from_secs(5), - retry_interval: Duration::from_secs(1), - }) - .await?; - - assert!(result); - Ok(()) - } -} diff --git a/crates/lock/src/remote_client.rs b/crates/lock/src/remote_client.rs deleted file mode 100644 index 789bf4073..000000000 --- a/crates/lock/src/remote_client.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::{Locker, lock_args::LockArgs}; -use async_trait::async_trait; -use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; -use std::io::{Error, Result}; -use tonic::Request; -use tracing::info; - -#[derive(Debug, Clone)] -pub struct RemoteClient { - addr: String, -} - -impl RemoteClient { - pub fn new(url: url::Url) -> Self { - let addr = format!("{}://{}:{}", url.scheme(), url.host_str().unwrap(), url.port().unwrap()); - Self { addr } - } -} - -#[async_trait] -impl Locker for RemoteClient { - async fn lock(&mut self, args: &LockArgs) -> Result { - info!("remote lock"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.lock(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn unlock(&mut self, args: &LockArgs) -> Result { - info!("remote unlock"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.un_lock(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn rlock(&mut self, args: &LockArgs) -> Result { - info!("remote rlock"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.r_lock(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn runlock(&mut self, args: &LockArgs) -> Result { - info!("remote runlock"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.r_un_lock(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn refresh(&mut self, args: &LockArgs) -> Result { - info!("remote refresh"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.refresh(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn force_unlock(&mut self, args: &LockArgs) -> Result { - info!("remote force_unlock"); - let args = serde_json::to_string(args)?; - let mut client = node_service_time_out_client(&self.addr) - .await - .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; - let request = Request::new(GenerallyLockRequest { args }); - - let response = client.force_un_lock(request).await.map_err(Error::other)?.into_inner(); - - if let Some(error_info) = response.error_info { - return Err(Error::other(error_info)); - } - - Ok(response.success) - } - - async fn close(&self) {} - - async fn is_online(&self) -> bool { - true - } - - async fn is_local(&self) -> bool { - false - } -} diff --git a/crates/lock/src/types.rs b/crates/lock/src/types.rs new file mode 100644 index 000000000..50aaa27ec --- /dev/null +++ b/crates/lock/src/types.rs @@ -0,0 +1,702 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use uuid::Uuid; + +/// Lock type enumeration +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LockType { + /// Exclusive lock (write lock) + Exclusive, + /// Shared lock (read lock) + Shared, +} + +/// Lock status enumeration +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum LockStatus { + /// Acquired + Acquired, + /// Waiting + Waiting, + /// Released + Released, + /// Expired + Expired, + /// Force released + ForceReleased, +} + +/// Lock priority +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] +pub enum LockPriority { + Low = 1, + #[default] + Normal = 2, + High = 3, + Critical = 4, +} + +/// Lock information structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockInfo { + /// Unique identifier for the lock + pub id: LockId, + /// Resource path + pub resource: String, + /// Lock type + pub lock_type: LockType, + /// Lock status + pub status: LockStatus, + /// Lock owner + pub owner: String, + /// Acquisition time + pub acquired_at: SystemTime, + /// Expiration time + pub expires_at: SystemTime, + /// Last refresh time + pub last_refreshed: SystemTime, + /// Lock metadata + pub metadata: LockMetadata, + /// Lock priority + pub priority: LockPriority, + /// Wait start time + pub wait_start_time: Option, +} + +impl LockInfo { + /// Check if the lock has expired + pub fn has_expired(&self) -> bool { + self.expires_at <= SystemTime::now() + } + + /// Get remaining time until expiration + pub fn remaining_time(&self) -> Duration { + let now = SystemTime::now(); + if self.expires_at > now { + self.expires_at.duration_since(now).unwrap_or(Duration::ZERO) + } else { + Duration::ZERO + } + } + + /// Check if the lock is still valid + pub fn is_valid(&self) -> bool { + !self.has_expired() && self.status == LockStatus::Acquired + } +} + +/// Lock ID type +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct LockId { + pub resource: String, + pub uuid: String, +} + +impl LockId { + /// Generate new lock ID for a resource + pub fn new(resource: &str) -> Self { + Self { + resource: resource.to_string(), + uuid: Uuid::new_v4().to_string(), + } + } + + /// Generate deterministic lock ID for a resource (same resource = same ID) + pub fn new_deterministic(resource: &str) -> Self { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + resource.hash(&mut hasher); + let hash = hasher.finish(); + + Self { + resource: resource.to_string(), + uuid: format!("{hash:016x}"), + } + } + + /// Create lock ID from resource and uuid + pub fn from_parts(resource: impl Into, uuid: impl Into) -> Self { + Self { + resource: resource.into(), + uuid: uuid.into(), + } + } + + /// Create lock ID from string (for compatibility, expects "resource:uuid") + pub fn from_string(id: impl Into) -> Self { + let s = id.into(); + if let Some((resource, uuid)) = s.split_once(":") { + Self { + resource: resource.to_string(), + uuid: uuid.to_string(), + } + } else { + // fallback: treat as uuid only + Self { + resource: "unknown".to_string(), + uuid: s, + } + } + } + + /// Get string representation of lock ID ("resource:uuid") + pub fn as_str(&self) -> String { + format!("{}:{}", self.resource, self.uuid) + } +} + +impl Default for LockId { + fn default() -> Self { + Self::new("default") + } +} + +impl std::fmt::Display for LockId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.resource, self.uuid) + } +} + +/// Lock metadata structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockMetadata { + /// Client information + pub client_info: Option, + /// Operation ID + pub operation_id: Option, + /// Priority (lower number = higher priority) + pub priority: Option, + /// Custom tags + pub tags: std::collections::HashMap, + /// Creation time + pub created_at: SystemTime, +} + +impl Default for LockMetadata { + fn default() -> Self { + Self { + client_info: None, + operation_id: None, + priority: None, + tags: std::collections::HashMap::new(), + created_at: SystemTime::now(), + } + } +} + +impl LockMetadata { + /// Create new lock metadata + pub fn new() -> Self { + Self::default() + } + + /// Set client information + pub fn with_client_info(mut self, client_info: impl Into) -> Self { + self.client_info = Some(client_info.into()); + self + } + + /// Set operation ID + pub fn with_operation_id(mut self, operation_id: impl Into) -> Self { + self.operation_id = Some(operation_id.into()); + self + } + + /// Set priority + pub fn with_priority(mut self, priority: i32) -> Self { + self.priority = Some(priority); + self + } + + /// Add tag + pub fn with_tag(mut self, key: impl Into, value: impl Into) -> Self { + self.tags.insert(key.into(), value.into()); + self + } +} + +/// Lock request structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockRequest { + /// Lock ID + pub lock_id: LockId, + /// Resource path + pub resource: String, + /// Lock type + pub lock_type: LockType, + /// Lock owner + pub owner: String, + /// Acquire timeout duration (how long to wait for lock acquisition) + pub acquire_timeout: Duration, + /// Lock TTL (Time To Live - how long the lock remains valid after acquisition) + pub ttl: Duration, + /// Lock metadata + pub metadata: LockMetadata, + /// Lock priority + pub priority: LockPriority, + /// Deadlock detection + pub deadlock_detection: bool, +} + +impl LockRequest { + /// Create new lock request + pub fn new(resource: impl Into, lock_type: LockType, owner: impl Into) -> Self { + let resource_str = resource.into(); + Self { + lock_id: LockId::new_deterministic(&resource_str), + resource: resource_str, + lock_type, + owner: owner.into(), + acquire_timeout: Duration::from_secs(10), // Default 10 seconds to acquire + ttl: Duration::from_secs(30), // Default 30 seconds lock lifetime + metadata: LockMetadata::default(), + priority: LockPriority::default(), + deadlock_detection: false, + } + } + + /// Set acquire timeout (how long to wait for lock acquisition) + pub fn with_acquire_timeout(mut self, timeout: Duration) -> Self { + self.acquire_timeout = timeout; + self + } + + /// Set lock TTL (how long the lock remains valid after acquisition) + pub fn with_ttl(mut self, ttl: Duration) -> Self { + self.ttl = ttl; + self + } + + /// Set metadata + pub fn with_metadata(mut self, metadata: LockMetadata) -> Self { + self.metadata = metadata; + self + } + + /// Set priority + pub fn with_priority(mut self, priority: LockPriority) -> Self { + self.priority = priority; + self + } + + /// Set deadlock detection + pub fn with_deadlock_detection(mut self, enabled: bool) -> Self { + self.deadlock_detection = enabled; + self + } +} + +/// Lock response structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockResponse { + /// Whether lock acquisition was successful + pub success: bool, + /// Lock information (if successful) + pub lock_info: Option, + /// Error message (if failed) + pub error: Option, + /// Wait time + pub wait_time: Duration, + /// Position in wait queue + pub position_in_queue: Option, +} + +impl LockResponse { + /// Create success response + pub fn success(lock_info: LockInfo, wait_time: Duration) -> Self { + Self { + success: true, + lock_info: Some(lock_info), + error: None, + wait_time, + position_in_queue: None, + } + } + + /// Create failure response + pub fn failure(error: impl Into, wait_time: Duration) -> Self { + Self { + success: false, + lock_info: None, + error: Some(error.into()), + wait_time, + position_in_queue: None, + } + } + + /// Create waiting response + pub fn waiting(wait_time: Duration, position: usize) -> Self { + Self { + success: false, + lock_info: None, + error: None, + wait_time, + position_in_queue: Some(position), + } + } + + /// Check if response indicates success + pub fn is_success(&self) -> bool { + self.success + } + + /// Check if response indicates failure + pub fn is_failure(&self) -> bool { + !self.success && self.error.is_some() + } + + /// Check if response indicates waiting + pub fn is_waiting(&self) -> bool { + !self.success && self.position_in_queue.is_some() + } + + /// Get lock info + pub fn lock_info(&self) -> Option<&LockInfo> { + self.lock_info.as_ref() + } +} + +/// Lock statistics structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockStats { + /// Total number of locks + pub total_locks: usize, + /// Number of exclusive locks + pub exclusive_locks: usize, + /// Number of shared locks + pub shared_locks: usize, + /// Number of waiting locks + pub waiting_locks: usize, + /// Number of deadlock detections + pub deadlock_detections: usize, + /// Number of priority upgrades + pub priority_upgrades: usize, + /// Last update time + pub last_updated: SystemTime, + /// Total releases + pub total_releases: usize, + /// Total hold time + pub total_hold_time: Duration, + /// Average hold time + pub average_hold_time: Duration, + /// Total wait queues + pub total_wait_queues: usize, + /// Queue entries + pub queue_entries: usize, + /// Average wait time + pub avg_wait_time: Duration, + /// Successful acquires + pub successful_acquires: usize, + /// Failed acquires + pub failed_acquires: usize, +} + +impl Default for LockStats { + fn default() -> Self { + Self { + total_locks: 0, + exclusive_locks: 0, + shared_locks: 0, + waiting_locks: 0, + deadlock_detections: 0, + priority_upgrades: 0, + last_updated: SystemTime::now(), + total_releases: 0, + total_hold_time: Duration::ZERO, + average_hold_time: Duration::ZERO, + total_wait_queues: 0, + queue_entries: 0, + avg_wait_time: Duration::ZERO, + successful_acquires: 0, + failed_acquires: 0, + } + } +} + +/// Node information structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeInfo { + /// Node ID + pub id: String, + /// Node address + pub address: String, + /// Node status + pub status: NodeStatus, + /// Last heartbeat time + pub last_heartbeat: SystemTime, + /// Node weight + pub weight: f64, +} + +/// Node status enumeration +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum NodeStatus { + /// Online + #[default] + Online, + /// Offline + Offline, + /// Degraded + Degraded, +} + +/// Cluster information structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClusterInfo { + /// Cluster ID + pub cluster_id: String, + /// List of nodes + pub nodes: Vec, + /// Quorum size + pub quorum: usize, + /// Cluster status + pub status: ClusterStatus, + /// Last update time + pub last_updated: SystemTime, +} + +/// Cluster status enumeration +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub enum ClusterStatus { + /// Healthy + #[default] + Healthy, + /// Degraded + Degraded, + /// Unhealthy + Unhealthy, +} + +/// Health check status +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum HealthStatus { + /// Healthy + Healthy, + /// Degraded + Degraded, + /// Unhealthy + Unhealthy, +} + +/// Health check information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthInfo { + /// Overall status + pub status: HealthStatus, + /// Node ID + pub node_id: String, + /// Last heartbeat time + pub last_heartbeat: SystemTime, + /// Connected nodes count + pub connected_nodes: usize, + /// Total nodes count + pub total_nodes: usize, + /// Lock statistics + pub lock_stats: LockStats, + /// Error message (if any) + pub error_message: Option, +} + +impl Default for HealthInfo { + fn default() -> Self { + Self { + status: HealthStatus::Healthy, + node_id: "unknown".to_string(), + last_heartbeat: SystemTime::now(), + connected_nodes: 1, + total_nodes: 1, + lock_stats: LockStats::default(), + error_message: None, + } + } +} + +/// Timestamp type alias +pub type Timestamp = u64; + +/// Get current timestamp +pub fn current_timestamp() -> Timestamp { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() +} + +/// Convert timestamp to system time +pub fn timestamp_to_system_time(timestamp: Timestamp) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(timestamp) +} + +/// Convert system time to timestamp +pub fn system_time_to_timestamp(time: SystemTime) -> Timestamp { + time.duration_since(UNIX_EPOCH).unwrap().as_secs() +} + +/// Deadlock detection result structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeadlockDetectionResult { + /// Whether deadlock exists + pub has_deadlock: bool, + /// Deadlock cycle + pub deadlock_cycle: Vec, + /// Suggested resolution + pub suggested_resolution: Option, + /// Affected resources + pub affected_resources: Vec, + /// Affected owners + pub affected_owners: Vec, +} + +/// Wait graph node structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WaitGraphNode { + /// Owner + pub owner: String, + /// Resources being waited for + pub waiting_for: Vec, + /// Resources currently held + pub held_resources: Vec, + /// Priority + pub priority: LockPriority, + /// Wait start time + pub wait_start_time: SystemTime, +} + +/// Wait queue item structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WaitQueueItem { + /// Owner + pub owner: String, + /// Lock type + pub lock_type: LockType, + /// Priority + pub priority: LockPriority, + /// Wait start time + pub wait_start_time: SystemTime, + /// Request time + pub request_time: SystemTime, +} + +impl WaitQueueItem { + /// Create new wait queue item + pub fn new(owner: &str, lock_type: LockType, priority: LockPriority) -> Self { + let now = SystemTime::now(); + Self { + owner: owner.to_string(), + lock_type, + priority, + wait_start_time: now, + request_time: now, + } + } + + /// Get wait duration + pub fn wait_duration(&self) -> Duration { + self.wait_start_time.elapsed().unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lock_id() { + let id1 = LockId::new("test-resource"); + let id2 = LockId::new("test-resource"); + assert_ne!(id1, id2); + + let id3 = LockId::from_string("test-resource:test-uuid"); + assert_eq!(id3.as_str(), "test-resource:test-uuid"); + } + + #[test] + fn test_lock_metadata() { + let metadata = LockMetadata::new() + .with_client_info("test-client") + .with_operation_id("test-op") + .with_priority(1) + .with_tag("key", "value"); + + assert_eq!(metadata.client_info, Some("test-client".to_string())); + assert_eq!(metadata.operation_id, Some("test-op".to_string())); + assert_eq!(metadata.priority, Some(1)); + assert_eq!(metadata.tags.get("key"), Some(&"value".to_string())); + } + + #[test] + fn test_lock_request() { + let request = LockRequest::new("test-resource", LockType::Exclusive, "test-owner") + .with_acquire_timeout(Duration::from_secs(60)) + .with_priority(LockPriority::High) + .with_deadlock_detection(true); + + assert_eq!(request.resource, "test-resource"); + assert_eq!(request.lock_type, LockType::Exclusive); + assert_eq!(request.owner, "test-owner"); + assert_eq!(request.acquire_timeout, Duration::from_secs(60)); + assert_eq!(request.priority, LockPriority::High); + assert!(request.deadlock_detection); + } + + #[test] + fn test_lock_response() { + let lock_info = LockInfo { + id: LockId::new("test-resource"), + resource: "test".to_string(), + lock_type: LockType::Exclusive, + status: LockStatus::Acquired, + owner: "test".to_string(), + acquired_at: SystemTime::now(), + expires_at: SystemTime::now() + Duration::from_secs(30), + last_refreshed: SystemTime::now(), + metadata: LockMetadata::default(), + priority: LockPriority::Normal, + wait_start_time: None, + }; + + let success = LockResponse::success(lock_info.clone(), Duration::ZERO); + assert!(success.is_success()); + + let failure = LockResponse::failure("error", Duration::ZERO); + assert!(failure.is_failure()); + + let waiting = LockResponse::waiting(Duration::ZERO, 1); + assert!(waiting.is_waiting()); + } + + #[test] + fn test_timestamp_conversion() { + let now = SystemTime::now(); + let timestamp = system_time_to_timestamp(now); + let converted = timestamp_to_system_time(timestamp); + + // Allow for small time differences + let diff = now.duration_since(converted).unwrap(); + assert!(diff < Duration::from_secs(1)); + } + + #[test] + fn test_serialization() { + let request = LockRequest::new("test", LockType::Exclusive, "owner"); + let serialized = serde_json::to_string(&request).unwrap(); + let deserialized: LockRequest = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(request.resource, deserialized.resource); + assert_eq!(request.lock_type, deserialized.lock_type); + assert_eq!(request.owner, deserialized.owner); + } +}