gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-04 04:17:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #290 from guojidan/feat/complete-lock-implementation

Browse Source 
refactor: reimplement lock
This commit is contained in:
guojidan
2025-07-24 12:11:19 +08:00
committed by GitHub
parent 1b48934f47 b48a5fdc94
commit 18b0134ddf
27 changed files with 4087 additions and 2629 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+88 -36
View File
@@ -504,9 +504,9 @@ dependencies = [
[[package]]
name = "async-io"
version = "2.4.1"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1237c0ae75a0f3765f58910ff9cdd0a12eeb39ab2f4c7de23262f337f0aacbb3"
checksum = "19634d6336019ef220f09fd31168ce5c184b295cbf80345437cc36094ef223ca"
dependencies = [
"async-lock",
"cfg-if",
@@ -517,8 +517,7 @@ dependencies = [
"polling",
"rustix 1.0.8",
"slab",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -534,9 +533,9 @@ dependencies = [
[[package]]
name = "async-process"
version = "2.3.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde3f4e40e6021d7acffc90095cbd6dc54cb593903d1de5832f435eb274b85dc"
checksum = "65daa13722ad51e6ab1a1b9c01299142bc75135b337923cfa10e79bbbd669f00"
dependencies = [
"async-channel",
"async-io",
@@ -548,7 +547,6 @@ dependencies = [
"event-listener",
"futures-lite",
"rustix 1.0.8",
"tracing",
]
[[package]]
@@ -564,9 +562,9 @@ dependencies = [
[[package]]
name = "async-signal"
version = "0.2.11"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7605a4e50d4b06df3898d5a70bf5fde51ed9059b0434b73105193bc27acce0d"
checksum = "f567af260ef69e1d52c2b560ce0ea230763e6fbb9214a85d768760a920e3e3c1"
dependencies = [
"async-io",
"async-lock",
@@ -577,7 +575,7 @@ dependencies = [
"rustix 1.0.8",
"signal-hook-registry",
"slab",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -688,9 +686,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
version = "1.13.2"
version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08b5d4e069cbc868041a64bd68dc8cb39a0d79585cd6c5a24caa8c2d622121be"
checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -760,7 +758,7 @@ dependencies = [
"http 0.2.12",
"http 1.3.1",
"http-body 0.4.6",
"lru",
"lru 0.12.5",
"percent-encoding",
"regex-lite",
"sha2 0.10.9",
@@ -908,9 +906,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
version = "1.8.4"
version = "1.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3aaec682eb189e43c8a19c3dab2fe54590ad5f2cc2d26ab27608a20f2acf81c"
checksum = "660f70d9d8af6876b4c9aa8dcb0dbaf0f89b04ee9a4455bea1b4ba03b15f26f6"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -932,9 +930,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.8.3"
version = "1.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9852b9226cb60b78ce9369022c0df678af1cac231c882d5da97a0c4e03be6e67"
checksum = "38280ac228bc479f347fcfccf4bf4d22d68f3bb4629685cb591cabd856567bbc"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
@@ -1479,9 +1477,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.2.29"
version = "1.2.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362"
checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
dependencies = [
"jobserver",
"libc",
@@ -3479,6 +3477,7 @@ dependencies = [
"rustfs-protos",
"serde",
"serde_json",
"serial_test",
"tokio",
"tonic",
"url",
@@ -4957,9 +4956,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "io-uring"
version = "0.7.8"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013"
checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4"
dependencies = [
"bitflags 2.9.1",
"cfg-if",
@@ -5361,13 +5360,13 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
[[package]]
name = "libredox"
version = "0.1.4"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0"
dependencies = [
"bitflags 2.9.1",
"libc",
"redox_syscall 0.5.13",
"redox_syscall 0.5.15",
]
[[package]]
@@ -5484,6 +5483,15 @@ dependencies = [
"hashbrown 0.15.4",
]
[[package]]
name = "lru"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86ea4e65087ff52f3862caff188d489f1fab49a0cb09e01b2e3f1a617b10aaed"
dependencies = [
"hashbrown 0.15.4",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
@@ -6567,7 +6575,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.5.13",
"redox_syscall 0.5.15",
"smallvec",
"windows-targets 0.52.6",
]
@@ -6946,17 +6954,16 @@ dependencies = [
[[package]]
name = "polling"
version = "3.8.0"
version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b53a684391ad002dd6a596ceb6c74fd004fdce75f4be2e3f615068abbea5fd50"
checksum = "8ee9b2fa7a4517d2c91ff5bc6c297a427a96749d15f98fcdbb22c05571a4d4b7"
dependencies = [
"cfg-if",
"concurrent-queue",
"hermit-abi",
"pin-project-lite",
"rustix 1.0.8",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -7474,9 +7481,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.5.13"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec"
dependencies = [
"bitflags 2.9.1",
]
@@ -8133,10 +8140,16 @@ name = "rustfs-lock"
version = "0.0.5"
dependencies = [
"async-trait",
"bytes",
"futures",
"lazy_static",
"lru 0.16.0",
"once_cell",
"rand 0.9.2",
"rustfs-protos",
"serde",
"serde_json",
"thiserror 2.0.12",
"tokio",
"tonic",
"tracing",
@@ -8653,6 +8666,15 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "scc"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4"
dependencies = [
"sdd",
]
[[package]]
name = "schannel"
version = "0.1.27"
@@ -8684,6 +8706,12 @@ dependencies = [
"untrusted",
]
[[package]]
name = "sdd"
version = "3.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"
[[package]]
name = "sec1"
version = "0.3.0"
@@ -8941,6 +8969,31 @@ dependencies = [
"unsafe-libyaml",
]
[[package]]
name = "serial_test"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9"
dependencies = [
"futures",
"log",
"once_cell",
"parking_lot",
"scc",
"serial_test_derive",
]
[[package]]
name = "serial_test_derive"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.104",
]
[[package]]
name = "server_fn"
version = "0.6.15"
@@ -9417,14 +9470,13 @@ dependencies = [
[[package]]
name = "strum_macros"
version = "0.27.1"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.104",
]
@@ -10905,9 +10957,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502"
checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2"
dependencies = [
"rustls-pki-types",
]
Cargo.toml
+1
View File
@@ -148,6 +148,7 @@ keyring = { version = "3.6.2", features = [
] }
lazy_static = "1.5.0"
libsystemd = { version = "0.7.2" }
lru = "0.16"
local-ip-address = "0.6.5"
lz4 = "1.28.1"
matchit = "0.8.4"
crates/e2e_test/Cargo.toml
+1
View File
@@ -38,3 +38,4 @@ url.workspace = true
rustfs-madmin.workspace = true
rustfs-filemeta.workspace = true
bytes.workspace = true
serial_test = "3.2.0"
crates/e2e_test/src/reliant/lock.rs
+424 -35
View File
@@ -13,28 +13,42 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_lock::{
drwmutex::Options,
lock_args::LockArgs,
namespace_lock::{NsLockMap, new_nslock},
new_lock_api,
};
use rustfs_ecstore::{disk::endpoint::Endpoint, lock_utils::create_unique_clients};
use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType};
use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager};
use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
use std::{error::Error, sync::Arc, time::Duration};
use tokio::sync::RwLock;
use serial_test::serial;
use std::{error::Error, time::Duration};
use tokio::time::sleep;
use tonic::Request;
use url::Url;
const CLUSTER_ADDR: &str = "http://localhost:9000";
fn get_cluster_endpoints() -> Vec<Endpoint> {
vec![Endpoint {
url: Url::parse(CLUSTER_ADDR).unwrap(),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
}]
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
let args = LockArgs {
uid: "1111".to_string(),
resources: vec!["dandan".to_string()],
let args = LockRequest {
lock_id: LockId::new_deterministic("dandan"),
resource: "dandan".to_string(),
lock_type: LockType::Exclusive,
owner: "dd".to_string(),
source: "".to_string(),
quorum: 3,
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args = serde_json::to_string(&args)?;
@@ -59,30 +73,405 @@ async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
let url = url::Url::parse("http://127.0.0.1:9000/data")?;
let locker = new_lock_api(false, Some(url));
let ns_mutex = Arc::new(RwLock::new(NsLockMap::new(true)));
let ns = new_nslock(
Arc::clone(&ns_mutex),
"local".to_string(),
"dandan".to_string(),
vec!["foo".to_string()],
vec![locker],
)
.await;
assert!(
ns.0.write()
.await
.get_lock(&Options {
timeout: Duration::from_secs(5),
retry_interval: Duration::from_secs(1),
})
.await
.unwrap()
);
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
let resources = vec!["foo".to_string()];
let result = ns_lock
.lock_batch(&resources, "dandan", Duration::from_secs(5), Duration::from_secs(10))
.await;
match &result {
Ok(success) => println!("Lock result: {success}"),
Err(e) => println!("Lock error: {e}"),
}
let result = result?;
assert!(result, "Lock should succeed, but got: {result}");
ns_lock.unlock_batch(&resources, "dandan").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_concurrent_lock_attempts() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
let resource = vec!["concurrent_resource".to_string()];
// First lock should succeed
println!("Attempting first lock...");
let result1 = ns_lock
.lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
println!("First lock result: {result1}");
assert!(result1, "First lock should succeed");
// Second lock should fail (resource already locked)
println!("Attempting second lock...");
let result2 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(1), Duration::from_secs(10))
.await?;
println!("Second lock result: {result2}");
assert!(!result2, "Second lock should fail");
// Unlock by first owner
println!("Unlocking first lock...");
ns_lock.unlock_batch(&resource, "owner1").await?;
println!("First lock unlocked");
// Now second owner should be able to lock
println!("Attempting third lock...");
let result3 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
println!("Third lock result: {result3}");
assert!(result3, "Lock should succeed after unlock");
// Clean up
println!("Cleaning up...");
ns_lock.unlock_batch(&resource, "owner2").await?;
println!("Test completed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_read_write_lock_compatibility() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_rw".to_string(), clients);
let resource = vec!["rw_resource".to_string()];
// First read lock should succeed
let result1 = ns_lock
.rlock_batch(&resource, "reader1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result1, "First read lock should succeed");
// Second read lock should also succeed (read locks are compatible)
let result2 = ns_lock
.rlock_batch(&resource, "reader2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result2, "Second read lock should succeed");
// Write lock should fail (read locks are held)
let result3 = ns_lock
.lock_batch(&resource, "writer1", Duration::from_secs(1), Duration::from_secs(10))
.await?;
assert!(!result3, "Write lock should fail when read locks are held");
// Release read locks
ns_lock.runlock_batch(&resource, "reader1").await?;
ns_lock.runlock_batch(&resource, "reader2").await?;
// Now write lock should succeed
let result4 = ns_lock
.lock_batch(&resource, "writer1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result4, "Write lock should succeed after read locks released");
// Clean up
ns_lock.unlock_batch(&resource, "writer1").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_timeout() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_timeout".to_string(), clients);
let resource = vec!["timeout_resource".to_string()];
// First lock with short timeout
let result1 = ns_lock
.lock_batch(&resource, "owner1", Duration::from_secs(2), Duration::from_secs(1))
.await?;
assert!(result1, "First lock should succeed");
// Wait for lock to expire
sleep(Duration::from_secs(5)).await;
// Second lock should succeed after timeout
let result2 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(1))
.await?;
assert!(result2, "Lock should succeed after timeout");
// Clean up
ns_lock.unlock_batch(&resource, "owner2").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_batch_lock_operations() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_batch".to_string(), clients);
let resources = vec![
"batch_resource1".to_string(),
"batch_resource2".to_string(),
"batch_resource3".to_string(),
];
// Lock all resources
let result = ns_lock
.lock_batch(&resources, "batch_owner", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result, "Batch lock should succeed");
// Try to lock one of the resources with different owner - should fail
let single_resource = vec!["batch_resource2".to_string()];
let result2 = ns_lock
.lock_batch(&single_resource, "other_owner", Duration::from_secs(1), Duration::from_secs(10))
.await?;
assert!(!result2, "Lock should fail for already locked resource");
// Unlock all resources
ns_lock.unlock_batch(&resources, "batch_owner").await?;
// Now should be able to lock single resource
let result3 = ns_lock
.lock_batch(&single_resource, "other_owner", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result3, "Lock should succeed after batch unlock");
// Clean up
ns_lock.unlock_batch(&single_resource, "other_owner").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_multiple_namespaces() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock1 = NamespaceLock::with_clients("namespace1".to_string(), clients.clone());
let ns_lock2 = NamespaceLock::with_clients("namespace2".to_string(), clients);
let resource = vec!["shared_resource".to_string()];
// Lock same resource in different namespaces - both should succeed
let result1 = ns_lock1
.lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result1, "Lock in namespace1 should succeed");
let result2 = ns_lock2
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result2, "Lock in namespace2 should succeed");
// Clean up
ns_lock1.unlock_batch(&resource, "owner1").await?;
ns_lock2.unlock_batch(&resource, "owner2").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_rpc_read_lock() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("read_resource"),
resource: "read_resource".to_string(),
lock_type: LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
// First read lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.r_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get read lock: {error_info}");
}
// Second read lock with different owner should also succeed
let args2 = LockRequest {
lock_id: LockId::new_deterministic("read_resource"),
resource: "read_resource".to_string(),
lock_type: LockType::Shared,
owner: "reader2".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args2_str = serde_json::to_string(&args2)?;
let request2 = Request::new(GenerallyLockRequest { args: args2_str });
let response2 = client.r_lock(request2).await?.into_inner();
if let Some(error_info) = response2.error_info {
panic!("can not get second read lock: {error_info}");
}
// Unlock both
let request = Request::new(GenerallyLockRequest { args: args_str });
let response = client.r_un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not unlock read lock: {error_info}");
}
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_refresh() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("refresh_resource"),
resource: "refresh_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "refresh_owner".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
// Acquire lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get lock: {error_info}");
}
// Refresh lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.refresh(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not refresh lock: {error_info}");
}
assert!(response.success, "Lock refresh should succeed");
// Unlock
let request = Request::new(GenerallyLockRequest { args: args_str });
let response = client.un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not unlock: {error_info}");
}
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_force_unlock() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("force_resource"),
resource: "force_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "force_owner".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
// Acquire lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get lock: {error_info}");
}
// Force unlock (even by different owner)
let force_args = LockRequest {
lock_id: LockId::new_deterministic("force_resource"),
resource: "force_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "admin".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let force_args_str = serde_json::to_string(&force_args)?;
let request = Request::new(GenerallyLockRequest { args: force_args_str });
let response = client.force_un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not force unlock: {error_info}");
}
assert!(response.success, "Force unlock should succeed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_global_lock_map_sharing() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock1 = NamespaceLock::with_clients("global_test".to_string(), clients.clone());
let ns_lock2 = NamespaceLock::with_clients("global_test".to_string(), clients);
let resource = vec!["global_test_resource".to_string()];
// First instance acquires lock
println!("First lock map attempting to acquire lock...");
let result1 = ns_lock1
.lock_batch(&resource, "owner1", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
.await?;
println!("First lock result: {result1}");
assert!(result1, "First lock should succeed");
// Second instance should fail to acquire the same lock
println!("Second lock map attempting to acquire lock...");
let result2 = ns_lock2
.lock_batch(&resource, "owner2", std::time::Duration::from_secs(1), std::time::Duration::from_secs(10))
.await?;
println!("Second lock result: {result2}");
assert!(!result2, "Second lock should fail because resource is already locked");
// Release lock from first instance
println!("First lock map releasing lock...");
ns_lock1.unlock_batch(&resource, "owner1").await?;
// Now second instance should be able to acquire lock
println!("Second lock map attempting to acquire lock again...");
let result3 = ns_lock2
.lock_batch(&resource, "owner2", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
.await?;
println!("Third lock result: {result3}");
assert!(result3, "Lock should succeed after first lock is released");
// Clean up
ns_lock2.unlock_batch(&resource, "owner2").await?;
ns.0.write().await.un_lock().await.unwrap();
Ok(())
}
crates/ecstore/src/client/object_handlers_common.rs
+1 -1
View File
@@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore;
use crate::store_api::{ObjectOptions, ObjectToDelete};
use rustfs_lock::local_locker::MAX_DELETE_LIST;
use rustfs_lock::MAX_DELETE_LIST;
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
let mut remaining = to_del;
crates/ecstore/src/error.rs
+6
View File
@@ -183,6 +183,9 @@ pub enum StorageError {
#[error("Io error: {0}")]
Io(std::io::Error),
#[error("Lock error: {0}")]
Lock(#[from] rustfs_lock::LockError),
}
impl StorageError {
@@ -409,6 +412,7 @@ impl Clone for StorageError {
StorageError::FirstDiskWait => StorageError::FirstDiskWait,
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
StorageError::NoHealRequired => StorageError::NoHealRequired,
StorageError::Lock(e) => StorageError::Lock(e.clone()),
}
}
}
@@ -471,6 +475,7 @@ impl StorageError {
StorageError::ConfigNotFound => 0x35,
StorageError::TooManyOpenFiles => 0x36,
StorageError::NoHealRequired => 0x37,
StorageError::Lock(_) => 0x38,
}
}
@@ -535,6 +540,7 @@ impl StorageError {
0x35 => Some(StorageError::ConfigNotFound),
0x36 => Some(StorageError::TooManyOpenFiles),
0x37 => Some(StorageError::NoHealRequired),
0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
_ => None,
}
}
crates/ecstore/src/lib.rs
+1
View File
@@ -30,6 +30,7 @@ pub mod erasure_coding;
pub mod error;
pub mod global;
pub mod heal;
pub mod lock_utils;
pub mod metrics_realtime;
pub mod notification_sys;
pub mod pools;
crates/ecstore/src/lock_utils.rs
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::endpoint::Endpoint;
use crate::error::Result;
use rustfs_lock::client::{LockClient, local::LocalClient, remote::RemoteClient};
use std::collections::HashMap;
use std::sync::Arc;
/// Create unique lock clients from endpoints
/// This function creates one client per unique host:port combination
/// to avoid duplicate connections to the same server
pub async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>> {
let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
// Collect unique endpoints based on host:port
for endpoint in endpoints {
if endpoint.is_local {
// For local endpoints, use "local" as the key
unique_endpoints.insert("local".to_string(), endpoint);
} else {
// For remote endpoints, use host:port as the key
let host_port = format!(
"{}:{}",
endpoint.url.host_str().unwrap_or("localhost"),
endpoint.url.port().unwrap_or(9000)
);
unique_endpoints.insert(host_port, endpoint);
}
}
let mut clients = Vec::new();
// Create clients for unique endpoints
for (_key, endpoint) in unique_endpoints {
if endpoint.is_local {
// For local endpoints, create a local lock client
let local_client = LocalClient::new();
clients.push(Arc::new(local_client) as Arc<dyn LockClient>);
} else {
// For remote endpoints, create a remote lock client
let remote_client = RemoteClient::new(endpoint.url.to_string());
clients.push(Arc::new(remote_client) as Arc<dyn LockClient>);
}
}
Ok(clients)
}
#[cfg(test)]
mod tests {
use super::*;
use url::Url;
#[tokio::test]
async fn test_create_unique_clients_local() {
let endpoints = vec![
Endpoint {
url: Url::parse("http://localhost:9000").unwrap(),
is_local: true,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
},
Endpoint {
url: Url::parse("http://localhost:9000").unwrap(),
is_local: true,
pool_idx: 0,
set_idx: 0,
disk_idx: 1,
},
];
let clients = create_unique_clients(&endpoints).await.unwrap();
// Should only create one client for local endpoints
assert_eq!(clients.len(), 1);
assert!(clients[0].is_local().await);
}
#[tokio::test]
async fn test_create_unique_clients_mixed() {
let endpoints = vec![
Endpoint {
url: Url::parse("http://localhost:9000").unwrap(),
is_local: true,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
},
Endpoint {
url: Url::parse("http://remote1:9000").unwrap(),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 1,
},
Endpoint {
url: Url::parse("http://remote1:9000").unwrap(),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 2,
},
Endpoint {
url: Url::parse("http://remote2:9000").unwrap(),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 3,
},
];
let clients = create_unique_clients(&endpoints).await.unwrap();
// Should create 3 clients: 1 local + 2 unique remote
assert_eq!(clients.len(), 3);
// Check that we have one local client
let local_count = clients.iter().filter(|c| futures::executor::block_on(c.is_local())).count();
assert_eq!(local_count, 1);
// Check that we have two remote clients
let remote_count = clients.iter().filter(|c| !futures::executor::block_on(c.is_local())).count();
assert_eq!(remote_count, 2);
}
}
crates/ecstore/src/rpc/mod.rs
+1 -1
View File
@@ -22,4 +22,4 @@ pub use http_auth::{build_auth_headers, verify_rpc_signature};
pub use peer_rest_client::PeerRestClient;
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, RemotePeerS3Client, S3PeerSys};
pub use remote_disk::RemoteDisk;
pub use tonic_service::make_server;
pub use tonic_service::{NodeService, make_server};
crates/ecstore/src/rpc/tonic_service.rs
+116 -77
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{collections::HashMap, io::Cursor, pin::Pin};
use std::{collections::HashMap, io::Cursor, pin::Pin, sync::Arc};
// use common::error::Error as EcsError;
use crate::{
@@ -34,9 +34,9 @@ use crate::{
};
use futures::{Stream, StreamExt};
use futures_util::future::join_all;
use rustfs_lock::{GLOBAL_LOCAL_SERVER, Locker, lock_args::LockArgs};
use rustfs_common::globals::GLOBAL_Local_Node_Name;
use rustfs_lock::{LockClient, LockRequest};
use bytes::Bytes;
use rmp_serde::{Deserializer, Serializer};
@@ -81,11 +81,16 @@ type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, tonic::Status>> + S
#[derive(Debug)]
pub struct NodeService {
local_peer: LocalPeerS3Client,
lock_manager: Arc<rustfs_lock::LocalClient>,
}
pub fn make_server() -> NodeService {
let local_peer = LocalPeerS3Client::new(None, None);
NodeService { local_peer }
let lock_manager = Arc::new(rustfs_lock::LocalClient::new());
NodeService {
local_peer,
lock_manager,
}
}
impl NodeService {
@@ -1526,122 +1531,156 @@ impl Node for NodeService {
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.lock(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
// Parse the request to extract resource and owner
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not lock, args: {args}, err: {err}")),
})),
},
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
match self.lock_manager.acquire_exclusive(&args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result.success,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
error_info: Some(format!(
"can not lock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.unlock(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not unlock, args: {args}, err: {err}")),
})),
},
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
error_info: Some(format!(
"can not unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn r_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.rlock(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not rlock, args: {args}, err: {err}")),
})),
},
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
match self.lock_manager.acquire_shared(&args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result.success,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
error_info: Some(format!(
"can not rlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn r_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.runlock(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not runlock, args: {args}, err: {err}")),
})),
},
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
error_info: Some(format!(
"can not runlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn force_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.force_unlock(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not force_unlock, args: {args}, err: {err}")),
})),
},
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
error_info: Some(format!(
"can not force_unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn refresh(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
match &serde_json::from_str::<LockArgs>(&request.args) {
Ok(args) => match GLOBAL_LOCAL_SERVER.write().await.refresh(args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
let _args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not refresh, args: {args}, err: {err}")),
})),
},
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
})),
}
error_info: Some(format!("can not decode args, err: {err}")),
}));
}
};
Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
}))
}
async fn local_storage_info(
@@ -3629,15 +3668,15 @@ mod tests {
// Note: signal_service test is skipped because it contains todo!() and would panic
#[test]
fn test_node_service_debug() {
#[tokio::test]
async fn test_node_service_debug() {
let service = create_test_node_service();
let debug_str = format!("{service:?}");
assert!(debug_str.contains("NodeService"));
}
#[test]
fn test_node_service_creation() {
#[tokio::test]
async fn test_node_service_creation() {
let service1 = make_server();
let service2 = make_server();
crates/ecstore/src/set_disk.rs
+27 -37
View File
@@ -83,7 +83,7 @@ use rustfs_filemeta::{
headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
merge_file_meta_versions,
};
use rustfs_lock::{LockApi, namespace_lock::NsLockMap};
use rustfs_lock::NamespaceLockManager;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader};
use rustfs_utils::{
@@ -121,11 +121,10 @@ use uuid::Uuid;
pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
#[derive(Debug, Clone)]
#[derive(Clone, Debug)]
pub struct SetDisks {
pub lockers: Vec<LockApi>,
pub namespace_lock: Arc<rustfs_lock::NamespaceLock>,
pub locker_owner: String,
pub ns_mutex: Arc<RwLock<NsLockMap>>,
pub disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
pub set_endpoints: Vec<Endpoint>,
pub set_drive_count: usize,
@@ -138,9 +137,8 @@ pub struct SetDisks {
impl SetDisks {
#[allow(clippy::too_many_arguments)]
pub async fn new(
lockers: Vec<LockApi>,
namespace_lock: Arc<rustfs_lock::NamespaceLock>,
locker_owner: String,
ns_mutex: Arc<RwLock<NsLockMap>>,
disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
set_drive_count: usize,
default_parity_count: usize,
@@ -150,9 +148,8 @@ impl SetDisks {
format: FormatV3,
) -> Arc<Self> {
Arc::new(SetDisks {
lockers,
namespace_lock,
locker_owner,
ns_mutex,
disks,
set_drive_count,
default_parity_count,
@@ -4066,33 +4063,22 @@ impl ObjectIO for SetDisks {
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
let disks = self.disks.read().await;
// let mut _ns = None;
// if !opts.no_lock {
// let paths = vec![object.to_string()];
// let ns_lock = new_nslock(
// Arc::clone(&self.ns_mutex),
// self.locker_owner.clone(),
// bucket.to_string(),
// paths,
// self.lockers.clone(),
// )
// .await;
// if !ns_lock
// .0
// .write()
// .await
// .get_lock(&Options {
// timeout: Duration::from_secs(5),
// retry_interval: Duration::from_secs(1),
// })
// .await
// .map_err(|err| Error::other(err.to_string()))?
// {
// return Err(Error::other("can not get lock. please retry".to_string()));
// }
if !opts.no_lock {
let paths = vec![object.to_string()];
let lock_acquired = self
.namespace_lock
.lock_batch(
&paths,
&self.locker_owner,
std::time::Duration::from_secs(5),
std::time::Duration::from_secs(10),
)
.await?;
// _ns = Some(ns_lock);
// }
if !lock_acquired {
return Err(Error::other("can not get lock. please retry".to_string()));
}
}
let mut user_defined = opts.user_defined.clone();
@@ -4298,9 +4284,13 @@ impl ObjectIO for SetDisks {
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
// if let Some(mut locker) = ns {
// locker.un_lock().await.map_err(|err| Error::other(err.to_string()))?;
// }
// Release lock if it was acquired
if !opts.no_lock {
let paths = vec![object.to_string()];
if let Err(err) = self.namespace_lock.unlock_batch(&paths, &self.locker_owner).await {
error!("Failed to unlock object {}: {}", object, err);
}
}
for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk {
crates/ecstore/src/sets.rs
+14 -16
View File
@@ -43,7 +43,7 @@ use futures::future::join_all;
use http::HeaderMap;
use rustfs_common::globals::GLOBAL_Local_Node_Name;
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LockApi, namespace_lock::NsLockMap, new_lock_api};
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use tokio::sync::RwLock;
@@ -55,12 +55,13 @@ use tokio::time::Duration;
use tracing::warn;
use tracing::{error, info};
use crate::lock_utils::create_unique_clients;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
// pub sets: Vec<Objects>,
// pub disk_set: Vec<Vec<Option<DiskStore>>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub lockers: Vec<Vec<LockApi>>,
pub disk_set: Vec<Arc<SetDisks>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub pool_idx: usize,
pub endpoints: PoolEndpoints,
@@ -93,27 +94,25 @@ impl Sets {
let set_count = fm.erasure.sets.len();
let set_drive_count = fm.erasure.sets[0].len();
let mut unique: Vec<Vec<String>> = vec![vec![]; set_count];
let mut lockers: Vec<Vec<LockApi>> = vec![vec![]; set_count];
endpoints.endpoints.as_ref().iter().enumerate().for_each(|(idx, endpoint)| {
let mut unique: Vec<Vec<String>> = (0..set_count).map(|_| vec![]).collect();
for (idx, endpoint) in endpoints.endpoints.as_ref().iter().enumerate() {
let set_idx = idx / set_drive_count;
if endpoint.is_local && !unique[set_idx].contains(&"local".to_string()) {
unique[set_idx].push("local".to_string());
lockers[set_idx].push(new_lock_api(true, None));
}
if !endpoint.is_local {
let host_port = format!("{}:{}", endpoint.url.host_str().unwrap(), endpoint.url.port().unwrap());
if !unique[set_idx].contains(&host_port) {
unique[set_idx].push(host_port);
lockers[set_idx].push(new_lock_api(false, Some(endpoint.url.clone())));
}
}
});
}
let mut disk_set = Vec::with_capacity(set_count);
for (i, locker) in lockers.iter().enumerate().take(set_count) {
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
let mut set_endpoints = Vec::with_capacity(set_drive_count);
for j in 0..set_drive_count {
@@ -121,7 +120,6 @@ impl Sets {
let mut disk = disks[idx].clone();
let endpoint = endpoints.endpoints.as_ref()[idx].clone();
// let endpoint = endpoints.endpoints.as_ref().get(idx).cloned();
set_endpoints.push(endpoint);
if disk.is_none() {
@@ -165,12 +163,13 @@ impl Sets {
}
}
// warn!("sets new set_drive {:?}", &set_drive);
let lock_clients = create_unique_clients(&set_endpoints).await?;
let namespace_lock = rustfs_lock::NamespaceLock::with_clients(format!("set-{i}"), lock_clients);
let set_disks = SetDisks::new(
locker.clone(),
Arc::new(namespace_lock),
GLOBAL_Local_Node_Name.read().await.to_string(),
Arc::new(RwLock::new(NsLockMap::new(is_dist_erasure().await))),
Arc::new(RwLock::new(set_drive)),
set_drive_count,
parity_count,
@@ -190,7 +189,6 @@ impl Sets {
id: fm.id,
// sets: todo!(),
disk_set,
lockers,
pool_idx,
endpoints: endpoints.clone(),
format: fm.clone(),
@@ -543,7 +541,7 @@ impl StorageAPI for Sets {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)> {
// 默认返回值
// Default return value
let mut del_objects = vec![DeletedObject::default(); objects.len()];
let mut del_errs = Vec::with_capacity(objects.len());
@@ -602,7 +600,7 @@ impl StorageAPI for Sets {
// del_errs.extend(errs);
// }
// TODO: 并发
// TODO: Implement concurrency
for (k, v) in set_obj_map {
let disks = self.get_disks(k);
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
crates/lock/Cargo.toml
+7 -1
View File
@@ -30,6 +30,9 @@ workspace = true
[dependencies]
async-trait.workspace = true
bytes.workspace = true
futures.workspace = true
lazy_static.workspace = true
rustfs-protos.workspace = true
rand.workspace = true
serde.workspace = true
@@ -38,4 +41,7 @@ tokio.workspace = true
tonic.workspace = true
tracing.workspace = true
url.workspace = true
uuid.workspace = true
uuid.workspace = true
thiserror.workspace = true
once_cell.workspace = true
lru.workspace = true
crates/lock/src/client/local.rs
+366
View File
@@ -0,0 +1,366 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use crate::{
client::LockClient,
error::Result,
local::LocalLockMap,
types::{LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockType},
};
/// Local lock client
///
/// Uses global singleton LocalLockMap to ensure all clients access the same lock instance
#[derive(Debug, Clone)]
pub struct LocalClient;
impl LocalClient {
/// Create new local client
pub fn new() -> Self {
Self
}
/// Get global lock map instance
pub fn get_lock_map(&self) -> Arc<LocalLockMap> {
crate::get_global_lock_map()
}
}
impl Default for LocalClient {
fn default() -> Self {
Self::new()
}
}
#[async_trait::async_trait]
impl LockClient for LocalClient {
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
let lock_map = self.get_lock_map();
let success = lock_map
.lock_with_ttl_id(request)
.await
.map_err(|e| crate::error::LockError::internal(format!("Lock acquisition failed: {e}")))?;
if success {
let lock_info = LockInfo {
id: crate::types::LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO))
}
}
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
let lock_map = self.get_lock_map();
let success = lock_map
.rlock_with_ttl_id(request)
.await
.map_err(|e| crate::error::LockError::internal(format!("Shared lock acquisition failed: {e}")))?;
if success {
let lock_info = LockInfo {
id: crate::types::LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: LockType::Shared,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
Ok(LockResponse::failure("Lock acquisition failed".to_string(), std::time::Duration::ZERO))
}
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
let lock_map = self.get_lock_map();
// Try to release the lock directly by ID
match lock_map.unlock_by_id(lock_id).await {
Ok(()) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// Try as read lock if exclusive unlock failed
match lock_map.runlock_by_id(lock_id).await {
Ok(()) => Ok(true),
Err(_) => Err(crate::error::LockError::internal("Lock ID not found".to_string())),
}
}
Err(e) => Err(crate::error::LockError::internal(format!("Release lock failed: {e}"))),
}
}
async fn refresh(&self, _lock_id: &LockId) -> Result<bool> {
// For local locks, refresh is not needed as they don't expire automatically
Ok(true)
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
self.release(lock_id).await
}
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
let lock_map = self.get_lock_map();
// Check if the lock exists in our locks map
let locks_guard = lock_map.locks.read().await;
if let Some(entry) = locks_guard.get(lock_id) {
let entry_guard = entry.read().await;
// Determine lock type and owner based on the entry
if let Some(owner) = &entry_guard.writer {
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: crate::types::LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
} else if !entry_guard.readers.is_empty() {
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: crate::types::LockType::Shared,
status: crate::types::LockStatus::Acquired,
owner: entry_guard.readers.iter().next().map(|(k, _)| k.clone()).unwrap_or_default(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
} else {
Ok(None)
}
} else {
Ok(None)
}
}
async fn get_stats(&self) -> Result<LockStats> {
Ok(LockStats::default())
}
async fn close(&self) -> Result<()> {
Ok(())
}
async fn is_online(&self) -> bool {
true
}
async fn is_local(&self) -> bool {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::LockType;
#[tokio::test]
async fn test_local_client_acquire_exclusive() {
let client = LocalClient::new();
let resource_name = format!("test-resource-exclusive-{}", uuid::Uuid::new_v4());
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(&request).await.unwrap();
assert!(response.is_success());
// Clean up
if let Some(lock_info) = response.lock_info() {
let _ = client.release(&lock_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_acquire_shared() {
let client = LocalClient::new();
let resource_name = format!("test-resource-shared-{}", uuid::Uuid::new_v4());
let request = LockRequest::new(&resource_name, LockType::Shared, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_shared(&request).await.unwrap();
assert!(response.is_success());
// Clean up
if let Some(lock_info) = response.lock_info() {
let _ = client.release(&lock_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_release() {
let client = LocalClient::new();
let resource_name = format!("test-resource-release-{}", uuid::Uuid::new_v4());
// First acquire a lock
let request = LockRequest::new(&resource_name, LockType::Exclusive, "test-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let response = client.acquire_exclusive(&request).await.unwrap();
assert!(response.is_success());
// Get the lock ID from the response
if let Some(lock_info) = response.lock_info() {
let result = client.release(&lock_info.id).await.unwrap();
assert!(result);
} else {
panic!("No lock info in response");
}
}
#[tokio::test]
async fn test_local_client_is_local() {
let client = LocalClient::new();
assert!(client.is_local().await);
}
#[tokio::test]
async fn test_local_client_read_write_lock_exclusion() {
let client = LocalClient::new();
let resource_name = format!("test-resource-exclusion-{}", uuid::Uuid::new_v4());
// First, acquire an exclusive lock
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
assert!(exclusive_response.is_success());
// Try to acquire a shared lock on the same resource - should fail
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
assert!(!shared_response.is_success(), "Shared lock should fail when exclusive lock exists");
// Clean up exclusive lock
if let Some(exclusive_info) = exclusive_response.lock_info() {
let _ = client.release(&exclusive_info.id).await;
}
// Now shared lock should succeed
let shared_request2 = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_millis(10));
let shared_response2 = client.acquire_shared(&shared_request2).await.unwrap();
assert!(
shared_response2.is_success(),
"Shared lock should succeed after exclusive lock is released"
);
// Clean up
if let Some(shared_info) = shared_response2.lock_info() {
let _ = client.release(&shared_info.id).await;
}
}
#[tokio::test]
async fn test_local_client_read_write_lock_distinction() {
let client = LocalClient::new();
let resource_name = format!("test-resource-rw-{}", uuid::Uuid::new_v4());
// Test exclusive lock
let exclusive_request = LockRequest::new(&resource_name, LockType::Exclusive, "exclusive-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let exclusive_response = client.acquire_exclusive(&exclusive_request).await.unwrap();
assert!(exclusive_response.is_success());
if let Some(exclusive_info) = exclusive_response.lock_info() {
assert_eq!(exclusive_info.lock_type, LockType::Exclusive);
// Check status should return correct lock type
let status = client.check_status(&exclusive_info.id).await.unwrap();
assert!(status.is_some());
assert_eq!(status.unwrap().lock_type, LockType::Exclusive);
// Release exclusive lock
let result = client.release(&exclusive_info.id).await.unwrap();
assert!(result);
}
// Test shared lock
let shared_request = LockRequest::new(&resource_name, LockType::Shared, "shared-owner")
.with_acquire_timeout(std::time::Duration::from_secs(30));
let shared_response = client.acquire_shared(&shared_request).await.unwrap();
assert!(shared_response.is_success());
if let Some(shared_info) = shared_response.lock_info() {
assert_eq!(shared_info.lock_type, LockType::Shared);
// Check status should return correct lock type
let status = client.check_status(&shared_info.id).await.unwrap();
assert!(status.is_some());
assert_eq!(status.unwrap().lock_type, LockType::Shared);
// Release shared lock
let result = client.release(&shared_info.id).await.unwrap();
assert!(result);
}
}
#[tokio::test]
async fn test_multiple_local_clients_exclusive_mutex() {
let client1 = LocalClient::new();
let client2 = LocalClient::new();
let resource_name = format!("test-multi-client-mutex-{}", uuid::Uuid::new_v4());
// client1 acquire exclusive lock
let req1 = LockRequest::new(&resource_name, LockType::Exclusive, "owner1")
.with_acquire_timeout(std::time::Duration::from_millis(50));
let resp1 = client1.acquire_exclusive(&req1).await.unwrap();
assert!(resp1.is_success(), "client1 should acquire exclusive lock");
// client2 try to acquire exclusive lock, should fail
let req2 = LockRequest::new(&resource_name, LockType::Exclusive, "owner2")
.with_acquire_timeout(std::time::Duration::from_millis(50));
let resp2 = client2.acquire_exclusive(&req2).await.unwrap();
assert!(!resp2.is_success(), "client2 should not acquire exclusive lock while client1 holds it");
// client1 release lock
if let Some(lock_info) = resp1.lock_info() {
let _ = client1.release(&lock_info.id).await;
}
// client2 try again, should succeed
let resp3 = client2.acquire_exclusive(&req2).await.unwrap();
assert!(resp3.is_success(), "client2 should acquire exclusive lock after client1 releases it");
// clean up
if let Some(lock_info) = resp3.lock_info() {
let _ = client2.release(&lock_info.id).await;
}
}
}
crates/lock/src/client/mod.rs
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod local;
pub mod remote;
use async_trait::async_trait;
use std::sync::Arc;
use crate::{
error::Result,
types::{LockId, LockInfo, LockRequest, LockResponse, LockStats},
};
/// Lock client trait
#[async_trait]
pub trait LockClient: Send + Sync + std::fmt::Debug {
/// Acquire exclusive lock
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse>;
/// Acquire shared lock
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse>;
/// Acquire lock (generic method)
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
match request.lock_type {
crate::types::LockType::Exclusive => self.acquire_exclusive(request).await,
crate::types::LockType::Shared => self.acquire_shared(request).await,
}
}
/// Release lock
async fn release(&self, lock_id: &LockId) -> Result<bool>;
/// Refresh lock
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
/// Force release lock
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
/// Check lock status
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>>;
/// Get statistics
async fn get_stats(&self) -> Result<LockStats>;
/// Close client
async fn close(&self) -> Result<()>;
/// Check if client is online
async fn is_online(&self) -> bool;
/// Check if client is local
async fn is_local(&self) -> bool;
}
/// Client factory
pub struct ClientFactory;
impl ClientFactory {
/// Create local client
pub fn create_local() -> Arc<dyn LockClient> {
Arc::new(local::LocalClient::new())
}
/// Create remote client
pub fn create_remote(endpoint: String) -> Arc<dyn LockClient> {
Arc::new(remote::RemoteClient::new(endpoint))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::LockType;
#[tokio::test]
async fn test_client_factory() {
let local_client = ClientFactory::create_local();
assert!(local_client.is_local().await);
let remote_client = ClientFactory::create_remote("http://localhost:8080".to_string());
assert!(!remote_client.is_local().await);
}
#[tokio::test]
async fn test_local_client_basic_operations() {
let client = ClientFactory::create_local();
let request = crate::types::LockRequest::new("test-resource", LockType::Exclusive, "test-owner");
// Test lock acquisition
let response = client.acquire_exclusive(&request).await;
assert!(response.is_ok());
if let Ok(response) = response {
if response.success {
let lock_info = response.lock_info.unwrap();
// Test status check
let status = client.check_status(&lock_info.id).await;
assert!(status.is_ok());
assert!(status.unwrap().is_some());
// Test lock release
let released = client.release(&lock_info.id).await;
assert!(released.is_ok());
assert!(released.unwrap());
}
}
}
}
crates/lock/src/client/remote.rs
+403
View File
@@ -0,0 +1,403 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use rustfs_protos::{
node_service_time_out_client,
proto_gen::node_service::{GenerallyLockRequest, PingRequest},
};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use tonic::Request;
use tracing::info;
use crate::{
error::{LockError, Result},
types::{LockId, LockInfo, LockRequest, LockResponse, LockStats},
};
use super::LockClient;
/// Remote lock client implementation
#[derive(Debug)]
pub struct RemoteClient {
addr: String,
// Track active locks with their original owner information
active_locks: Arc<RwLock<HashMap<LockId, String>>>, // lock_id -> owner
}
impl Clone for RemoteClient {
fn clone(&self) -> Self {
Self {
addr: self.addr.clone(),
active_locks: self.active_locks.clone(),
}
}
}
impl RemoteClient {
pub fn new(endpoint: String) -> Self {
Self {
addr: endpoint,
active_locks: Arc::new(RwLock::new(HashMap::new())),
}
}
pub fn from_url(url: url::Url) -> Self {
Self {
addr: url.to_string(),
active_locks: Arc::new(RwLock::new(HashMap::new())),
}
}
/// Create a minimal LockRequest for unlock operations
fn create_unlock_request(&self, lock_id: &LockId, owner: &str) -> LockRequest {
LockRequest {
lock_id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: crate::types::LockType::Exclusive, // Type doesn't matter for unlock
owner: owner.to_string(),
acquire_timeout: std::time::Duration::from_secs(30),
ttl: std::time::Duration::from_secs(300),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
}
}
}
#[async_trait]
impl LockClient for RemoteClient {
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
info!("remote acquire_exclusive for {}", request.resource);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
// Check for explicit error first
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
// Check if the lock acquisition was successful
if resp.success {
// Save the lock information for later release
let mut locks = self.active_locks.write().await;
locks.insert(request.lock_id.clone(), request.owner.clone());
Ok(LockResponse::success(
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
} else {
// Lock acquisition failed
Ok(LockResponse::failure(
"Lock acquisition failed on remote server".to_string(),
std::time::Duration::ZERO,
))
}
}
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
info!("remote acquire_shared for {}", request.resource);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.r_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
// Check for explicit error first
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
// Check if the lock acquisition was successful
if resp.success {
// Save the lock information for later release
let mut locks = self.active_locks.write().await;
locks.insert(request.lock_id.clone(), request.owner.clone());
Ok(LockResponse::success(
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: crate::types::LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
} else {
// Lock acquisition failed
Ok(LockResponse::failure(
"Shared lock acquisition failed on remote server".to_string(),
std::time::Duration::ZERO,
))
}
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote release for {}", lock_id);
// Get the original owner for this lock
let owner = {
let locks = self.active_locks.read().await;
locks.get(lock_id).cloned().unwrap_or_else(|| "remote".to_string())
};
let unlock_request = self.create_unlock_request(lock_id, &owner);
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
// Try UnLock first (for exclusive locks)
let req = Request::new(GenerallyLockRequest {
args: request_string.clone(),
});
let resp = client.un_lock(req).await;
let success = if resp.is_err() {
// If that fails, try RUnLock (for shared locks)
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = client
.r_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
resp.success
} else {
let resp = resp.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
resp.success
};
// Remove the lock from our tracking if successful
if success {
let mut locks = self.active_locks.write().await;
locks.remove(lock_id);
}
Ok(success)
}
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
info!("remote refresh for {}", lock_id);
let refresh_request = self.create_unlock_request(lock_id, "remote");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.refresh(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = self.create_unlock_request(lock_id, "remote");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.force_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
info!("remote check_status for {}", lock_id);
// Since there's no direct status query in the gRPC service,
// we attempt a non-blocking lock acquisition to check if the resource is available
let status_request = self.create_unlock_request(lock_id, "remote");
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))?;
// Try to acquire a very short-lived lock to test availability
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
// Try exclusive lock first with very short timeout
let resp = client.lock(req).await;
match resp {
Ok(response) => {
let resp = response.into_inner();
if resp.success {
// If we successfully acquired the lock, the resource was free
// Immediately release it
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = client.un_lock(release_req).await; // Best effort release
// Return None since no one was holding the lock
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it
// We can't determine the exact details remotely, so return a generic status
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.as_str().to_string(),
lock_type: crate::types::LockType::Exclusive, // We can't know the exact type
status: crate::types::LockStatus::Acquired,
owner: "unknown".to_string(), // Remote client can't determine owner
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
wait_start_time: None,
}))
}
}
Err(_) => {
// Communication error or lock is held
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.as_str().to_string(),
lock_type: crate::types::LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
wait_start_time: None,
}))
}
}
}
async fn get_stats(&self) -> Result<LockStats> {
info!("remote get_stats from {}", self.addr);
// Since there's no direct statistics endpoint in the gRPC service,
// we return basic stats indicating this is a remote client
let stats = LockStats {
last_updated: std::time::SystemTime::now(),
..Default::default()
};
// We could potentially enhance this by:
// 1. Keeping local counters of operations performed
// 2. Adding a stats gRPC method to the service
// 3. Querying server health endpoints
// For now, return minimal stats indicating remote connectivity
Ok(stats)
}
async fn close(&self) -> Result<()> {
Ok(())
}
async fn is_online(&self) -> bool {
// Use Ping interface to test if remote service is online
let mut client = match node_service_time_out_client(&self.addr).await {
Ok(client) => client,
Err(_) => {
info!("remote client {} connection failed", self.addr);
return false;
}
};
let ping_req = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::new(),
});
match client.ping(ping_req).await {
Ok(_) => {
info!("remote client {} is online", self.addr);
true
}
Err(_) => {
info!("remote client {} ping failed", self.addr);
false
}
}
}
async fn is_local(&self) -> bool {
false
}
}
crates/lock/src/drwmutex.rs
-1212
View File
File diff suppressed because it is too large Load Diff
crates/lock/src/error.rs
+329
View File
@@ -0,0 +1,329 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::types::LockId;
use std::time::Duration;
use thiserror::Error;
/// Lock operation related error types
#[derive(Error, Debug)]
pub enum LockError {
/// Lock acquisition timeout
#[error("Lock acquisition timeout for resource '{resource}' after {timeout:?}")]
Timeout { resource: String, timeout: Duration },
/// Resource not found
#[error("Resource not found: {resource}")]
ResourceNotFound { resource: String },
/// Permission denied
#[error("Permission denied: {reason}")]
PermissionDenied { reason: String },
/// Network error
#[error("Network error: {message}")]
Network {
message: String,
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// Internal error
#[error("Internal error: {message}")]
Internal { message: String },
/// Resource is already locked
#[error("Resource '{resource}' is already locked by {owner}")]
AlreadyLocked { resource: String, owner: String },
/// Invalid lock handle
#[error("Invalid lock handle: {handle_id}")]
InvalidHandle { handle_id: String },
/// Configuration error
#[error("Configuration error: {message}")]
Configuration { message: String },
/// Serialization error
#[error("Serialization error: {message}")]
Serialization {
message: String,
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// Deserialization error
#[error("Deserialization error: {message}")]
Deserialization {
message: String,
#[source]
source: Box<dyn std::error::Error + Send + Sync>,
},
/// Insufficient nodes for quorum
#[error("Insufficient nodes for quorum: required {required}, available {available}")]
InsufficientNodes { required: usize, available: usize },
/// Quorum not reached
#[error("Quorum not reached: required {required}, achieved {achieved}")]
QuorumNotReached { required: usize, achieved: usize },
/// Queue is full
#[error("Queue is full: {message}")]
QueueFull { message: String },
/// Not the lock owner
#[error("Not the lock owner: lock_id {lock_id}, owner {owner}")]
NotOwner { lock_id: LockId, owner: String },
}
impl Clone for LockError {
fn clone(&self) -> Self {
match self {
LockError::Timeout { resource, timeout } => LockError::Timeout {
resource: resource.clone(),
timeout: *timeout,
},
LockError::ResourceNotFound { resource } => LockError::ResourceNotFound {
resource: resource.clone(),
},
LockError::PermissionDenied { reason } => LockError::PermissionDenied { reason: reason.clone() },
LockError::Network { message, source: _ } => LockError::Network {
message: message.clone(),
source: Box::new(std::io::Error::other(message.clone())),
},
LockError::Internal { message } => LockError::Internal {
message: message.clone(),
},
LockError::AlreadyLocked { resource, owner } => LockError::AlreadyLocked {
resource: resource.clone(),
owner: owner.clone(),
},
LockError::InvalidHandle { handle_id } => LockError::InvalidHandle {
handle_id: handle_id.clone(),
},
LockError::Configuration { message } => LockError::Configuration {
message: message.clone(),
},
LockError::Serialization { message, source: _ } => LockError::Serialization {
message: message.clone(),
source: Box::new(std::io::Error::other(message.clone())),
},
LockError::Deserialization { message, source: _ } => LockError::Deserialization {
message: message.clone(),
source: Box::new(std::io::Error::other(message.clone())),
},
LockError::InsufficientNodes { required, available } => LockError::InsufficientNodes {
required: *required,
available: *available,
},
LockError::QuorumNotReached { required, achieved } => LockError::QuorumNotReached {
required: *required,
achieved: *achieved,
},
LockError::QueueFull { message } => LockError::QueueFull {
message: message.clone(),
},
LockError::NotOwner { lock_id, owner } => LockError::NotOwner {
lock_id: lock_id.clone(),
owner: owner.clone(),
},
}
}
}
impl LockError {
/// Create timeout error
pub fn timeout(resource: impl Into<String>, timeout: Duration) -> Self {
Self::Timeout {
resource: resource.into(),
timeout,
}
}
/// Create resource not found error
pub fn resource_not_found(resource: impl Into<String>) -> Self {
Self::ResourceNotFound {
resource: resource.into(),
}
}
/// Create permission denied error
pub fn permission_denied(reason: impl Into<String>) -> Self {
Self::PermissionDenied { reason: reason.into() }
}
/// Create network error
pub fn network(message: impl Into<String>, source: impl std::error::Error + Send + Sync + 'static) -> Self {
Self::Network {
message: message.into(),
source: Box::new(source),
}
}
/// Create internal error
pub fn internal(message: impl Into<String>) -> Self {
Self::Internal { message: message.into() }
}
/// Create lock already locked error
pub fn already_locked(resource: impl Into<String>, owner: impl Into<String>) -> Self {
Self::AlreadyLocked {
resource: resource.into(),
owner: owner.into(),
}
}
/// Create invalid handle error
pub fn invalid_handle(handle_id: impl Into<String>) -> Self {
Self::InvalidHandle {
handle_id: handle_id.into(),
}
}
/// Create configuration error
pub fn configuration(message: impl Into<String>) -> Self {
Self::Configuration { message: message.into() }
}
/// Create serialization error
pub fn serialization(message: impl Into<String>, source: impl std::error::Error + Send + Sync + 'static) -> Self {
Self::Serialization {
message: message.into(),
source: Box::new(source),
}
}
/// Create deserialization error
pub fn deserialization(message: impl Into<String>, source: impl std::error::Error + Send + Sync + 'static) -> Self {
Self::Deserialization {
message: message.into(),
source: Box::new(source),
}
}
/// Check if it is a retryable error
pub fn is_retryable(&self) -> bool {
matches!(self, Self::Timeout { .. } | Self::Network { .. } | Self::Internal { .. })
}
/// Check if it is a fatal error
pub fn is_fatal(&self) -> bool {
matches!(
self,
Self::ResourceNotFound { .. } | Self::PermissionDenied { .. } | Self::Configuration { .. }
)
}
}
/// Lock operation Result type
pub type Result<T> = std::result::Result<T, LockError>;
/// Convert from std::io::Error
impl From<std::io::Error> for LockError {
fn from(err: std::io::Error) -> Self {
match err.kind() {
std::io::ErrorKind::TimedOut => Self::Internal {
message: "IO timeout".to_string(),
},
std::io::ErrorKind::NotFound => Self::ResourceNotFound {
resource: "unknown".to_string(),
},
std::io::ErrorKind::PermissionDenied => Self::PermissionDenied { reason: err.to_string() },
_ => Self::Internal {
message: err.to_string(),
},
}
}
}
/// Convert from serde_json::Error
impl From<serde_json::Error> for LockError {
fn from(err: serde_json::Error) -> Self {
if err.is_io() {
Self::network("JSON serialization IO error", err)
} else if err.is_syntax() {
Self::deserialization("JSON syntax error", err)
} else if err.is_data() {
Self::deserialization("JSON data error", err)
} else {
Self::serialization("JSON serialization error", err)
}
}
}
/// Convert from tonic::Status
impl From<tonic::Status> for LockError {
fn from(status: tonic::Status) -> Self {
match status.code() {
tonic::Code::DeadlineExceeded => Self::Internal {
message: "gRPC deadline exceeded".to_string(),
},
tonic::Code::NotFound => Self::ResourceNotFound {
resource: "unknown".to_string(),
},
tonic::Code::PermissionDenied => Self::PermissionDenied {
reason: status.message().to_string(),
},
tonic::Code::Unavailable => Self::Network {
message: "gRPC service unavailable".to_string(),
source: Box::new(status),
},
_ => Self::Internal {
message: status.message().to_string(),
},
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_error_creation() {
let timeout_err = LockError::timeout("test-resource", Duration::from_secs(5));
assert!(matches!(timeout_err, LockError::Timeout { .. }));
let not_found_err = LockError::resource_not_found("missing-resource");
assert!(matches!(not_found_err, LockError::ResourceNotFound { .. }));
let permission_err = LockError::permission_denied("insufficient privileges");
assert!(matches!(permission_err, LockError::PermissionDenied { .. }));
}
#[test]
fn test_error_retryable() {
let timeout_err = LockError::timeout("test", Duration::from_secs(1));
assert!(timeout_err.is_retryable());
let network_err = LockError::network("connection failed", std::io::Error::new(std::io::ErrorKind::ConnectionRefused, ""));
assert!(network_err.is_retryable());
let not_found_err = LockError::resource_not_found("test");
assert!(!not_found_err.is_retryable());
}
#[test]
fn test_error_fatal() {
let not_found_err = LockError::resource_not_found("test");
assert!(not_found_err.is_fatal());
let permission_err = LockError::permission_denied("test");
assert!(permission_err.is_fatal());
let timeout_err = LockError::timeout("test", Duration::from_secs(1));
assert!(!timeout_err.is_fatal());
}
}
crates/lock/src/lib.rs
+70 -107
View File
@@ -1,4 +1,4 @@
#![allow(dead_code)]
// #![allow(dead_code)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,115 +13,78 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use local_locker::LocalLocker;
use lock_args::LockArgs;
use remote_client::RemoteClient;
use std::io::Result;
// ============================================================================
// Core Module Declarations
// ============================================================================
// Application Layer Modules
pub mod namespace;
// Abstraction Layer Modules
pub mod client;
// Local Layer Modules
pub mod local;
// Core Modules
pub mod error;
pub mod types;
// ============================================================================
// Public API Exports
// ============================================================================
// Re-export main types for easy access
pub use crate::{
// Client interfaces
client::{LockClient, local::LocalClient, remote::RemoteClient},
// Error types
error::{LockError, Result},
local::LocalLockMap,
// Main components
namespace::{NamespaceLock, NamespaceLockManager},
// Core types
types::{
HealthInfo, HealthStatus, LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockStatus,
LockType,
},
};
// ============================================================================
// Version Information
// ============================================================================
/// Current version of the lock crate
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Build timestamp
pub const BUILD_TIMESTAMP: &str = "unknown";
/// Maximum number of items in delete list
pub const MAX_DELETE_LIST: usize = 1000;
// ============================================================================
// Global Lock Map
// ============================================================================
// Global singleton lock map shared across all lock implementations
use once_cell::sync::OnceCell;
use std::sync::Arc;
use std::sync::LazyLock;
use tokio::sync::RwLock;
pub mod drwmutex;
pub mod local_locker;
pub mod lock_args;
pub mod lrwmutex;
pub mod namespace_lock;
pub mod remote_client;
static GLOBAL_LOCK_MAP: OnceCell<Arc<local::LocalLockMap>> = OnceCell::new();
pub static GLOBAL_LOCAL_SERVER: LazyLock<Arc<RwLock<LocalLocker>>> = LazyLock::new(|| Arc::new(RwLock::new(LocalLocker::new())));
type LockClient = dyn Locker;
#[async_trait]
pub trait Locker {
async fn lock(&mut self, args: &LockArgs) -> Result<bool>;
async fn unlock(&mut self, args: &LockArgs) -> Result<bool>;
async fn rlock(&mut self, args: &LockArgs) -> Result<bool>;
async fn runlock(&mut self, args: &LockArgs) -> Result<bool>;
async fn refresh(&mut self, args: &LockArgs) -> Result<bool>;
async fn force_unlock(&mut self, args: &LockArgs) -> Result<bool>;
async fn close(&self);
async fn is_online(&self) -> bool;
async fn is_local(&self) -> bool;
/// Get the global shared lock map instance
pub fn get_global_lock_map() -> Arc<local::LocalLockMap> {
GLOBAL_LOCK_MAP.get_or_init(|| Arc::new(local::LocalLockMap::new())).clone()
}
#[derive(Debug, Clone)]
pub enum LockApi {
Local,
Remote(RemoteClient),
}
#[async_trait]
impl Locker for LockApi {
async fn lock(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.lock(args).await,
LockApi::Remote(r) => r.lock(args).await,
}
}
async fn unlock(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.unlock(args).await,
LockApi::Remote(r) => r.unlock(args).await,
}
}
async fn rlock(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.rlock(args).await,
LockApi::Remote(r) => r.rlock(args).await,
}
}
async fn runlock(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.runlock(args).await,
LockApi::Remote(r) => r.runlock(args).await,
}
}
async fn refresh(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.refresh(args).await,
LockApi::Remote(r) => r.refresh(args).await,
}
}
async fn force_unlock(&mut self, args: &LockArgs) -> Result<bool> {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.force_unlock(args).await,
LockApi::Remote(r) => r.force_unlock(args).await,
}
}
async fn close(&self) {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.read().await.close().await,
LockApi::Remote(r) => r.close().await,
}
}
async fn is_online(&self) -> bool {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.read().await.is_online().await,
LockApi::Remote(r) => r.is_online().await,
}
}
async fn is_local(&self) -> bool {
match self {
LockApi::Local => GLOBAL_LOCAL_SERVER.write().await.is_local().await,
LockApi::Remote(r) => r.is_local().await,
}
}
}
pub fn new_lock_api(is_local: bool, url: Option<url::Url>) -> LockApi {
if is_local {
return LockApi::Local;
}
LockApi::Remote(RemoteClient::new(url.unwrap()))
// ============================================================================
// Convenience Functions
// ============================================================================
/// Create a new namespace lock
pub fn create_namespace_lock(namespace: String, _distributed: bool) -> NamespaceLock {
// The distributed behavior is now determined by the type of clients added to the NamespaceLock
// This function just creates an empty NamespaceLock
NamespaceLock::new(namespace)
}
crates/lock/src/local.rs
+925
View File
@@ -0,0 +1,925 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use crate::LockRequest;
/// local lock entry
#[derive(Debug)]
pub struct LocalLockEntry {
/// current writer
pub writer: Option<String>,
/// current readers with their lock counts
pub readers: HashMap<String, usize>,
/// lock expiration time
pub expires_at: Option<Instant>,
}
/// local lock map
#[derive(Debug)]
pub struct LocalLockMap {
/// LockId to lock object map
pub locks: Arc<RwLock<HashMap<crate::types::LockId, Arc<RwLock<LocalLockEntry>>>>>,
/// Shutdown flag for background tasks
shutdown: Arc<AtomicBool>,
}
impl Default for LocalLockMap {
fn default() -> Self {
Self::new()
}
}
impl LocalLockMap {
/// create new local lock map
pub fn new() -> Self {
let map = Self {
locks: Arc::new(RwLock::new(HashMap::new())),
shutdown: Arc::new(AtomicBool::new(false)),
};
map.spawn_expiry_task();
map
}
/// spawn expiry task to clean up expired locks
fn spawn_expiry_task(&self) {
let locks = self.locks.clone();
let shutdown = self.shutdown.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(1));
loop {
interval.tick().await;
if shutdown.load(Ordering::Relaxed) {
tracing::debug!("Expiry task shutting down");
break;
}
let now = Instant::now();
let mut to_remove = Vec::new();
{
let locks_guard = locks.read().await;
for (key, entry) in locks_guard.iter() {
if let Ok(mut entry_guard) = entry.try_write() {
if let Some(exp) = entry_guard.expires_at {
if exp <= now {
entry_guard.writer = None;
entry_guard.readers.clear();
entry_guard.expires_at = None;
if entry_guard.writer.is_none() && entry_guard.readers.is_empty() {
to_remove.push(key.clone());
}
}
}
}
}
}
if !to_remove.is_empty() {
let mut locks_guard = locks.write().await;
for key in to_remove {
locks_guard.remove(&key);
}
}
}
});
}
/// write lock with TTL, support timeout, use LockRequest
pub async fn lock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result<bool> {
let start = Instant::now();
let expires_at = Some(Instant::now() + request.ttl);
loop {
// get or create lock entry
let entry = {
let mut locks_guard = self.locks.write().await;
locks_guard
.entry(request.lock_id.clone())
.or_insert_with(|| {
Arc::new(RwLock::new(LocalLockEntry {
writer: None,
readers: HashMap::new(),
expires_at: None,
}))
})
.clone()
};
// try to get write lock to modify state
if let Ok(mut entry_guard) = entry.try_write() {
// check expired state
let now = Instant::now();
if let Some(exp) = entry_guard.expires_at {
if exp <= now {
entry_guard.writer = None;
entry_guard.readers.clear();
entry_guard.expires_at = None;
}
}
// check if can get write lock
if entry_guard.writer.is_none() && entry_guard.readers.is_empty() {
entry_guard.writer = Some(request.owner.clone());
entry_guard.expires_at = expires_at;
tracing::debug!("Write lock acquired for resource '{}' by owner '{}'", request.resource, request.owner);
return Ok(true);
}
}
if start.elapsed() >= request.acquire_timeout {
return Ok(false);
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
/// read lock with TTL, support timeout, use LockRequest
pub async fn rlock_with_ttl_id(&self, request: &LockRequest) -> std::io::Result<bool> {
let start = Instant::now();
let expires_at = Some(Instant::now() + request.ttl);
loop {
// get or create lock entry
let entry = {
let mut locks_guard = self.locks.write().await;
locks_guard
.entry(request.lock_id.clone())
.or_insert_with(|| {
Arc::new(RwLock::new(LocalLockEntry {
writer: None,
readers: HashMap::new(),
expires_at: None,
}))
})
.clone()
};
// try to get write lock to modify state
if let Ok(mut entry_guard) = entry.try_write() {
// check expired state
let now = Instant::now();
if let Some(exp) = entry_guard.expires_at {
if exp <= now {
entry_guard.writer = None;
entry_guard.readers.clear();
entry_guard.expires_at = None;
}
}
// check if can get read lock
if entry_guard.writer.is_none() {
// increase read lock count
*entry_guard.readers.entry(request.owner.clone()).or_insert(0) += 1;
entry_guard.expires_at = expires_at;
tracing::debug!("Read lock acquired for resource '{}' by owner '{}'", request.resource, request.owner);
return Ok(true);
}
}
if start.elapsed() >= request.acquire_timeout {
return Ok(false);
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
/// unlock by LockId and owner - need to specify owner to correctly unlock
pub async fn unlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> {
println!("Unlocking lock_id: {lock_id:?}, owner: {owner}");
let mut need_remove = false;
{
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(lock_id) {
println!("Found lock entry, attempting to acquire write lock...");
match entry.try_write() {
Ok(mut entry_guard) => {
println!("Successfully acquired write lock for unlock");
// try to release write lock
if entry_guard.writer.as_ref() == Some(&owner.to_string()) {
println!("Releasing write lock for owner: {owner}");
entry_guard.writer = None;
}
// try to release read lock
else if let Some(count) = entry_guard.readers.get_mut(owner) {
println!("Releasing read lock for owner: {owner} (count: {count})");
*count -= 1;
if *count == 0 {
entry_guard.readers.remove(owner);
println!("Removed owner {owner} from readers");
}
} else {
println!("Owner {owner} not found in writers or readers");
}
// check if need to remove
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
println!("Lock entry is empty, marking for removal");
entry_guard.expires_at = None;
need_remove = true;
} else {
println!(
"Lock entry still has content: writer={:?}, readers={:?}",
entry_guard.writer, entry_guard.readers
);
}
}
Err(_) => {
println!("Failed to acquire write lock for unlock - this is the problem!");
return Err(std::io::Error::new(
std::io::ErrorKind::WouldBlock,
"Failed to acquire write lock for unlock",
));
}
}
} else {
println!("Lock entry not found for lock_id: {lock_id:?}");
}
}
// only here, entry's Ref is really dropped, can safely remove
if need_remove {
println!("Removing lock entry from map...");
let mut locks_guard = self.locks.write().await;
let removed = locks_guard.remove(lock_id);
println!("Lock entry removed: {:?}", removed.is_some());
}
println!("Unlock operation completed");
Ok(())
}
/// unlock by LockId - smart release (compatible with old interface, but may be inaccurate)
pub async fn unlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> {
let mut need_remove = false;
{
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(lock_id) {
if let Ok(mut entry_guard) = entry.try_write() {
// release write lock first
if entry_guard.writer.is_some() {
entry_guard.writer = None;
}
// if no write lock, release first read lock
else if let Some((owner, _)) = entry_guard.readers.iter().next() {
let owner = owner.clone();
if let Some(count) = entry_guard.readers.get_mut(&owner) {
*count -= 1;
if *count == 0 {
entry_guard.readers.remove(&owner);
}
}
}
// if completely idle, clean entry
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
entry_guard.expires_at = None;
need_remove = true;
}
}
}
}
if need_remove {
let mut locks_guard = self.locks.write().await;
locks_guard.remove(lock_id);
}
Ok(())
}
/// runlock by LockId and owner - need to specify owner to correctly unlock read lock
pub async fn runlock_by_id_and_owner(&self, lock_id: &crate::types::LockId, owner: &str) -> std::io::Result<()> {
let mut need_remove = false;
{
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(lock_id) {
if let Ok(mut entry_guard) = entry.try_write() {
// release read lock
if let Some(count) = entry_guard.readers.get_mut(owner) {
*count -= 1;
if *count == 0 {
entry_guard.readers.remove(owner);
}
}
// if completely idle, clean entry
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
entry_guard.expires_at = None;
need_remove = true;
}
}
}
}
if need_remove {
let mut locks_guard = self.locks.write().await;
locks_guard.remove(lock_id);
}
Ok(())
}
/// runlock by LockId - smart release read lock (compatible with old interface)
pub async fn runlock_by_id(&self, lock_id: &crate::types::LockId) -> std::io::Result<()> {
let mut need_remove = false;
{
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(lock_id) {
if let Ok(mut entry_guard) = entry.try_write() {
// release first read lock
if let Some((owner, _)) = entry_guard.readers.iter().next() {
let owner = owner.clone();
if let Some(count) = entry_guard.readers.get_mut(&owner) {
*count -= 1;
if *count == 0 {
entry_guard.readers.remove(&owner);
}
}
}
// if completely idle, clean entry
if entry_guard.readers.is_empty() && entry_guard.writer.is_none() {
entry_guard.expires_at = None;
need_remove = true;
}
}
}
}
if need_remove {
let mut locks_guard = self.locks.write().await;
locks_guard.remove(lock_id);
}
Ok(())
}
/// check if resource is locked
pub async fn is_locked(&self, resource: &str) -> bool {
let lock_id = crate::types::LockId::new_deterministic(resource);
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(&lock_id) {
let entry_guard = entry.read().await;
entry_guard.writer.is_some() || !entry_guard.readers.is_empty()
} else {
false
}
}
/// get lock info for a resource
pub async fn get_lock(&self, resource: &str) -> Option<crate::types::LockInfo> {
let lock_id = crate::types::LockId::new_deterministic(resource);
let locks_guard = self.locks.read().await;
if let Some(entry) = locks_guard.get(&lock_id) {
let entry_guard = entry.read().await;
if let Some(owner) = &entry_guard.writer {
Some(crate::types::LockInfo {
id: lock_id,
resource: resource.to_string(),
lock_type: crate::types::LockType::Exclusive,
status: crate::types::LockStatus::Acquired,
owner: owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
last_refreshed: std::time::SystemTime::now(),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
wait_start_time: None,
})
} else if !entry_guard.readers.is_empty() {
let owner = entry_guard.readers.keys().next().unwrap().clone();
Some(crate::types::LockInfo {
id: lock_id,
resource: resource.to_string(),
lock_type: crate::types::LockType::Shared,
status: crate::types::LockStatus::Acquired,
owner,
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(30),
last_refreshed: std::time::SystemTime::now(),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
wait_start_time: None,
})
} else {
None
}
} else {
None
}
}
/// get statistics
pub async fn get_stats(&self) -> crate::types::LockStats {
let mut stats = crate::types::LockStats::default();
let locks_guard = self.locks.read().await;
for (_, entry) in locks_guard.iter() {
let entry_guard = entry.read().await;
if entry_guard.writer.is_some() {
stats.exclusive_locks += 1;
}
stats.shared_locks += entry_guard.readers.len();
}
stats.total_locks = stats.exclusive_locks + stats.shared_locks;
stats
}
/// shutdown background tasks
pub async fn shutdown(&self) {
self.shutdown.store(true, Ordering::Relaxed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::time::Duration;
use tokio::task;
use tokio::time::{sleep, timeout};
/// Test basic write lock operations
#[tokio::test]
async fn test_write_lock_basic() {
let lock_map = LocalLockMap::new();
// create a simple lock request
let request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("test_resource"),
resource: "test_resource".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "test_owner".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
// try to acquire lock
println!("Attempting to acquire lock...");
let result = lock_map.lock_with_ttl_id(&request).await;
println!("Lock acquisition result: {result:?}");
match result {
Ok(success) => {
if success {
println!("Lock acquired successfully");
// check lock state
let is_locked = lock_map.is_locked("test_resource").await;
println!("Is locked: {is_locked}");
// try to unlock
println!("Attempting to unlock...");
let unlock_result = lock_map.unlock_by_id_and_owner(&request.lock_id, "test_owner").await;
println!("Unlock result: {unlock_result:?}");
// check lock state again
let is_locked_after = lock_map.is_locked("test_resource").await;
println!("Is locked after unlock: {is_locked_after}");
assert!(!is_locked_after, "Should be unlocked after release");
} else {
println!("Lock acquisition failed (timeout)");
}
}
Err(e) => {
println!("Lock acquisition error: {e:?}");
panic!("Lock acquisition failed with error: {e:?}");
}
}
}
/// Test basic read lock operations
#[tokio::test]
async fn test_read_lock_basic() {
let lock_map = LocalLockMap::new();
// Test successful acquisition
let request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("bar"),
resource: "bar".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok = lock_map.rlock_with_ttl_id(&request).await.unwrap();
assert!(ok, "Read lock should be successfully acquired");
assert!(lock_map.is_locked("bar").await, "Lock state should be locked");
// Test lock info
let lock_info = lock_map.get_lock("bar").await;
assert!(lock_info.is_some(), "Lock info should exist");
let info = lock_info.unwrap();
assert_eq!(info.owner, "reader1");
assert_eq!(info.lock_type, crate::types::LockType::Shared);
// Test unlock with owner
lock_map.runlock_by_id_and_owner(&request.lock_id, "reader1").await.unwrap();
assert!(!lock_map.is_locked("bar").await, "Should be unlocked after release");
}
/// Test write lock mutual exclusion
#[tokio::test]
async fn test_write_lock_mutex() {
let lock_map = Arc::new(LocalLockMap::new());
// Owner1 acquires write lock
let request1 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_mutex_test"),
resource: "res_mutex_test".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "owner1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok = lock_map.lock_with_ttl_id(&request1).await.unwrap();
assert!(ok, "First write lock should succeed");
// Owner2 tries to acquire write lock on same resource - should fail due to timeout
let lock_map2 = lock_map.clone();
let request2 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_mutex_test"),
resource: "res_mutex_test".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "owner2".to_string(),
acquire_timeout: Duration::from_millis(50),
ttl: Duration::from_millis(50),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let request2_clone = request2.clone();
let result = timeout(Duration::from_millis(100), async move {
lock_map2.lock_with_ttl_id(&request2_clone).await.unwrap()
})
.await;
assert!(result.is_ok(), "Lock attempt should complete");
assert!(!result.unwrap(), "Second write lock should fail due to conflict");
// Release first lock
lock_map.unlock_by_id_and_owner(&request1.lock_id, "owner1").await.unwrap();
// Now owner2 should be able to acquire the lock
let ok = lock_map.lock_with_ttl_id(&request2).await.unwrap();
assert!(ok, "Write lock should succeed after first is released");
lock_map.unlock_by_id_and_owner(&request2.lock_id, "owner2").await.unwrap();
}
/// Test read lock sharing
#[tokio::test]
async fn test_read_lock_sharing() {
let lock_map = LocalLockMap::new();
// Multiple readers should be able to acquire read locks
let request1 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_sharing_test"),
resource: "res_sharing_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let request2 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_sharing_test"),
resource: "res_sharing_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader2".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let request3 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_sharing_test"),
resource: "res_sharing_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader3".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok1 = lock_map.rlock_with_ttl_id(&request1).await.unwrap();
let ok2 = lock_map.rlock_with_ttl_id(&request2).await.unwrap();
let ok3 = lock_map.rlock_with_ttl_id(&request3).await.unwrap();
assert!(ok1 && ok2 && ok3, "All read locks should succeed");
assert!(lock_map.is_locked("res_sharing_test").await, "Resource should be locked");
// Release readers one by one
lock_map.runlock_by_id_and_owner(&request1.lock_id, "reader1").await.unwrap();
assert!(
lock_map.is_locked("res_sharing_test").await,
"Should still be locked with remaining readers"
);
lock_map.runlock_by_id_and_owner(&request2.lock_id, "reader2").await.unwrap();
assert!(lock_map.is_locked("res_sharing_test").await, "Should still be locked with one reader");
lock_map.runlock_by_id_and_owner(&request3.lock_id, "reader3").await.unwrap();
assert!(
!lock_map.is_locked("res_sharing_test").await,
"Should be unlocked when all readers release"
);
}
/// Test read-write lock exclusion
#[tokio::test]
async fn test_read_write_exclusion() {
let lock_map = LocalLockMap::new();
// Reader acquires read lock
let read_request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_rw_test"),
resource: "res_rw_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok = lock_map.rlock_with_ttl_id(&read_request).await.unwrap();
assert!(ok, "Read lock should succeed");
// Writer tries to acquire write lock - should fail
let write_request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_rw_test"),
resource: "res_rw_test".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "writer1".to_string(),
acquire_timeout: Duration::from_millis(50),
ttl: Duration::from_millis(50),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let result = timeout(Duration::from_millis(100), async {
lock_map.lock_with_ttl_id(&write_request).await.unwrap()
})
.await;
assert!(result.is_ok(), "Write lock attempt should complete");
assert!(!result.unwrap(), "Write lock should fail when read lock is held");
// Release read lock
lock_map
.runlock_by_id_and_owner(&read_request.lock_id, "reader1")
.await
.unwrap();
// Now writer should be able to acquire the lock with longer TTL
let write_request_long_ttl = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_rw_test"),
resource: "res_rw_test".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "writer1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(200), // Longer TTL to prevent expiration during test
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok = lock_map.lock_with_ttl_id(&write_request_long_ttl).await.unwrap();
assert!(ok, "Write lock should succeed after read lock is released");
// Reader tries to acquire read lock while write lock is held - should fail
let read_request2 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res_rw_test"),
resource: "res_rw_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader2".to_string(),
acquire_timeout: Duration::from_millis(50),
ttl: Duration::from_millis(50),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let result = timeout(Duration::from_millis(100), async {
lock_map.rlock_with_ttl_id(&read_request2).await.unwrap()
})
.await;
assert!(result.is_ok(), "Read lock attempt should complete");
assert!(!result.unwrap(), "Read lock should fail when write lock is held");
// Release write lock
lock_map
.unlock_by_id_and_owner(&write_request_long_ttl.lock_id, "writer1")
.await
.unwrap();
}
/// Test statistics
#[tokio::test]
async fn test_statistics() {
let lock_map = LocalLockMap::new();
// Initially no locks
let stats = lock_map.get_stats().await;
assert_eq!(stats.total_locks, 0, "Should have no locks initially");
assert_eq!(stats.exclusive_locks, 0, "Should have no exclusive locks initially");
assert_eq!(stats.shared_locks, 0, "Should have no shared locks initially");
// Add some locks
let write_request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res1_stats_test"),
resource: "res1_stats_test".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "owner1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let read_request1 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res2_stats_test"),
resource: "res2_stats_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let read_request2 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("res2_stats_test"),
resource: "res2_stats_test".to_string(),
lock_type: crate::types::LockType::Shared,
owner: "reader2".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(100),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
lock_map.lock_with_ttl_id(&write_request).await.unwrap();
lock_map.rlock_with_ttl_id(&read_request1).await.unwrap();
lock_map.rlock_with_ttl_id(&read_request2).await.unwrap();
let stats = lock_map.get_stats().await;
assert_eq!(stats.exclusive_locks, 1, "Should have 1 exclusive lock");
assert_eq!(stats.shared_locks, 2, "Should have 2 shared locks");
assert_eq!(stats.total_locks, 3, "Should have 3 total locks");
// Clean up
lock_map
.unlock_by_id_and_owner(&write_request.lock_id, "owner1")
.await
.unwrap();
lock_map
.runlock_by_id_and_owner(&read_request1.lock_id, "reader1")
.await
.unwrap();
lock_map
.runlock_by_id_and_owner(&read_request2.lock_id, "reader2")
.await
.unwrap();
}
/// Test concurrent access
#[tokio::test]
async fn test_concurrent_access() {
let lock_map = Arc::new(LocalLockMap::new());
let num_tasks = 10;
let num_iterations = 100;
let mut handles = Vec::new();
for i in 0..num_tasks {
let lock_map = lock_map.clone();
let owner = format!("owner{i}");
let handle = task::spawn(async move {
for j in 0..num_iterations {
let resource = format!("resource{}", j % 5);
let request = LockRequest {
lock_id: crate::types::LockId::new_deterministic(&resource),
resource: resource.clone(),
lock_type: if j % 2 == 0 {
crate::types::LockType::Exclusive
} else {
crate::types::LockType::Shared
},
owner: owner.clone(),
acquire_timeout: Duration::from_millis(10),
ttl: Duration::from_millis(10),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
if request.lock_type == crate::types::LockType::Exclusive {
if lock_map.lock_with_ttl_id(&request).await.unwrap() {
sleep(Duration::from_micros(100)).await;
lock_map.unlock_by_id_and_owner(&request.lock_id, &owner).await.unwrap();
}
} else if lock_map.rlock_with_ttl_id(&request).await.unwrap() {
sleep(Duration::from_micros(100)).await;
lock_map.runlock_by_id_and_owner(&request.lock_id, &owner).await.unwrap();
}
}
});
handles.push(handle);
}
for handle in handles {
handle.await.unwrap();
}
// Verify no locks remain
let stats = lock_map.get_stats().await;
assert_eq!(stats.total_locks, 0, "No locks should remain after concurrent access");
}
#[tokio::test]
async fn test_write_lock_timeout_and_reacquire() {
let lock_map = LocalLockMap::new();
// 1. acquire lock
let request = LockRequest {
lock_id: crate::types::LockId::new_deterministic("timeout_resource"),
resource: "timeout_resource".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "owner1".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(200),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok = lock_map.lock_with_ttl_id(&request).await.unwrap();
assert!(ok, "First lock should succeed");
// 2. try to acquire lock again, should fail
let request2 = LockRequest {
lock_id: crate::types::LockId::new_deterministic("timeout_resource"),
resource: "timeout_resource".to_string(),
lock_type: crate::types::LockType::Exclusive,
owner: "owner2".to_string(),
acquire_timeout: Duration::from_millis(100),
ttl: Duration::from_millis(200),
metadata: crate::types::LockMetadata::default(),
priority: crate::types::LockPriority::Normal,
deadlock_detection: false,
};
let ok2 = lock_map.lock_with_ttl_id(&request2).await.unwrap();
assert!(!ok2, "Second lock should fail before timeout");
// 3. wait for TTL to expire
tokio::time::sleep(Duration::from_millis(300)).await;
// 4. try to acquire lock again, should succeed
let ok3 = lock_map.lock_with_ttl_id(&request2).await.unwrap();
assert!(ok3, "Lock should succeed after timeout");
}
}
crates/lock/src/local_locker.rs
-427
View File
@@ -1,427 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use std::io::{Error, Result};
use std::{
collections::HashMap,
time::{Duration, Instant},
};
use crate::{Locker, lock_args::LockArgs};
pub const MAX_DELETE_LIST: usize = 1000;
#[derive(Clone, Debug)]
struct LockRequesterInfo {
name: String,
writer: bool,
uid: String,
time_stamp: Instant,
time_last_refresh: Instant,
source: String,
group: bool,
owner: String,
quorum: usize,
idx: usize,
}
impl Default for LockRequesterInfo {
fn default() -> Self {
Self {
name: Default::default(),
writer: Default::default(),
uid: Default::default(),
time_stamp: Instant::now(),
time_last_refresh: Instant::now(),
source: Default::default(),
group: Default::default(),
owner: Default::default(),
quorum: Default::default(),
idx: Default::default(),
}
}
}
fn is_write_lock(lri: &[LockRequesterInfo]) -> bool {
lri.len() == 1 && lri[0].writer
}
#[derive(Debug, Default)]
pub struct LockStats {
total: usize,
writes: usize,
reads: usize,
}
#[derive(Debug, Default)]
pub struct LocalLocker {
lock_map: HashMap<String, Vec<LockRequesterInfo>>,
lock_uid: HashMap<String, String>,
}
impl LocalLocker {
pub fn new() -> Self {
LocalLocker::default()
}
}
impl LocalLocker {
fn can_take_lock(&self, resource: &[String]) -> bool {
resource.iter().fold(true, |acc, x| !self.lock_map.contains_key(x) && acc)
}
pub fn stats(&self) -> LockStats {
let mut st = LockStats {
total: self.lock_map.len(),
..Default::default()
};
self.lock_map.iter().for_each(|(_, value)| {
if !value.is_empty() {
if value[0].writer {
st.writes += 1;
} else {
st.reads += 1;
}
}
});
st
}
fn dump_lock_map(&mut self) -> HashMap<String, Vec<LockRequesterInfo>> {
let mut lock_copy = HashMap::new();
self.lock_map.iter().for_each(|(key, value)| {
lock_copy.insert(key.to_string(), value.to_vec());
});
lock_copy
}
fn expire_old_locks(&mut self, interval: Duration) {
self.lock_map.iter_mut().for_each(|(_, lris)| {
lris.retain(|lri| {
if Instant::now().duration_since(lri.time_last_refresh) > interval {
let mut key = lri.uid.to_string();
format_uuid(&mut key, &lri.idx);
self.lock_uid.remove(&key);
return false;
}
true
});
});
}
}
#[async_trait]
impl Locker for LocalLocker {
async fn lock(&mut self, args: &LockArgs) -> Result<bool> {
if args.resources.len() > MAX_DELETE_LIST {
return Err(Error::other(format!(
"internal error: LocalLocker.lock called with more than {MAX_DELETE_LIST} resources"
)));
}
if !self.can_take_lock(&args.resources) {
return Ok(false);
}
args.resources.iter().enumerate().for_each(|(idx, resource)| {
self.lock_map.insert(
resource.to_string(),
vec![LockRequesterInfo {
name: resource.to_string(),
writer: true,
source: args.source.to_string(),
owner: args.owner.to_string(),
uid: args.uid.to_string(),
group: args.resources.len() > 1,
quorum: args.quorum,
idx,
..Default::default()
}],
);
let mut uuid = args.uid.to_string();
format_uuid(&mut uuid, &idx);
self.lock_uid.insert(uuid, resource.to_string());
});
Ok(true)
}
async fn unlock(&mut self, args: &LockArgs) -> Result<bool> {
if args.resources.len() > MAX_DELETE_LIST {
return Err(Error::other(format!(
"internal error: LocalLocker.unlock called with more than {MAX_DELETE_LIST} resources"
)));
}
let mut reply = false;
let mut err_info = String::new();
for resource in args.resources.iter() {
match self.lock_map.get_mut(resource) {
Some(lris) => {
if !is_write_lock(lris) {
if err_info.is_empty() {
err_info = format!("unlock attempted on a read locked entity: {resource}");
} else {
err_info.push_str(&format!(", {resource}"));
}
} else {
lris.retain(|lri| {
if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) {
let mut key = args.uid.to_string();
format_uuid(&mut key, &lri.idx);
self.lock_uid.remove(&key).unwrap();
reply |= true;
return false;
}
true
});
}
if lris.is_empty() {
self.lock_map.remove(resource);
}
}
None => {
continue;
}
};
}
Ok(reply)
}
async fn rlock(&mut self, args: &LockArgs) -> Result<bool> {
if args.resources.len() != 1 {
return Err(Error::other("internal error: localLocker.RLock called with more than one resource"));
}
let resource = &args.resources[0];
match self.lock_map.get_mut(resource) {
Some(lri) => {
if !is_write_lock(lri) {
lri.push(LockRequesterInfo {
name: resource.to_string(),
writer: false,
source: args.source.to_string(),
owner: args.owner.to_string(),
uid: args.uid.to_string(),
quorum: args.quorum,
..Default::default()
});
} else {
return Ok(false);
}
}
None => {
self.lock_map.insert(
resource.to_string(),
vec![LockRequesterInfo {
name: resource.to_string(),
writer: false,
source: args.source.to_string(),
owner: args.owner.to_string(),
uid: args.uid.to_string(),
quorum: args.quorum,
..Default::default()
}],
);
}
}
let mut uuid = args.uid.to_string();
format_uuid(&mut uuid, &0);
self.lock_uid.insert(uuid, resource.to_string());
Ok(true)
}
async fn runlock(&mut self, args: &LockArgs) -> Result<bool> {
if args.resources.len() != 1 {
return Err(Error::other("internal error: localLocker.RLock called with more than one resource"));
}
let mut reply = false;
let resource = &args.resources[0];
match self.lock_map.get_mut(resource) {
Some(lris) => {
if is_write_lock(lris) {
return Err(Error::other(format!("runlock attempted on a write locked entity: {resource}")));
} else {
lris.retain(|lri| {
if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) {
let mut key = args.uid.to_string();
format_uuid(&mut key, &lri.idx);
self.lock_uid.remove(&key).unwrap();
reply |= true;
return false;
}
true
});
}
if lris.is_empty() {
self.lock_map.remove(resource);
}
}
None => {
return Ok(reply);
}
};
Ok(reply)
}
async fn refresh(&mut self, args: &LockArgs) -> Result<bool> {
let mut idx = 0;
let mut key = args.uid.to_string();
format_uuid(&mut key, &idx);
match self.lock_uid.get(&key) {
Some(resource) => {
let mut resource = resource;
loop {
match self.lock_map.get_mut(resource) {
Some(_lris) => {}
None => {
let mut key = args.uid.to_string();
format_uuid(&mut key, &0);
self.lock_uid.remove(&key);
return Ok(idx > 0);
}
}
idx += 1;
let mut key = args.uid.to_string();
format_uuid(&mut key, &idx);
resource = match self.lock_uid.get(&key) {
Some(resource) => resource,
None => return Ok(true),
};
}
}
None => Ok(false),
}
}
// TODO: need add timeout mechanism
async fn force_unlock(&mut self, args: &LockArgs) -> Result<bool> {
if args.uid.is_empty() {
args.resources.iter().for_each(|resource| {
if let Some(lris) = self.lock_map.get(resource) {
lris.iter().for_each(|lri| {
let mut key = lri.uid.to_string();
format_uuid(&mut key, &lri.idx);
self.lock_uid.remove(&key);
});
if lris.is_empty() {
self.lock_map.remove(resource);
}
}
});
return Ok(true);
}
let mut idx = 0;
let mut need_remove_resource = Vec::new();
let mut need_remove_map_id = Vec::new();
let reply = loop {
let mut map_id = args.uid.to_string();
format_uuid(&mut map_id, &idx);
match self.lock_uid.get(&map_id) {
Some(resource) => match self.lock_map.get_mut(resource) {
Some(lris) => {
{
lris.retain(|lri| {
if lri.uid == args.uid && (args.owner.is_empty() || lri.owner == args.owner) {
let mut key = args.uid.to_string();
format_uuid(&mut key, &lri.idx);
need_remove_map_id.push(key);
return false;
}
true
});
}
idx += 1;
if lris.is_empty() {
need_remove_resource.push(resource.to_string());
}
}
None => {
need_remove_map_id.push(map_id);
idx += 1;
continue;
}
},
None => {
break idx > 0;
}
}
};
need_remove_resource.into_iter().for_each(|resource| {
self.lock_map.remove(&resource);
});
need_remove_map_id.into_iter().for_each(|map_id| {
self.lock_uid.remove(&map_id);
});
Ok(reply)
}
async fn close(&self) {}
async fn is_online(&self) -> bool {
true
}
async fn is_local(&self) -> bool {
true
}
}
fn format_uuid(s: &mut String, idx: &usize) {
s.push_str(&idx.to_string());
}
#[cfg(test)]
mod test {
use super::LocalLocker;
use crate::{Locker, lock_args::LockArgs};
use std::io::Result;
use tokio;
#[tokio::test]
async fn test_lock_unlock() -> Result<()> {
let mut local_locker = LocalLocker::new();
let args = LockArgs {
uid: "1111".to_string(),
resources: vec!["dandan".to_string()],
owner: "dd".to_string(),
source: "".to_string(),
quorum: 3,
};
local_locker.lock(&args).await?;
println!("lock local_locker: {local_locker:?} \n");
local_locker.unlock(&args).await?;
println!("unlock local_locker: {local_locker:?}");
Ok(())
}
}
crates/lock/src/lock_args.rs
-35
View File
@@ -1,35 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use std::fmt::Display;
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct LockArgs {
pub uid: String,
pub resources: Vec<String>,
pub owner: String,
pub source: String,
pub quorum: usize,
}
impl Display for LockArgs {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"LockArgs[ uid: {}, resources: {:?}, owner: {}, source:{}, quorum: {} ]",
self.uid, self.resources, self.owner, self.source, self.quorum
)
}
}
crates/lock/src/lrwmutex.rs
-191
View File
@@ -1,191 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rand::Rng;
use std::time::{Duration, Instant};
use tokio::{sync::RwLock, time::sleep};
use tracing::info;
#[derive(Debug, Default)]
pub struct LRWMutex {
id: RwLock<String>,
source: RwLock<String>,
is_write: RwLock<bool>,
reference: RwLock<usize>,
}
impl LRWMutex {
pub async fn lock(&self) -> bool {
let is_write = true;
let id = self.id.read().await.clone();
let source = self.source.read().await.clone();
let timeout = Duration::from_secs(10000);
self.look_loop(
&id, &source, &timeout, // big enough
is_write,
)
.await
}
pub async fn get_lock(&self, id: &str, source: &str, timeout: &Duration) -> bool {
let is_write = true;
self.look_loop(id, source, timeout, is_write).await
}
pub async fn r_lock(&self) -> bool {
let is_write: bool = false;
let id = self.id.read().await.clone();
let source = self.source.read().await.clone();
let timeout = Duration::from_secs(10000);
self.look_loop(
&id, &source, &timeout, // big enough
is_write,
)
.await
}
pub async fn get_r_lock(&self, id: &str, source: &str, timeout: &Duration) -> bool {
let is_write = false;
self.look_loop(id, source, timeout, is_write).await
}
async fn inner_lock(&self, id: &str, source: &str, is_write: bool) -> bool {
*self.id.write().await = id.to_string();
*self.source.write().await = source.to_string();
let mut locked = false;
if is_write {
if *self.reference.read().await == 0 && !*self.is_write.read().await {
*self.reference.write().await = 1;
*self.is_write.write().await = true;
locked = true;
}
} else if !*self.is_write.read().await {
*self.reference.write().await += 1;
locked = true;
}
locked
}
async fn look_loop(&self, id: &str, source: &str, timeout: &Duration, is_write: bool) -> bool {
let start = Instant::now();
loop {
if self.inner_lock(id, source, is_write).await {
return true;
} else {
if Instant::now().duration_since(start) > *timeout {
return false;
}
let sleep_time: u64;
{
let mut rng = rand::rng();
sleep_time = rng.random_range(10..=50);
}
sleep(Duration::from_millis(sleep_time)).await;
}
}
}
pub async fn un_lock(&self) {
let is_write = true;
if !self.unlock(is_write).await {
info!("Trying to un_lock() while no Lock() is active")
}
}
pub async fn un_r_lock(&self) {
let is_write = false;
if !self.unlock(is_write).await {
info!("Trying to un_r_lock() while no Lock() is active")
}
}
async fn unlock(&self, is_write: bool) -> bool {
let mut unlocked = false;
if is_write {
if *self.is_write.read().await && *self.reference.read().await == 1 {
*self.reference.write().await = 0;
*self.is_write.write().await = false;
unlocked = true;
}
} else if !*self.is_write.read().await && *self.reference.read().await > 0 {
*self.reference.write().await -= 1;
unlocked = true;
}
unlocked
}
pub async fn force_un_lock(&self) {
*self.reference.write().await = 0;
*self.is_write.write().await = false;
}
}
#[cfg(test)]
mod test {
use std::{sync::Arc, time::Duration};
use std::io::Result;
use tokio::time::sleep;
use crate::lrwmutex::LRWMutex;
#[tokio::test]
async fn test_lock_unlock() -> Result<()> {
let l_rw_lock = LRWMutex::default();
let id = "foo";
let source = "dandan";
let timeout = Duration::from_secs(5);
assert!(l_rw_lock.get_lock(id, source, &timeout).await);
l_rw_lock.un_lock().await;
l_rw_lock.lock().await;
assert!(!l_rw_lock.get_r_lock(id, source, &timeout).await);
l_rw_lock.un_lock().await;
assert!(l_rw_lock.get_r_lock(id, source, &timeout).await);
Ok(())
}
#[tokio::test]
async fn multi_thread_test() -> Result<()> {
let l_rw_lock = Arc::new(LRWMutex::default());
let id = "foo";
let source = "dandan";
let one_fn = async {
let one = Arc::clone(&l_rw_lock);
let timeout = Duration::from_secs(1);
assert!(one.get_lock(id, source, &timeout).await);
sleep(Duration::from_secs(5)).await;
l_rw_lock.un_lock().await;
};
let two_fn = async {
let two = Arc::clone(&l_rw_lock);
let timeout = Duration::from_secs(2);
assert!(!two.get_r_lock(id, source, &timeout).await);
sleep(Duration::from_secs(5)).await;
assert!(two.get_r_lock(id, source, &timeout).await);
two.un_r_lock().await;
};
tokio::join!(one_fn, two_fn);
Ok(())
}
}
crates/lock/src/namespace.rs
+346
View File
@@ -0,0 +1,346 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use std::sync::Arc;
use std::time::Duration;
use crate::{
client::LockClient,
error::{LockError, Result},
types::{LockId, LockInfo, LockRequest, LockResponse, LockStatus, LockType},
};
/// Namespace lock for managing locks by resource namespaces
#[derive(Debug)]
pub struct NamespaceLock {
/// Lock clients for this namespace
clients: Vec<Arc<dyn LockClient>>,
/// Namespace identifier
namespace: String,
/// Quorum size for operations (1 for local, majority for distributed)
quorum: usize,
}
impl NamespaceLock {
/// Create new namespace lock
pub fn new(namespace: String) -> Self {
Self {
clients: Vec::new(),
namespace,
quorum: 1,
}
}
/// Create namespace lock with clients
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
let quorum = if clients.len() > 1 {
// For multiple clients (distributed mode), require majority
(clients.len() / 2) + 1
} else {
// For single client (local mode), only need 1
1
};
Self {
clients,
namespace,
quorum,
}
}
/// Create namespace lock with client (compatibility)
pub fn with_client(client: Arc<dyn LockClient>) -> Self {
Self::with_clients("default".to_string(), vec![client])
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
&self.namespace
}
/// Get resource key for this namespace
fn get_resource_key(&self, resource: &str) -> String {
format!("{}:{}", self.namespace, resource)
}
/// Acquire lock using clients
pub async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For single client, use it directly
if self.clients.len() == 1 {
return self.clients[0].acquire_lock(request).await;
}
// For multiple clients, try to acquire from all clients and require quorum
let futures: Vec<_> = self
.clients
.iter()
.map(|client| async move { client.acquire_lock(request).await })
.collect();
let results = futures::future::join_all(futures).await;
let successful = results.into_iter().filter_map(|r| r.ok()).filter(|r| r.success).count();
if successful >= self.quorum {
Ok(LockResponse::success(
LockInfo {
id: LockId::new_deterministic(&request.resource),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
Duration::ZERO,
))
} else {
Ok(LockResponse::failure("Failed to acquire quorum".to_string(), Duration::ZERO))
}
}
/// Release lock using clients
pub async fn release_lock(&self, lock_id: &LockId) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For single client, use it directly
if self.clients.len() == 1 {
return self.clients[0].release(lock_id).await;
}
// For multiple clients, try to release from all clients
let futures: Vec<_> = self
.clients
.iter()
.map(|client| {
let id = lock_id.clone();
async move { client.release(&id).await }
})
.collect();
let results = futures::future::join_all(futures).await;
let successful = results.into_iter().filter_map(|r| r.ok()).filter(|&r| r).count();
// For release, if any succeed, consider it successful
Ok(successful > 0)
}
/// Get health information
pub async fn get_health(&self) -> crate::types::HealthInfo {
let lock_stats = self.get_stats().await;
let mut health = crate::types::HealthInfo {
node_id: self.namespace.clone(),
lock_stats,
..Default::default()
};
// Check client status
let mut connected_clients = 0;
for client in &self.clients {
if client.is_online().await {
connected_clients += 1;
}
}
health.status = if connected_clients > 0 {
crate::types::HealthStatus::Healthy
} else {
crate::types::HealthStatus::Degraded
};
health.connected_nodes = connected_clients;
health.total_nodes = self.clients.len();
health
}
/// Get namespace statistics
pub async fn get_stats(&self) -> crate::types::LockStats {
let mut stats = crate::types::LockStats::default();
// Try to get stats from clients
for client in &self.clients {
if let Ok(client_stats) = client.get_stats().await {
stats.successful_acquires += client_stats.successful_acquires;
stats.failed_acquires += client_stats.failed_acquires;
}
}
stats
}
}
impl Default for NamespaceLock {
fn default() -> Self {
Self::new("default".to_string())
}
}
/// Namespace lock manager trait
#[async_trait]
pub trait NamespaceLockManager: Send + Sync {
/// Batch get write lock
async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool>;
/// Batch release write lock
async fn unlock_batch(&self, resources: &[String], owner: &str) -> Result<()>;
/// Batch get read lock
async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool>;
/// Batch release read lock
async fn runlock_batch(&self, resources: &[String], owner: &str) -> Result<()>;
}
#[async_trait]
impl NamespaceLockManager for NamespaceLock {
async fn lock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For each resource, create a lock request and try to acquire using clients
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let request = LockRequest::new(&namespaced_resource, LockType::Exclusive, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
let response = self.acquire_lock(&request).await?;
if !response.success {
return Ok(false);
}
}
Ok(true)
}
async fn unlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For each resource, create a lock ID and try to release using clients
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let lock_id = LockId::new_deterministic(&namespaced_resource);
let _ = self.release_lock(&lock_id).await?;
}
Ok(())
}
async fn rlock_batch(&self, resources: &[String], owner: &str, timeout: Duration, ttl: Duration) -> Result<bool> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For each resource, create a shared lock request and try to acquire using clients
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let request = LockRequest::new(&namespaced_resource, LockType::Shared, owner)
.with_acquire_timeout(timeout)
.with_ttl(ttl);
let response = self.acquire_lock(&request).await?;
if !response.success {
return Ok(false);
}
}
Ok(true)
}
async fn runlock_batch(&self, resources: &[String], _owner: &str) -> Result<()> {
if self.clients.is_empty() {
return Err(LockError::internal("No lock clients available"));
}
// For each resource, create a lock ID and try to release using clients
for resource in resources {
let namespaced_resource = self.get_resource_key(resource);
let lock_id = LockId::new_deterministic(&namespaced_resource);
let _ = self.release_lock(&lock_id).await?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use crate::LocalClient;
use super::*;
#[tokio::test]
async fn test_namespace_lock_local() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
let resources = vec!["test1".to_string(), "test2".to_string()];
// Test batch lock
let result = ns_lock
.lock_batch(&resources, "test_owner", Duration::from_millis(100), Duration::from_secs(10))
.await;
assert!(result.is_ok());
assert!(result.unwrap());
// Test batch unlock
let result = ns_lock.unlock_batch(&resources, "test_owner").await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_connection_health() {
let local_lock = NamespaceLock::new("test-namespace".to_string());
let health = local_lock.get_health().await;
assert_eq!(health.status, crate::types::HealthStatus::Degraded); // No clients
}
#[tokio::test]
async fn test_namespace_lock_creation() {
let ns_lock = NamespaceLock::new("test-namespace".to_string());
assert_eq!(ns_lock.namespace(), "test-namespace");
}
#[tokio::test]
async fn test_namespace_lock_new_local() {
let ns_lock = NamespaceLock::with_client(Arc::new(LocalClient::new()));
assert_eq!(ns_lock.namespace(), "default");
assert_eq!(ns_lock.clients.len(), 1);
assert!(ns_lock.clients[0].is_local().await);
// Test that it can perform lock operations
let resources = vec!["test-resource".to_string()];
let result = ns_lock
.lock_batch(&resources, "test-owner", Duration::from_millis(100), Duration::from_secs(10))
.await;
assert!(result.is_ok());
assert!(result.unwrap());
}
#[tokio::test]
async fn test_namespace_lock_resource_key() {
let ns_lock = NamespaceLock::new("test-namespace".to_string());
// Test resource key generation
let resource_key = ns_lock.get_resource_key("test-resource");
assert_eq!(resource_key, "test-namespace:test-resource");
}
}
crates/lock/src/namespace_lock.rs
-306
View File
@@ -1,306 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use std::{collections::HashMap, path::Path, sync::Arc, time::Duration};
use tokio::sync::RwLock;
use uuid::Uuid;
use crate::{
LockApi,
drwmutex::{DRWMutex, Options},
lrwmutex::LRWMutex,
};
use std::io::Result;
pub type RWLockerImpl = Box<dyn RWLocker + Send + Sync>;
#[async_trait]
pub trait RWLocker {
async fn get_lock(&mut self, opts: &Options) -> Result<bool>;
async fn un_lock(&mut self) -> Result<()>;
async fn get_u_lock(&mut self, opts: &Options) -> Result<bool>;
async fn un_r_lock(&mut self) -> Result<()>;
}
#[derive(Debug)]
struct NsLock {
reference: usize,
lock: LRWMutex,
}
#[derive(Debug, Default)]
pub struct NsLockMap {
is_dist_erasure: bool,
lock_map: RwLock<HashMap<String, NsLock>>,
}
impl NsLockMap {
pub fn new(is_dist_erasure: bool) -> Self {
Self {
is_dist_erasure,
..Default::default()
}
}
async fn lock(
&mut self,
volume: &String,
path: &String,
lock_source: &str,
ops_id: &str,
read_lock: bool,
timeout: Duration,
) -> bool {
let resource = Path::new(volume).join(path).to_str().unwrap().to_string();
let mut w_lock_map = self.lock_map.write().await;
let nslk = w_lock_map.entry(resource.clone()).or_insert(NsLock {
reference: 0,
lock: LRWMutex::default(),
});
nslk.reference += 1;
let locked = if read_lock {
nslk.lock.get_r_lock(ops_id, lock_source, &timeout).await
} else {
nslk.lock.get_lock(ops_id, lock_source, &timeout).await
};
if !locked {
nslk.reference -= 1;
if nslk.reference == 0 {
w_lock_map.remove(&resource);
}
}
locked
}
async fn un_lock(&mut self, volume: &String, path: &String, read_lock: bool) {
let resource = Path::new(volume).join(path).to_str().unwrap().to_string();
let mut w_lock_map = self.lock_map.write().await;
if let Some(nslk) = w_lock_map.get_mut(&resource) {
if read_lock {
nslk.lock.un_r_lock().await;
} else {
nslk.lock.un_lock().await;
}
nslk.reference -= 1;
if nslk.reference == 0 {
w_lock_map.remove(&resource);
}
}
}
}
pub struct WrapperLocker(pub Arc<RwLock<RWLockerImpl>>);
impl Drop for WrapperLocker {
fn drop(&mut self) {
let inner = self.0.clone();
tokio::spawn(async move {
let _ = inner.write().await.un_lock().await;
});
}
}
pub async fn new_nslock(
ns: Arc<RwLock<NsLockMap>>,
owner: String,
volume: String,
paths: Vec<String>,
lockers: Vec<LockApi>,
) -> WrapperLocker {
if ns.read().await.is_dist_erasure {
let names = paths
.iter()
.map(|path| Path::new(&volume).join(path).to_str().unwrap().to_string())
.collect();
return WrapperLocker(Arc::new(RwLock::new(Box::new(DistLockInstance::new(owner, names, lockers)))));
}
WrapperLocker(Arc::new(RwLock::new(Box::new(LocalLockInstance::new(ns, volume, paths)))))
}
struct DistLockInstance {
lock: Box<DRWMutex>,
ops_id: String,
}
impl DistLockInstance {
fn new(owner: String, names: Vec<String>, lockers: Vec<LockApi>) -> Self {
let ops_id = Uuid::new_v4().to_string();
Self {
lock: Box::new(DRWMutex::new(owner, names, lockers)),
ops_id,
}
}
}
#[async_trait]
impl RWLocker for DistLockInstance {
async fn get_lock(&mut self, opts: &Options) -> Result<bool> {
let source = "".to_string();
Ok(self.lock.get_lock(&self.ops_id, &source, opts).await)
}
async fn un_lock(&mut self) -> Result<()> {
self.lock.un_lock().await;
Ok(())
}
async fn get_u_lock(&mut self, opts: &Options) -> Result<bool> {
let source = "".to_string();
Ok(self.lock.get_r_lock(&self.ops_id, &source, opts).await)
}
async fn un_r_lock(&mut self) -> Result<()> {
self.lock.un_r_lock().await;
Ok(())
}
}
struct LocalLockInstance {
ns: Arc<RwLock<NsLockMap>>,
volume: String,
paths: Vec<String>,
ops_id: String,
}
impl LocalLockInstance {
fn new(ns: Arc<RwLock<NsLockMap>>, volume: String, paths: Vec<String>) -> Self {
let ops_id = Uuid::new_v4().to_string();
Self {
ns,
volume,
paths,
ops_id,
}
}
}
#[async_trait]
impl RWLocker for LocalLockInstance {
async fn get_lock(&mut self, opts: &Options) -> Result<bool> {
let source = "".to_string();
let read_lock = false;
let mut success = vec![false; self.paths.len()];
for (idx, path) in self.paths.iter().enumerate() {
if !self
.ns
.write()
.await
.lock(&self.volume, path, &source, &self.ops_id, read_lock, opts.timeout)
.await
{
for (i, x) in success.iter().enumerate() {
if *x {
self.ns.write().await.un_lock(&self.volume, &self.paths[i], read_lock).await;
}
}
return Ok(false);
}
success[idx] = true;
}
Ok(true)
}
async fn un_lock(&mut self) -> Result<()> {
let read_lock = false;
for path in self.paths.iter() {
self.ns.write().await.un_lock(&self.volume, path, read_lock).await;
}
Ok(())
}
async fn get_u_lock(&mut self, opts: &Options) -> Result<bool> {
let source = "".to_string();
let read_lock = true;
let mut success = Vec::with_capacity(self.paths.len());
for (idx, path) in self.paths.iter().enumerate() {
if !self
.ns
.write()
.await
.lock(&self.volume, path, &source, &self.ops_id, read_lock, opts.timeout)
.await
{
for (i, x) in success.iter().enumerate() {
if *x {
self.ns.write().await.un_lock(&self.volume, &self.paths[i], read_lock).await;
}
}
return Ok(false);
}
success[idx] = true;
}
Ok(true)
}
async fn un_r_lock(&mut self) -> Result<()> {
let read_lock = true;
for path in self.paths.iter() {
self.ns.write().await.un_lock(&self.volume, path, read_lock).await;
}
Ok(())
}
}
#[cfg(test)]
mod test {
use std::{sync::Arc, time::Duration};
use std::io::Result;
use tokio::sync::RwLock;
use crate::{
drwmutex::Options,
namespace_lock::{NsLockMap, new_nslock},
};
#[tokio::test]
async fn test_local_instance() -> Result<()> {
let ns_lock_map = Arc::new(RwLock::new(NsLockMap::default()));
let ns = new_nslock(
Arc::clone(&ns_lock_map),
"local".to_string(),
"test".to_string(),
vec!["foo".to_string()],
Vec::new(),
)
.await;
let result =
ns.0.write()
.await
.get_lock(&Options {
timeout: Duration::from_secs(5),
retry_interval: Duration::from_secs(1),
})
.await?;
assert!(result);
Ok(())
}
}
crates/lock/src/remote_client.rs
-147
View File
@@ -1,147 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Locker, lock_args::LockArgs};
use async_trait::async_trait;
use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
use std::io::{Error, Result};
use tonic::Request;
use tracing::info;
#[derive(Debug, Clone)]
pub struct RemoteClient {
addr: String,
}
impl RemoteClient {
pub fn new(url: url::Url) -> Self {
let addr = format!("{}://{}:{}", url.scheme(), url.host_str().unwrap(), url.port().unwrap());
Self { addr }
}
}
#[async_trait]
impl Locker for RemoteClient {
async fn lock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote lock");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.lock(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn unlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote unlock");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.un_lock(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn rlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote rlock");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.r_lock(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn runlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote runlock");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.r_un_lock(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn refresh(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote refresh");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.refresh(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn force_unlock(&mut self, args: &LockArgs) -> Result<bool> {
info!("remote force_unlock");
let args = serde_json::to_string(args)?;
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(GenerallyLockRequest { args });
let response = client.force_un_lock(request).await.map_err(Error::other)?.into_inner();
if let Some(error_info) = response.error_info {
return Err(Error::other(error_info));
}
Ok(response.success)
}
async fn close(&self) {}
async fn is_online(&self) -> bool {
true
}
async fn is_local(&self) -> bool {
false
}
}
crates/lock/src/types.rs
+702
View File
@@ -0,0 +1,702 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use uuid::Uuid;
/// Lock type enumeration
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum LockType {
/// Exclusive lock (write lock)
Exclusive,
/// Shared lock (read lock)
Shared,
}
/// Lock status enumeration
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum LockStatus {
/// Acquired
Acquired,
/// Waiting
Waiting,
/// Released
Released,
/// Expired
Expired,
/// Force released
ForceReleased,
}
/// Lock priority
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)]
pub enum LockPriority {
Low = 1,
#[default]
Normal = 2,
High = 3,
Critical = 4,
}
/// Lock information structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockInfo {
/// Unique identifier for the lock
pub id: LockId,
/// Resource path
pub resource: String,
/// Lock type
pub lock_type: LockType,
/// Lock status
pub status: LockStatus,
/// Lock owner
pub owner: String,
/// Acquisition time
pub acquired_at: SystemTime,
/// Expiration time
pub expires_at: SystemTime,
/// Last refresh time
pub last_refreshed: SystemTime,
/// Lock metadata
pub metadata: LockMetadata,
/// Lock priority
pub priority: LockPriority,
/// Wait start time
pub wait_start_time: Option<SystemTime>,
}
impl LockInfo {
/// Check if the lock has expired
pub fn has_expired(&self) -> bool {
self.expires_at <= SystemTime::now()
}
/// Get remaining time until expiration
pub fn remaining_time(&self) -> Duration {
let now = SystemTime::now();
if self.expires_at > now {
self.expires_at.duration_since(now).unwrap_or(Duration::ZERO)
} else {
Duration::ZERO
}
}
/// Check if the lock is still valid
pub fn is_valid(&self) -> bool {
!self.has_expired() && self.status == LockStatus::Acquired
}
}
/// Lock ID type
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct LockId {
pub resource: String,
pub uuid: String,
}
impl LockId {
/// Generate new lock ID for a resource
pub fn new(resource: &str) -> Self {
Self {
resource: resource.to_string(),
uuid: Uuid::new_v4().to_string(),
}
}
/// Generate deterministic lock ID for a resource (same resource = same ID)
pub fn new_deterministic(resource: &str) -> Self {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
resource.hash(&mut hasher);
let hash = hasher.finish();
Self {
resource: resource.to_string(),
uuid: format!("{hash:016x}"),
}
}
/// Create lock ID from resource and uuid
pub fn from_parts(resource: impl Into<String>, uuid: impl Into<String>) -> Self {
Self {
resource: resource.into(),
uuid: uuid.into(),
}
}
/// Create lock ID from string (for compatibility, expects "resource:uuid")
pub fn from_string(id: impl Into<String>) -> Self {
let s = id.into();
if let Some((resource, uuid)) = s.split_once(":") {
Self {
resource: resource.to_string(),
uuid: uuid.to_string(),
}
} else {
// fallback: treat as uuid only
Self {
resource: "unknown".to_string(),
uuid: s,
}
}
}
/// Get string representation of lock ID ("resource:uuid")
pub fn as_str(&self) -> String {
format!("{}:{}", self.resource, self.uuid)
}
}
impl Default for LockId {
fn default() -> Self {
Self::new("default")
}
}
impl std::fmt::Display for LockId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}:{}", self.resource, self.uuid)
}
}
/// Lock metadata structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockMetadata {
/// Client information
pub client_info: Option<String>,
/// Operation ID
pub operation_id: Option<String>,
/// Priority (lower number = higher priority)
pub priority: Option<i32>,
/// Custom tags
pub tags: std::collections::HashMap<String, String>,
/// Creation time
pub created_at: SystemTime,
}
impl Default for LockMetadata {
fn default() -> Self {
Self {
client_info: None,
operation_id: None,
priority: None,
tags: std::collections::HashMap::new(),
created_at: SystemTime::now(),
}
}
}
impl LockMetadata {
/// Create new lock metadata
pub fn new() -> Self {
Self::default()
}
/// Set client information
pub fn with_client_info(mut self, client_info: impl Into<String>) -> Self {
self.client_info = Some(client_info.into());
self
}
/// Set operation ID
pub fn with_operation_id(mut self, operation_id: impl Into<String>) -> Self {
self.operation_id = Some(operation_id.into());
self
}
/// Set priority
pub fn with_priority(mut self, priority: i32) -> Self {
self.priority = Some(priority);
self
}
/// Add tag
pub fn with_tag(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.tags.insert(key.into(), value.into());
self
}
}
/// Lock request structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockRequest {
/// Lock ID
pub lock_id: LockId,
/// Resource path
pub resource: String,
/// Lock type
pub lock_type: LockType,
/// Lock owner
pub owner: String,
/// Acquire timeout duration (how long to wait for lock acquisition)
pub acquire_timeout: Duration,
/// Lock TTL (Time To Live - how long the lock remains valid after acquisition)
pub ttl: Duration,
/// Lock metadata
pub metadata: LockMetadata,
/// Lock priority
pub priority: LockPriority,
/// Deadlock detection
pub deadlock_detection: bool,
}
impl LockRequest {
/// Create new lock request
pub fn new(resource: impl Into<String>, lock_type: LockType, owner: impl Into<String>) -> Self {
let resource_str = resource.into();
Self {
lock_id: LockId::new_deterministic(&resource_str),
resource: resource_str,
lock_type,
owner: owner.into(),
acquire_timeout: Duration::from_secs(10), // Default 10 seconds to acquire
ttl: Duration::from_secs(30), // Default 30 seconds lock lifetime
metadata: LockMetadata::default(),
priority: LockPriority::default(),
deadlock_detection: false,
}
}
/// Set acquire timeout (how long to wait for lock acquisition)
pub fn with_acquire_timeout(mut self, timeout: Duration) -> Self {
self.acquire_timeout = timeout;
self
}
/// Set lock TTL (how long the lock remains valid after acquisition)
pub fn with_ttl(mut self, ttl: Duration) -> Self {
self.ttl = ttl;
self
}
/// Set metadata
pub fn with_metadata(mut self, metadata: LockMetadata) -> Self {
self.metadata = metadata;
self
}
/// Set priority
pub fn with_priority(mut self, priority: LockPriority) -> Self {
self.priority = priority;
self
}
/// Set deadlock detection
pub fn with_deadlock_detection(mut self, enabled: bool) -> Self {
self.deadlock_detection = enabled;
self
}
}
/// Lock response structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockResponse {
/// Whether lock acquisition was successful
pub success: bool,
/// Lock information (if successful)
pub lock_info: Option<LockInfo>,
/// Error message (if failed)
pub error: Option<String>,
/// Wait time
pub wait_time: Duration,
/// Position in wait queue
pub position_in_queue: Option<usize>,
}
impl LockResponse {
/// Create success response
pub fn success(lock_info: LockInfo, wait_time: Duration) -> Self {
Self {
success: true,
lock_info: Some(lock_info),
error: None,
wait_time,
position_in_queue: None,
}
}
/// Create failure response
pub fn failure(error: impl Into<String>, wait_time: Duration) -> Self {
Self {
success: false,
lock_info: None,
error: Some(error.into()),
wait_time,
position_in_queue: None,
}
}
/// Create waiting response
pub fn waiting(wait_time: Duration, position: usize) -> Self {
Self {
success: false,
lock_info: None,
error: None,
wait_time,
position_in_queue: Some(position),
}
}
/// Check if response indicates success
pub fn is_success(&self) -> bool {
self.success
}
/// Check if response indicates failure
pub fn is_failure(&self) -> bool {
!self.success && self.error.is_some()
}
/// Check if response indicates waiting
pub fn is_waiting(&self) -> bool {
!self.success && self.position_in_queue.is_some()
}
/// Get lock info
pub fn lock_info(&self) -> Option<&LockInfo> {
self.lock_info.as_ref()
}
}
/// Lock statistics structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LockStats {
/// Total number of locks
pub total_locks: usize,
/// Number of exclusive locks
pub exclusive_locks: usize,
/// Number of shared locks
pub shared_locks: usize,
/// Number of waiting locks
pub waiting_locks: usize,
/// Number of deadlock detections
pub deadlock_detections: usize,
/// Number of priority upgrades
pub priority_upgrades: usize,
/// Last update time
pub last_updated: SystemTime,
/// Total releases
pub total_releases: usize,
/// Total hold time
pub total_hold_time: Duration,
/// Average hold time
pub average_hold_time: Duration,
/// Total wait queues
pub total_wait_queues: usize,
/// Queue entries
pub queue_entries: usize,
/// Average wait time
pub avg_wait_time: Duration,
/// Successful acquires
pub successful_acquires: usize,
/// Failed acquires
pub failed_acquires: usize,
}
impl Default for LockStats {
fn default() -> Self {
Self {
total_locks: 0,
exclusive_locks: 0,
shared_locks: 0,
waiting_locks: 0,
deadlock_detections: 0,
priority_upgrades: 0,
last_updated: SystemTime::now(),
total_releases: 0,
total_hold_time: Duration::ZERO,
average_hold_time: Duration::ZERO,
total_wait_queues: 0,
queue_entries: 0,
avg_wait_time: Duration::ZERO,
successful_acquires: 0,
failed_acquires: 0,
}
}
}
/// Node information structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeInfo {
/// Node ID
pub id: String,
/// Node address
pub address: String,
/// Node status
pub status: NodeStatus,
/// Last heartbeat time
pub last_heartbeat: SystemTime,
/// Node weight
pub weight: f64,
}
/// Node status enumeration
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub enum NodeStatus {
/// Online
#[default]
Online,
/// Offline
Offline,
/// Degraded
Degraded,
}
/// Cluster information structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ClusterInfo {
/// Cluster ID
pub cluster_id: String,
/// List of nodes
pub nodes: Vec<NodeInfo>,
/// Quorum size
pub quorum: usize,
/// Cluster status
pub status: ClusterStatus,
/// Last update time
pub last_updated: SystemTime,
}
/// Cluster status enumeration
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub enum ClusterStatus {
/// Healthy
#[default]
Healthy,
/// Degraded
Degraded,
/// Unhealthy
Unhealthy,
}
/// Health check status
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum HealthStatus {
/// Healthy
Healthy,
/// Degraded
Degraded,
/// Unhealthy
Unhealthy,
}
/// Health check information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealthInfo {
/// Overall status
pub status: HealthStatus,
/// Node ID
pub node_id: String,
/// Last heartbeat time
pub last_heartbeat: SystemTime,
/// Connected nodes count
pub connected_nodes: usize,
/// Total nodes count
pub total_nodes: usize,
/// Lock statistics
pub lock_stats: LockStats,
/// Error message (if any)
pub error_message: Option<String>,
}
impl Default for HealthInfo {
fn default() -> Self {
Self {
status: HealthStatus::Healthy,
node_id: "unknown".to_string(),
last_heartbeat: SystemTime::now(),
connected_nodes: 1,
total_nodes: 1,
lock_stats: LockStats::default(),
error_message: None,
}
}
}
/// Timestamp type alias
pub type Timestamp = u64;
/// Get current timestamp
pub fn current_timestamp() -> Timestamp {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()
}
/// Convert timestamp to system time
pub fn timestamp_to_system_time(timestamp: Timestamp) -> SystemTime {
UNIX_EPOCH + Duration::from_secs(timestamp)
}
/// Convert system time to timestamp
pub fn system_time_to_timestamp(time: SystemTime) -> Timestamp {
time.duration_since(UNIX_EPOCH).unwrap().as_secs()
}
/// Deadlock detection result structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeadlockDetectionResult {
/// Whether deadlock exists
pub has_deadlock: bool,
/// Deadlock cycle
pub deadlock_cycle: Vec<String>,
/// Suggested resolution
pub suggested_resolution: Option<String>,
/// Affected resources
pub affected_resources: Vec<String>,
/// Affected owners
pub affected_owners: Vec<String>,
}
/// Wait graph node structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WaitGraphNode {
/// Owner
pub owner: String,
/// Resources being waited for
pub waiting_for: Vec<String>,
/// Resources currently held
pub held_resources: Vec<String>,
/// Priority
pub priority: LockPriority,
/// Wait start time
pub wait_start_time: SystemTime,
}
/// Wait queue item structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WaitQueueItem {
/// Owner
pub owner: String,
/// Lock type
pub lock_type: LockType,
/// Priority
pub priority: LockPriority,
/// Wait start time
pub wait_start_time: SystemTime,
/// Request time
pub request_time: SystemTime,
}
impl WaitQueueItem {
/// Create new wait queue item
pub fn new(owner: &str, lock_type: LockType, priority: LockPriority) -> Self {
let now = SystemTime::now();
Self {
owner: owner.to_string(),
lock_type,
priority,
wait_start_time: now,
request_time: now,
}
}
/// Get wait duration
pub fn wait_duration(&self) -> Duration {
self.wait_start_time.elapsed().unwrap_or_default()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_lock_id() {
let id1 = LockId::new("test-resource");
let id2 = LockId::new("test-resource");
assert_ne!(id1, id2);
let id3 = LockId::from_string("test-resource:test-uuid");
assert_eq!(id3.as_str(), "test-resource:test-uuid");
}
#[test]
fn test_lock_metadata() {
let metadata = LockMetadata::new()
.with_client_info("test-client")
.with_operation_id("test-op")
.with_priority(1)
.with_tag("key", "value");
assert_eq!(metadata.client_info, Some("test-client".to_string()));
assert_eq!(metadata.operation_id, Some("test-op".to_string()));
assert_eq!(metadata.priority, Some(1));
assert_eq!(metadata.tags.get("key"), Some(&"value".to_string()));
}
#[test]
fn test_lock_request() {
let request = LockRequest::new("test-resource", LockType::Exclusive, "test-owner")
.with_acquire_timeout(Duration::from_secs(60))
.with_priority(LockPriority::High)
.with_deadlock_detection(true);
assert_eq!(request.resource, "test-resource");
assert_eq!(request.lock_type, LockType::Exclusive);
assert_eq!(request.owner, "test-owner");
assert_eq!(request.acquire_timeout, Duration::from_secs(60));
assert_eq!(request.priority, LockPriority::High);
assert!(request.deadlock_detection);
}
#[test]
fn test_lock_response() {
let lock_info = LockInfo {
id: LockId::new("test-resource"),
resource: "test".to_string(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "test".to_string(),
acquired_at: SystemTime::now(),
expires_at: SystemTime::now() + Duration::from_secs(30),
last_refreshed: SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
};
let success = LockResponse::success(lock_info.clone(), Duration::ZERO);
assert!(success.is_success());
let failure = LockResponse::failure("error", Duration::ZERO);
assert!(failure.is_failure());
let waiting = LockResponse::waiting(Duration::ZERO, 1);
assert!(waiting.is_waiting());
}
#[test]
fn test_timestamp_conversion() {
let now = SystemTime::now();
let timestamp = system_time_to_timestamp(now);
let converted = timestamp_to_system_time(timestamp);
// Allow for small time differences
let diff = now.duration_since(converted).unwrap();
assert!(diff < Duration::from_secs(1));
}
#[test]
fn test_serialization() {
let request = LockRequest::new("test", LockType::Exclusive, "owner");
let serialized = serde_json::to_string(&request).unwrap();
let deserialized: LockRequest = serde_json::from_str(&serialized).unwrap();
assert_eq!(request.resource, deserialized.resource);
assert_eq!(request.lock_type, deserialized.lock_type);
assert_eq!(request.owner, deserialized.owner);
}
}