gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 22:33:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: NamespaceLock (nslock), AHM→Heal Crate, and Lock/Clippy Fixes (#1664)

Browse Source 
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: weisd <2057561+weisd@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
weisd
2026-01-30 13:13:41 +08:00
committed by GitHub
parent 1c085590ca
commit dce117840c
80 changed files with 3787 additions and 16746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/reliant/grpc_lock_client.rs
+316
View File
@@ -0,0 +1,316 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
#![allow(dead_code)]
use async_trait::async_trait;
use rustfs_ecstore::rpc::node_service_time_out_client_no_auth;
use rustfs_lock::{
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
types::{LockMetadata, LockPriority},
};
use rustfs_protos::proto_gen::node_service::{GenerallyLockRequest, PingRequest};
use tonic::Request;
use tracing::{info, warn};
/// gRPC lock client without authentication for testing
/// Similar to RemoteClient but uses no_auth client
#[derive(Debug, Clone)]
pub struct GrpcLockClient {
addr: String,
}
impl GrpcLockClient {
pub fn new(endpoint: String) -> Self {
Self { addr: endpoint }
}
async fn get_client(
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, rustfs_ecstore::rpc::TonicInterceptor>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
.await
.map_err(|err| LockError::internal(format!("can not get client, err: {err}")))
}
/// Create a minimal LockRequest for unlock operations using only lock_id
fn create_unlock_request(lock_id: &LockId) -> LockRequest {
LockRequest {
lock_id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // Type doesn't matter for unlock
owner: String::new(), // Owner not needed, server uses lock_id
acquire_timeout: std::time::Duration::from_secs(30),
ttl: std::time::Duration::from_secs(300),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
}
}
}
#[async_trait]
impl LockClient for GrpcLockClient {
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
info!("grpc acquire_lock for {}", request.resource);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
// Check for explicit error first
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
// Check if the lock acquisition was successful
if resp.success {
// Try to deserialize lock_info from response
let lock_info = if let Some(lock_info_json) = resp.lock_info {
match serde_json::from_str::<LockInfo>(&lock_info_json) {
Ok(info) => info,
Err(e) => {
// If deserialization fails, fall back to constructing from request
warn!("Failed to deserialize lock_info from response: {}, using request data", e);
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
}
}
}
} else {
// If lock_info is not provided, construct from request
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
}
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
// Lock acquisition failed
Ok(LockResponse::failure(
"Lock acquisition failed on remote server".to_string(),
std::time::Duration::ZERO,
))
}
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
info!("grpc release for {}", lock_id);
let unlock_request = Self::create_unlock_request(lock_id);
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: request_string.clone(),
});
let resp = client
.un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
info!("grpc refresh for {}", lock_id);
let refresh_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.refresh(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("grpc force_release for {}", lock_id);
let force_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.force_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(resp.success)
}
async fn check_status(&self, lock_id: &LockId) -> Result<Option<LockInfo>> {
info!("grpc check_status for {}", lock_id);
// Since there's no direct status query in the gRPC service,
// we attempt a non-blocking lock acquisition to check if the resource is available
let status_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
// Try to acquire a very short-lived lock to test availability
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
// Try exclusive lock first with very short timeout
let resp = client.lock(req).await;
match resp {
Ok(response) => {
let resp = response.into_inner();
if resp.success {
// If we successfully acquired the lock, the resource was free
// Immediately release it
let release_req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&status_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let _ = client.un_lock(release_req).await; // Best effort release
// Return None since no one was holding the lock
Ok(None)
} else {
// Lock acquisition failed, meaning someone is holding it
// We can't determine the exact details remotely, so return a generic status
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // We can't know the exact type
status: LockStatus::Acquired,
owner: "unknown".to_string(), // Remote client can't determine owner
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
}
Err(_) => {
// Communication error or lock is held
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600),
last_refreshed: std::time::SystemTime::now(),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
wait_start_time: None,
}))
}
}
}
async fn get_stats(&self) -> Result<LockStats> {
info!("grpc get_stats from {}", self.addr);
// Since there's no direct statistics endpoint in the gRPC service,
// we return basic stats indicating this is a remote client
let stats = LockStats {
last_updated: std::time::SystemTime::now(),
..Default::default()
};
Ok(stats)
}
async fn close(&self) -> Result<()> {
Ok(())
}
async fn is_online(&self) -> bool {
// Use Ping interface to test if remote service is online
let mut client = match self.get_client().await {
Ok(client) => client,
Err(_) => {
info!("grpc client {} connection failed", self.addr);
return false;
}
};
let ping_req = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::new(),
});
match client.ping(ping_req).await {
Ok(_) => {
info!("grpc client {} is online", self.addr);
true
}
Err(_) => {
info!("grpc client {} ping failed", self.addr);
false
}
}
}
async fn is_local(&self) -> bool {
false
}
}
crates/e2e_test/src/reliant/grpc_lock_server.rs
+711
View File
@@ -0,0 +1,711 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
#![allow(dead_code)]
use bytes::Bytes;
use futures::Stream;
use rustfs_lock::{LockClient, LockRequest};
use rustfs_protos::{
models::PingBodyBuilder,
proto_gen::node_service::{
GenerallyLockRequest, GenerallyLockResponse, PingRequest, PingResponse, node_service_server::NodeService,
},
};
use std::pin::Pin;
use std::sync::Arc;
use tokio::net::TcpListener;
use tokio_stream::wrappers::TcpListenerStream;
use tonic::{Request, Response, Status};
use tracing::debug;
type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send>>;
/// Minimal NodeService implementation that only supports Lock RPCs
/// Used for testing distributed lock scenarios with real gRPC
#[derive(Debug)]
pub struct MinimalLockNodeService {
lock_client: Arc<dyn LockClient>,
}
impl MinimalLockNodeService {
pub fn new(lock_client: Arc<dyn LockClient>) -> Self {
Self { lock_client }
}
}
#[tonic::async_trait]
impl NodeService for MinimalLockNodeService {
async fn ping(&self, _request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> {
debug!("MinimalLockNodeService: PING");
let mut fbb = flatbuffers::FlatBufferBuilder::new();
let payload = fbb.create_vector(b"pong");
let mut builder = PingBodyBuilder::new(&mut fbb);
builder.add_payload(payload);
let root = builder.finish();
fbb.finish(root, None);
let finished_data = fbb.finished_data();
Ok(Response::new(PingResponse {
version: 1,
body: Bytes::copy_from_slice(finished_data),
}))
}
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
}));
}
};
match self.lock_client.acquire_lock(&args).await {
Ok(result) => {
let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
Ok(Response::new(GenerallyLockResponse {
success: result.success,
error_info: None,
lock_info: lock_info_json,
}))
}
Err(err) => Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!(
"can not lock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
lock_info: None,
})),
}
}
async fn un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
}));
}
};
match self.lock_client.release(&args.lock_id).await {
Ok(success) => Ok(Response::new(GenerallyLockResponse {
success,
error_info: None,
lock_info: None,
})),
Err(err) => Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!(
"can not unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
lock_info: None,
})),
}
}
async fn force_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
}));
}
};
match self.lock_client.force_release(&args.lock_id).await {
Ok(success) => Ok(Response::new(GenerallyLockResponse {
success,
error_info: None,
lock_info: None,
})),
Err(err) => Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!(
"can not force_unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
lock_info: None,
})),
}
}
async fn refresh(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not decode args, err: {err}")),
lock_info: None,
}));
}
};
match self.lock_client.refresh(&args.lock_id).await {
Ok(success) => Ok(Response::new(GenerallyLockResponse {
success,
error_info: None,
lock_info: None,
})),
Err(err) => Ok(Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not refresh, err: {err}")),
lock_info: None,
})),
}
}
// All other methods return unimplemented
async fn heal_bucket(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::HealBucketRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::HealBucketResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn list_bucket(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ListBucketRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ListBucketResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn make_bucket(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MakeBucketRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::MakeBucketResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_bucket_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetBucketInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetBucketInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_bucket(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteBucketRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteBucketResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_all(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadAllRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadAllResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn write_all(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::WriteAllRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteAllResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn verify_file(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::VerifyFileRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::VerifyFileResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_parts(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadPartsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadPartsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn check_parts(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::CheckPartsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::CheckPartsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_part(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenamePartRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenamePartResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_file(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameFileRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameFileResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn write(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::WriteRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
type WriteStreamStream = ResponseStream<rustfs_protos::proto_gen::node_service::WriteResponse>;
async fn write_stream(
&self,
_request: Request<tonic::Streaming<rustfs_protos::proto_gen::node_service::WriteRequest>>,
) -> Result<Response<Self::WriteStreamStream>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
type ReadAtStream = ResponseStream<rustfs_protos::proto_gen::node_service::ReadAtResponse>;
async fn read_at(
&self,
_request: Request<tonic::Streaming<rustfs_protos::proto_gen::node_service::ReadAtRequest>>,
) -> Result<Response<Self::ReadAtStream>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn list_dir(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ListDirRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ListDirResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
type WalkDirStream = ResponseStream<rustfs_protos::proto_gen::node_service::WalkDirResponse>;
async fn walk_dir(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::WalkDirRequest>,
) -> Result<Response<Self::WalkDirStream>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn rename_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn make_volumes(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MakeVolumesRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::MakeVolumesResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn make_volume(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::MakeVolumeRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::MakeVolumeResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn list_volumes(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ListVolumesRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ListVolumesResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn stat_volume(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::StatVolumeRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::StatVolumeResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_paths(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeletePathsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeletePathsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn update_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::UpdateMetadataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::UpdateMetadataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadMetadataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadMetadataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn write_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::WriteMetadataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteMetadataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_version(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadVersionRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadVersionResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_xl(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadXlRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadXlResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_version(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVersionResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_versions(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVersionsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn read_multiple(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReadMultipleRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReadMultipleResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_volume(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteVolumeRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVolumeResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn disk_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DiskInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DiskInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn local_storage_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LocalStorageInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LocalStorageInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn server_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ServerInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ServerInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_cpus(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetCpusRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetCpusResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_net_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetNetInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetNetInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_partitions(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetPartitionsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetPartitionsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_os_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetOsInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetOsInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_se_linux_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetSeLinuxInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetSeLinuxInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_sys_config(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetSysConfigRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetSysConfigResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_sys_errors(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetSysErrorsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetSysErrorsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_mem_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetMemInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetMemInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_proc_info(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetProcInfoRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetProcInfoResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_bucket_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadBucketMetadataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadBucketMetadataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_policy(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadPolicyRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadPolicyResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_group(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadGroupRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadGroupResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_policy_mapping(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadPolicyMappingRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadPolicyMappingResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_rebalance_meta(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadRebalanceMetaRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadRebalanceMetaResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_metrics(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetMetricsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetMetricsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn start_profiling(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::StartProfilingRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::StartProfilingResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn download_profile_data(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DownloadProfileDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DownloadProfileDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_bucket_stats(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetBucketStatsDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetBucketStatsDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_sr_metrics(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetSrMetricsDataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetSrMetricsDataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_all_bucket_stats(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetAllBucketStatsRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetAllBucketStatsResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_bucket_metadata(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteBucketMetadataRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteBucketMetadataResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_policy(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeletePolicyRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeletePolicyResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_user(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteUserRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteUserResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn delete_service_account(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::DeleteServiceAccountRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteServiceAccountResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_user(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadUserRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadUserResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_service_account(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadServiceAccountRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadServiceAccountResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn reload_site_replication_config(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReloadSiteReplicationConfigRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReloadSiteReplicationConfigResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn signal_service(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::SignalServiceRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::SignalServiceResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn background_heal_status(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::BackgroundHealStatusResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn get_metacache_listing(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::GetMetacacheListingRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::GetMetacacheListingResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn update_metacache_listing(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::UpdateMetacacheListingRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::UpdateMetacacheListingResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn reload_pool_meta(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::ReloadPoolMetaRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::ReloadPoolMetaResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn stop_rebalance(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::StopRebalanceRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::StopRebalanceResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn load_transition_tier_config(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::LoadTransitionTierConfigRequest>,
) -> Result<Response<rustfs_protos::proto_gen::node_service::LoadTransitionTierConfigResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
}
/// Spawn a gRPC lock server on a random port
/// Returns the address and a shutdown handle
pub async fn spawn_lock_server(
lock_client: Arc<dyn LockClient>,
) -> std::result::Result<(String, tokio::task::JoinHandle<()>), Box<dyn std::error::Error>> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
let addr_str = format!("http://127.0.0.1:{}", addr.port());
let service = MinimalLockNodeService::new(lock_client);
let server = tonic::transport::Server::builder()
.add_service(rustfs_protos::proto_gen::node_service::node_service_server::NodeServiceServer::new(
service,
))
.serve_with_incoming(TcpListenerStream::new(listener));
let handle = tokio::spawn(async move {
if let Err(e) = server.await {
eprintln!("gRPC server error: {}", e);
}
});
Ok((addr_str, handle))
}
crates/e2e_test/src/reliant/lock.rs
+91 -753
View File
@@ -13,780 +13,118 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use async_trait::async_trait;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::rpc::RemoteClient;
use rustfs_lock::client::{LockClient, local::LocalClient};
use rustfs_lock::types::{LockInfo, LockResponse, LockStats};
use rustfs_lock::{LockId, LockMetadata, LockPriority, LockType};
use rustfs_lock::{LockRequest, NamespaceLock, NamespaceLockManager};
use rustfs_protos::proto_gen::node_service::GenerallyLockRequest;
use serial_test::serial;
use std::{collections::HashMap, error::Error, sync::Arc, time::Duration};
use tokio::time::sleep;
use tonic::Request;
use url::Url;
const CLUSTER_ADDR: &str = "http://localhost:9000";
fn get_cluster_endpoints() -> Vec<Endpoint> {
vec![Endpoint {
url: Url::parse(CLUSTER_ADDR).unwrap(),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
}]
}
async fn create_unique_clients(endpoints: &[Endpoint]) -> Result<Vec<Arc<dyn LockClient>>, Box<dyn Error>> {
let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
for endpoint in endpoints {
if endpoint.is_local {
unique_endpoints.insert("local".to_string(), endpoint);
} else {
let host_port = format!(
"{}:{}",
endpoint.url.host_str().unwrap_or("localhost"),
endpoint.url.port().unwrap_or(9000)
);
unique_endpoints.insert(host_port, endpoint);
}
}
let mut clients = Vec::new();
for (_key, endpoint) in unique_endpoints {
if endpoint.is_local {
clients.push(Arc::new(LocalClient::new()) as Arc<dyn LockClient>);
} else {
clients.push(Arc::new(RemoteClient::new(endpoint.url.to_string())) as Arc<dyn LockClient>);
}
}
Ok(clients)
}
use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
use rustfs_lock::{GlobalLockManager, NamespaceLock, ObjectKey, client::local::LocalClient};
use std::sync::Arc;
use std::time::Duration;
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_guard_drop_releases_exclusive_lock_local() -> Result<(), Box<dyn Error>> {
// Single local client; no external server required
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let ns_lock = NamespaceLock::with_clients("e2e_guard_local".to_string(), vec![client]);
async fn test_distributed_lock_4_nodes_grpc() {
// Spawn 4 gRPC lock servers, each with its own GlobalLockManager
let manager1 = Arc::new(GlobalLockManager::new());
let manager2 = Arc::new(GlobalLockManager::new());
let manager3 = Arc::new(GlobalLockManager::new());
let manager4 = Arc::new(GlobalLockManager::new());
// Acquire exclusive guard
let g1 = ns_lock
.lock_guard("guard_exclusive", "owner1", Duration::from_millis(100), Duration::from_secs(5))
.await?;
assert!(g1.is_some(), "first guard acquisition should succeed");
let client1: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager1));
let client2: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager2));
let client3: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager3));
let client4: Arc<dyn rustfs_lock::LockClient> = Arc::new(LocalClient::with_manager(manager4));
// While g1 is alive, second exclusive acquisition should fail
let g2 = ns_lock
.lock_guard("guard_exclusive", "owner2", Duration::from_millis(50), Duration::from_secs(5))
.await?;
assert!(g2.is_none(), "second guard acquisition should fail while first is held");
// Spawn 4 gRPC servers on random ports
let (addr1, handle1) = spawn_lock_server(client1).await.expect("Failed to spawn server 1");
let (addr2, handle2) = spawn_lock_server(client2).await.expect("Failed to spawn server 2");
let (addr3, handle3) = spawn_lock_server(client3).await.expect("Failed to spawn server 3");
let (addr4, handle4) = spawn_lock_server(client4).await.expect("Failed to spawn server 4");
// Drop first guard to trigger background release
drop(g1);
// Give the background unlock worker a short moment to process
sleep(Duration::from_millis(80)).await;
// Give servers a moment to start
tokio::time::sleep(Duration::from_millis(100)).await;
// Now acquisition should succeed
let g3 = ns_lock
.lock_guard("guard_exclusive", "owner2", Duration::from_millis(100), Duration::from_secs(5))
.await?;
assert!(g3.is_some(), "acquisition should succeed after guard drop releases the lock");
drop(g3);
// Create 4 gRPC clients (no auth)
let grpc_client1: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr1));
let grpc_client2: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr2));
let grpc_client3: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr3));
let grpc_client4: Arc<dyn rustfs_lock::LockClient> = Arc::new(GrpcLockClient::new(addr4));
Ok(())
}
let clients = vec![grpc_client1, grpc_client2, grpc_client3, grpc_client4];
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_guard_shared_then_write_after_drop() -> Result<(), Box<dyn Error>> {
// Two shared read guards should coexist; write should be blocked until they drop
let client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let ns_lock = NamespaceLock::with_clients("e2e_guard_rw".to_string(), vec![client]);
// Create NamespaceLock with 4 clients and quorum=3
let lock = NamespaceLock::with_clients_and_quorum("grpc-4-node".to_string(), clients, 3);
assert_eq!(lock.namespace(), "grpc-4-node");
// Acquire two read guards
let r1 = ns_lock
.rlock_guard("rw_resource", "reader1", Duration::from_millis(100), Duration::from_secs(5))
.await?;
let r2 = ns_lock
.rlock_guard("rw_resource", "reader2", Duration::from_millis(100), Duration::from_secs(5))
.await?;
assert!(r1.is_some() && r2.is_some(), "both read guards should be acquired");
// Attempt write while readers hold the lock should fail
let w_fail = ns_lock
.lock_guard("rw_resource", "writer", Duration::from_millis(50), Duration::from_secs(5))
.await?;
assert!(w_fail.is_none(), "write should be blocked when read guards are active");
// Drop read guards to release
drop(r1);
drop(r2);
sleep(Duration::from_millis(80)).await;
// Now write should succeed
let w_ok = ns_lock
.lock_guard("rw_resource", "writer", Duration::from_millis(150), Duration::from_secs(5))
.await?;
assert!(w_ok.is_some(), "write should succeed after read guards are dropped");
drop(w_ok);
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("dandan"),
resource: "dandan".to_string(),
lock_type: LockType::Exclusive,
owner: "dd".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
let resource = ObjectKey {
bucket: Arc::from("test-bucket"),
object: Arc::from("test-object"),
version: None,
};
let args = serde_json::to_string(&args)?;
let mut client = RemoteClient::new(CLUSTER_ADDR.to_string()).get_client().await?;
println!("got client");
let request = Request::new(GenerallyLockRequest { args: args.clone() });
// Test 1: Owner A acquires write lock successfully
let mut guard_a = lock
.get_write_lock(resource.clone(), "owner-a", Duration::from_secs(5))
.await
.expect("Owner A should acquire write lock");
println!("start request");
let response = client.lock(request).await?.into_inner();
println!("request ended");
if let Some(error_info) = response.error_info {
panic!("can not get lock: {error_info}");
}
let request = Request::new(GenerallyLockRequest { args });
let response = client.un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get un_lock: {error_info}");
}
Ok(())
}
/// Mock client that simulates remote node failures
#[derive(Debug)]
struct FailingMockClient {
local_client: Arc<dyn LockClient>,
should_fail_acquire: bool,
should_fail_release: bool,
}
impl FailingMockClient {
fn new(should_fail_acquire: bool, should_fail_release: bool) -> Self {
Self {
local_client: Arc::new(LocalClient::new()),
should_fail_acquire,
should_fail_release,
// Verify it's a Standard guard (DistributedLock path)
match &guard_a {
rustfs_lock::NamespaceLockGuard::Standard(_) => {
// Expected for distributed lock
}
rustfs_lock::NamespaceLockGuard::Fast(_) => {
panic!("Expected Standard guard for distributed lock");
}
}
}
#[async_trait]
impl LockClient for FailingMockClient {
async fn acquire_exclusive(&self, request: &LockRequest) -> rustfs_lock::error::Result<LockResponse> {
if self.should_fail_acquire {
// Simulate network timeout or remote node failure
return Ok(LockResponse::failure("Simulated remote node failure", Duration::from_millis(100)));
}
self.local_client.acquire_exclusive(request).await
}
async fn acquire_shared(&self, request: &LockRequest) -> rustfs_lock::error::Result<LockResponse> {
if self.should_fail_acquire {
return Ok(LockResponse::failure("Simulated remote node failure", Duration::from_millis(100)));
}
self.local_client.acquire_shared(request).await
}
async fn release(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
if self.should_fail_release {
return Err(rustfs_lock::error::LockError::internal("Simulated release failure"));
}
self.local_client.release(lock_id).await
}
async fn refresh(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
self.local_client.refresh(lock_id).await
}
async fn force_release(&self, lock_id: &LockId) -> rustfs_lock::error::Result<bool> {
self.local_client.force_release(lock_id).await
}
async fn check_status(&self, lock_id: &LockId) -> rustfs_lock::error::Result<Option<LockInfo>> {
self.local_client.check_status(lock_id).await
}
async fn get_stats(&self) -> rustfs_lock::error::Result<LockStats> {
self.local_client.get_stats().await
}
async fn close(&self) -> rustfs_lock::error::Result<()> {
self.local_client.close().await
}
async fn is_online(&self) -> bool {
if self.should_fail_acquire {
return false; // Simulate offline node
}
true // Simulate online node
}
async fn is_local(&self) -> bool {
false // Simulate remote client
}
}
#[tokio::test]
#[serial]
async fn test_transactional_lock_with_remote_failure() -> Result<(), Box<dyn Error>> {
println!("🧪 Testing transactional lock with simulated remote node failure");
// Create a two-node cluster: one local (success) + one remote (failure)
let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
let clients = vec![local_client, failing_remote_client];
let ns_lock = NamespaceLock::with_clients("test_transactional".to_string(), clients);
let resource = "critical_resource".to_string();
// Test single lock operation with 2PC
println!("📝 Testing single lock with remote failure...");
let request = LockRequest::new(&resource, LockType::Exclusive, "test_owner").with_ttl(Duration::from_secs(30));
let response = ns_lock.acquire_lock(&request).await?;
// Should fail because quorum (2/2) is not met due to remote failure
assert!(!response.success, "Lock should fail due to remote node failure");
println!("✅ Single lock correctly failed due to remote node failure");
// Verify no locks are left behind on the local node
let local_client_direct = LocalClient::new();
let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(&resource));
let lock_status = local_client_direct.check_status(&lock_id).await?;
assert!(lock_status.is_none(), "No lock should remain on local node after rollback");
println!("✅ Verified rollback: no locks left on local node");
Ok(())
}
#[tokio::test]
#[serial]
async fn test_transactional_batch_lock_with_mixed_failures() -> Result<(), Box<dyn Error>> {
println!("🧪 Testing transactional batch lock with mixed node failures");
// Create a cluster with different failure patterns
let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
let clients = vec![local_client, failing_remote_client];
let ns_lock = NamespaceLock::with_clients("test_batch_transactional".to_string(), clients);
let resources = vec!["resource_1".to_string(), "resource_2".to_string(), "resource_3".to_string()];
println!("📝 Testing batch lock with remote failure...");
let result = ns_lock
.lock_batch(&resources, "batch_owner", Duration::from_millis(100), Duration::from_secs(30))
.await?;
// Should fail because remote node cannot acquire locks
assert!(!result, "Batch lock should fail due to remote node failure");
println!("✅ Batch lock correctly failed due to remote node failure");
// Verify no locks are left behind on any resource
let local_client_direct = LocalClient::new();
for resource in &resources {
let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(resource));
let lock_status = local_client_direct.check_status(&lock_id).await?;
assert!(lock_status.is_none(), "No lock should remain for resource: {resource}");
}
println!("✅ Verified rollback: no locks left on any resource");
Ok(())
}
#[tokio::test]
#[serial]
async fn test_transactional_lock_with_quorum_success() -> Result<(), Box<dyn Error>> {
println!("🧪 Testing transactional lock with quorum success");
// Create a three-node cluster where 2 succeed and 1 fails (quorum = 2 automatically)
let local_client1: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let local_client2: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let failing_remote_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(true, false));
let clients = vec![local_client1, local_client2, failing_remote_client];
let ns_lock = NamespaceLock::with_clients("test_quorum".to_string(), clients);
let resource = "quorum_resource".to_string();
println!("📝 Testing lock with automatic quorum=2, 2 success + 1 failure...");
let request = LockRequest::new(&resource, LockType::Exclusive, "quorum_owner").with_ttl(Duration::from_secs(30));
let response = ns_lock.acquire_lock(&request).await?;
// Should fail because we require all nodes to succeed for consistency
// (even though quorum is met, the implementation requires all nodes for consistency)
assert!(!response.success, "Lock should fail due to consistency requirement");
println!("✅ Lock correctly failed due to consistency requirement (partial success rolled back)");
Ok(())
}
#[tokio::test]
#[serial]
async fn test_transactional_lock_rollback_on_release_failure() -> Result<(), Box<dyn Error>> {
println!("🧪 Testing rollback behavior when release fails");
// Create clients where acquire succeeds but release fails
let local_client: Arc<dyn LockClient> = Arc::new(LocalClient::new());
let failing_release_client: Arc<dyn LockClient> = Arc::new(FailingMockClient::new(false, true));
let clients = vec![local_client, failing_release_client];
let ns_lock = NamespaceLock::with_clients("test_release_failure".to_string(), clients);
let resource = "release_test_resource".to_string();
println!("📝 Testing lock acquisition with release failure handling...");
let request = LockRequest::new(&resource, LockType::Exclusive, "test_owner").with_ttl(Duration::from_secs(30));
// This should fail because both LocalClient instances share the same global lock map
// The first client (LocalClient) will acquire the lock, but the second client
// (FailingMockClient's internal LocalClient) will fail to acquire the same resource
let response = ns_lock.acquire_lock(&request).await?;
// The operation should fail due to lock contention between the two LocalClient instances
assert!(
!response.success,
"Lock should fail due to lock contention between LocalClient instances sharing global lock map"
);
println!("✅ Lock correctly failed due to lock contention (both clients use same global lock map)");
// Verify no locks are left behind after rollback
let local_client_direct = LocalClient::new();
let lock_id = LockId::new_deterministic(&ns_lock.get_resource_key(&resource));
let lock_status = local_client_direct.check_status(&lock_id).await?;
assert!(lock_status.is_none(), "No lock should remain after rollback");
println!("✅ Verified rollback: no locks left after failed acquisition");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
let resources = vec!["foo".to_string()];
let result = ns_lock
.lock_batch(&resources, "dandan", Duration::from_secs(5), Duration::from_secs(10))
// Test 2: Owner B tries to acquire write lock while A holds it - should fail
// Since all 4 backends are holding locks from owner-a, owner-b cannot acquire on any backend
// This means 0 successes < quorum(3), so acquisition should fail
let result_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_millis(100))
.await;
match &result {
Ok(success) => println!("Lock result: {success}"),
Err(e) => println!("Lock error: {e}"),
}
let result = result?;
assert!(result, "Lock should succeed, but got: {result}");
ns_lock.unlock_batch(&resources, "dandan").await?;
Ok(())
}
assert!(result_b.is_err(), "Owner B should fail to acquire lock while owner A holds it");
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_concurrent_lock_attempts() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test".to_string(), clients);
let resource = vec!["concurrent_resource".to_string()];
// First lock should succeed
println!("Attempting first lock...");
let result1 = ns_lock
.lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
println!("First lock result: {result1}");
assert!(result1, "First lock should succeed");
// Second lock should fail (resource already locked)
println!("Attempting second lock...");
let result2 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(1), Duration::from_secs(10))
.await?;
println!("Second lock result: {result2}");
assert!(!result2, "Second lock should fail");
// Unlock by first owner
println!("Unlocking first lock...");
ns_lock.unlock_batch(&resource, "owner1").await?;
println!("First lock unlocked");
// Now second owner should be able to lock
println!("Attempting third lock...");
let result3 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
println!("Third lock result: {result3}");
assert!(result3, "Lock should succeed after unlock");
// Clean up
println!("Cleaning up...");
ns_lock.unlock_batch(&resource, "owner2").await?;
println!("Test completed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_read_write_lock_compatibility() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_rw".to_string(), clients);
let resource = vec!["rw_resource".to_string()];
// First read lock should succeed
let result1 = ns_lock
.rlock_batch(&resource, "reader1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result1, "First read lock should succeed");
// Second read lock should also succeed (read locks are compatible)
let result2 = ns_lock
.rlock_batch(&resource, "reader2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result2, "Second read lock should succeed");
// Write lock should fail (read locks are held)
let result3 = ns_lock
.lock_batch(&resource, "writer1", Duration::from_secs(1), Duration::from_secs(10))
.await?;
assert!(!result3, "Write lock should fail when read locks are held");
// Release read locks
ns_lock.runlock_batch(&resource, "reader1").await?;
ns_lock.runlock_batch(&resource, "reader2").await?;
// Now write lock should succeed
let result4 = ns_lock
.lock_batch(&resource, "writer1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result4, "Write lock should succeed after read locks released");
// Clean up
ns_lock.unlock_batch(&resource, "writer1").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_timeout() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_timeout".to_string(), clients);
let resource = vec!["timeout_resource".to_string()];
// First lock with short timeout
let result1 = ns_lock
.lock_batch(&resource, "owner1", Duration::from_secs(2), Duration::from_secs(1))
.await?;
assert!(result1, "First lock should succeed");
// Wait for lock to expire
sleep(Duration::from_secs(5)).await;
// Second lock should succeed after timeout
let result2 = ns_lock
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(1))
.await?;
assert!(result2, "Lock should succeed after timeout");
// Clean up
ns_lock.unlock_batch(&resource, "owner2").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_batch_lock_operations() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock = NamespaceLock::with_clients("test_batch".to_string(), clients);
let resources = vec![
"batch_resource1".to_string(),
"batch_resource2".to_string(),
"batch_resource3".to_string(),
];
// Lock all resources
let result = ns_lock
.lock_batch(&resources, "batch_owner", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result, "Batch lock should succeed");
// Try to lock one of the resources with different owner - should fail
let single_resource = vec!["batch_resource2".to_string()];
let result2 = ns_lock
.lock_batch(&single_resource, "other_owner", Duration::from_secs(1), Duration::from_secs(10))
.await?;
assert!(!result2, "Lock should fail for already locked resource");
// Unlock all resources
ns_lock.unlock_batch(&resources, "batch_owner").await?;
// Now should be able to lock single resource
let result3 = ns_lock
.lock_batch(&single_resource, "other_owner", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result3, "Lock should succeed after batch unlock");
// Clean up
ns_lock.unlock_batch(&single_resource, "other_owner").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_multiple_namespaces() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock1 = NamespaceLock::with_clients("namespace1".to_string(), clients.clone());
let ns_lock2 = NamespaceLock::with_clients("namespace2".to_string(), clients);
let resource = vec!["shared_resource".to_string()];
// Lock same resource in different namespaces - both should succeed
let result1 = ns_lock1
.lock_batch(&resource, "owner1", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result1, "Lock in namespace1 should succeed");
let result2 = ns_lock2
.lock_batch(&resource, "owner2", Duration::from_secs(5), Duration::from_secs(10))
.await?;
assert!(result2, "Lock in namespace2 should succeed");
// Clean up
ns_lock1.unlock_batch(&resource, "owner1").await?;
ns_lock2.unlock_batch(&resource, "owner2").await?;
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_rpc_read_lock() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("read_resource"),
resource: "read_resource".to_string(),
lock_type: LockType::Shared,
owner: "reader1".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = RemoteClient::new(CLUSTER_ADDR.to_string()).get_client().await?;
// First read lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.r_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get read lock: {error_info}");
// Verify the error is a timeout or quorum failure
if let Err(err) = result_b {
let err_str = err.to_string().to_lowercase();
assert!(
err_str.contains("timeout") || err_str.contains("quorum") || err_str.contains("not reached"),
"Error should be timeout or quorum related, got: {}",
err
);
}
// Second read lock with different owner should also succeed
let args2 = LockRequest {
lock_id: LockId::new_deterministic("read_resource"),
resource: "read_resource".to_string(),
lock_type: LockType::Shared,
owner: "reader2".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args2_str = serde_json::to_string(&args2)?;
let request2 = Request::new(GenerallyLockRequest { args: args2_str });
let response2 = client.r_lock(request2).await?.into_inner();
if let Some(error_info) = response2.error_info {
panic!("can not get second read lock: {error_info}");
// Test 3: Release owner A's lock
assert!(guard_a.release(), "Should release guard_a successfully");
assert!(guard_a.is_released(), "Guard A should be marked as released");
// Test 4: Owner B should now be able to acquire the lock
let guard_b = lock
.get_write_lock(resource.clone(), "owner-b", Duration::from_secs(5))
.await
.expect("Owner B should acquire write lock after A releases");
match &guard_b {
rustfs_lock::NamespaceLockGuard::Standard(_) => {
// Expected for distributed lock
}
rustfs_lock::NamespaceLockGuard::Fast(_) => {
panic!("Expected Standard guard for distributed lock");
}
}
// Unlock both
let request = Request::new(GenerallyLockRequest { args: args_str });
let response = client.r_un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not unlock read lock: {error_info}");
}
// Test 5: Verify health check shows 4 nodes
let health = lock.get_health().await;
assert_eq!(health.node_id, "grpc-4-node");
assert_eq!(health.total_nodes, 4);
assert_eq!(health.connected_nodes, 4);
assert_eq!(health.status, rustfs_lock::types::HealthStatus::Healthy);
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_lock_refresh() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("refresh_resource"),
resource: "refresh_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "refresh_owner".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = RemoteClient::new(CLUSTER_ADDR.to_string()).get_client().await?;
// Acquire lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get lock: {error_info}");
}
// Refresh lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.refresh(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not refresh lock: {error_info}");
}
assert!(response.success, "Lock refresh should succeed");
// Unlock
let request = Request::new(GenerallyLockRequest { args: args_str });
let response = client.un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not unlock: {error_info}");
}
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_force_unlock() -> Result<(), Box<dyn Error>> {
let args = LockRequest {
lock_id: LockId::new_deterministic("force_resource"),
resource: "force_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "force_owner".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let args_str = serde_json::to_string(&args)?;
let mut client = RemoteClient::new(CLUSTER_ADDR.to_string()).get_client().await?;
// Acquire lock
let request = Request::new(GenerallyLockRequest { args: args_str.clone() });
let response = client.lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not get lock: {error_info}");
}
// Force unlock (even by different owner)
let force_args = LockRequest {
lock_id: LockId::new_deterministic("force_resource"),
resource: "force_resource".to_string(),
lock_type: LockType::Exclusive,
owner: "admin".to_string(),
acquire_timeout: Duration::from_secs(30),
ttl: Duration::from_secs(30),
metadata: LockMetadata::default(),
priority: LockPriority::Normal,
deadlock_detection: false,
};
let force_args_str = serde_json::to_string(&force_args)?;
let request = Request::new(GenerallyLockRequest { args: force_args_str });
let response = client.force_un_lock(request).await?.into_inner();
if let Some(error_info) = response.error_info {
panic!("can not force unlock: {error_info}");
}
assert!(response.success, "Force unlock should succeed");
Ok(())
}
#[tokio::test]
#[serial]
#[ignore = "requires running RustFS server at localhost:9000"]
async fn test_global_lock_map_sharing() -> Result<(), Box<dyn Error>> {
let endpoints = get_cluster_endpoints();
let clients = create_unique_clients(&endpoints).await?;
let ns_lock1 = NamespaceLock::with_clients("global_test".to_string(), clients.clone());
let ns_lock2 = NamespaceLock::with_clients("global_test".to_string(), clients);
let resource = vec!["global_test_resource".to_string()];
// First instance acquires lock
println!("First lock map attempting to acquire lock...");
let result1 = ns_lock1
.lock_batch(&resource, "owner1", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
.await?;
println!("First lock result: {result1}");
assert!(result1, "First lock should succeed");
// Second instance should fail to acquire the same lock
println!("Second lock map attempting to acquire lock...");
let result2 = ns_lock2
.lock_batch(&resource, "owner2", std::time::Duration::from_secs(1), std::time::Duration::from_secs(10))
.await?;
println!("Second lock result: {result2}");
assert!(!result2, "Second lock should fail because resource is already locked");
// Release lock from first instance
println!("First lock map releasing lock...");
ns_lock1.unlock_batch(&resource, "owner1").await?;
// Now second instance should be able to acquire lock
println!("Second lock map attempting to acquire lock again...");
let result3 = ns_lock2
.lock_batch(&resource, "owner2", std::time::Duration::from_secs(5), std::time::Duration::from_secs(10))
.await?;
println!("Third lock result: {result3}");
assert!(result3, "Lock should succeed after first lock is released");
// Clean up
ns_lock2.unlock_batch(&resource, "owner2").await?;
Ok(())
// Cleanup
drop(guard_b);
// Shutdown servers
handle1.abort();
handle2.abort();
handle3.abort();
handle4.abort();
}
crates/e2e_test/src/reliant/mod.rs
+2
View File
@@ -14,6 +14,8 @@
mod conditional_writes;
mod get_deleted_object_test;
mod grpc_lock_client;
mod grpc_lock_server;
mod head_deleted_object_versioning_test;
mod lifecycle;
mod lock;