gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-24 05:06:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(lock): unify NamespaceLock client model and LockRequest API

Browse Source 
- Refactor NamespaceLock to use a unified client vector and quorum mechanism,
  removing legacy local/distributed lock split and related code.
- Update LockRequest to split timeout into acquire_timeout and ttl, and add
  builder methods for both.
- Adjust all batch lock APIs to accept ttl and use new LockRequest fields.
- Update all affected tests and documentation for the new API.

Signed-off-by: junxiang Mu <1948535941@qq.com>
This commit is contained in:
junxiang Mu
2025-07-23 16:00:46 +08:00
parent dc44cde081
commit cad005bc21
20 changed files with 1729 additions and 2668 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/error.rs
+6
View File
@@ -183,6 +183,9 @@ pub enum StorageError {
#[error("Io error: {0}")]
Io(std::io::Error),
#[error("Lock error: {0}")]
Lock(#[from] rustfs_lock::LockError),
}
impl StorageError {
@@ -409,6 +412,7 @@ impl Clone for StorageError {
StorageError::FirstDiskWait => StorageError::FirstDiskWait,
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
StorageError::NoHealRequired => StorageError::NoHealRequired,
StorageError::Lock(e) => StorageError::Lock(e.clone()),
}
}
}
@@ -471,6 +475,7 @@ impl StorageError {
StorageError::ConfigNotFound => 0x35,
StorageError::TooManyOpenFiles => 0x36,
StorageError::NoHealRequired => 0x37,
StorageError::Lock(_) => 0x38,
}
}
@@ -535,6 +540,7 @@ impl StorageError {
0x35 => Some(StorageError::ConfigNotFound),
0x36 => Some(StorageError::TooManyOpenFiles),
0x37 => Some(StorageError::NoHealRequired),
0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
_ => None,
}
}
crates/ecstore/src/lib.rs
+1
View File
@@ -30,6 +30,7 @@ pub mod erasure_coding;
pub mod error;
pub mod global;
pub mod heal;
pub mod lock_utils;
pub mod metrics_realtime;
pub mod notification_sys;
pub mod pools;
crates/ecstore/src/rpc/mod.rs
+1 -1
View File
@@ -22,4 +22,4 @@ pub use http_auth::{build_auth_headers, verify_rpc_signature};
pub use peer_rest_client::PeerRestClient;
pub use peer_s3_client::{LocalPeerS3Client, PeerS3Client, RemotePeerS3Client, S3PeerSys};
pub use remote_disk::RemoteDisk;
pub use tonic_service::{make_server, NodeService};
pub use tonic_service::{NodeService, make_server};
crates/ecstore/src/rpc/tonic_service.rs
+37 -79
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::{collections::HashMap, io::Cursor, pin::Pin};
use std::{collections::HashMap, io::Cursor, pin::Pin, sync::Arc};
// use common::error::Error as EcsError;
use crate::{
@@ -36,6 +36,7 @@ use futures::{Stream, StreamExt};
use futures_util::future::join_all;
use rustfs_common::globals::GLOBAL_Local_Node_Name;
use rustfs_lock::{LockClient, LockRequest};
use bytes::Bytes;
use rmp_serde::{Deserializer, Serializer};
@@ -80,12 +81,12 @@ type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, tonic::Status>> + S
#[derive(Debug)]
pub struct NodeService {
local_peer: LocalPeerS3Client,
lock_manager: rustfs_lock::NsLockMap,
lock_manager: Arc<rustfs_lock::LocalClient>,
}
pub fn make_server() -> NodeService {
let local_peer = LocalPeerS3Client::new(None, None);
let lock_manager = rustfs_lock::NsLockMap::new(false, None);
let lock_manager = Arc::new(rustfs_lock::LocalClient::new());
NodeService {
local_peer,
lock_manager,
@@ -1531,7 +1532,7 @@ impl Node for NodeService {
async fn lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
// Parse the request to extract resource and owner
let args: serde_json::Value = match serde_json::from_str(&request.args) {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
@@ -1541,35 +1542,24 @@ impl Node for NodeService {
}
};
let resource = args["resources"][0].as_str().unwrap_or("");
let owner = args["owner"].as_str().unwrap_or("");
if resource.is_empty() {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some("No resource specified".to_string()),
}));
}
match self
.lock_manager
.lock_batch_with_ttl(&[resource.to_string()], owner, std::time::Duration::from_secs(30), Some(std::time::Duration::from_secs(30)))
.await
{
match self.lock_manager.acquire_exclusive(&args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
success: result.success,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not lock, resource: {resource}, owner: {owner}, err: {err}")),
error_info: Some(format!(
"can not lock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: serde_json::Value = match serde_json::from_str(&request.args) {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
@@ -1579,31 +1569,24 @@ impl Node for NodeService {
}
};
let resource = args["resources"][0].as_str().unwrap_or("");
let owner = args["owner"].as_str().unwrap_or("");
if resource.is_empty() {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some("No resource specified".to_string()),
}));
}
match self.lock_manager.unlock_batch(&[resource.to_string()], owner).await {
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not unlock, resource: {resource}, owner: {owner}, err: {err}")),
error_info: Some(format!(
"can not unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn r_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: serde_json::Value = match serde_json::from_str(&request.args) {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
@@ -1613,35 +1596,24 @@ impl Node for NodeService {
}
};
let resource = args["resources"][0].as_str().unwrap_or("");
let owner = args["owner"].as_str().unwrap_or("");
if resource.is_empty() {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some("No resource specified".to_string()),
}));
}
match self
.lock_manager
.rlock_batch_with_ttl(&[resource.to_string()], owner, std::time::Duration::from_secs(30), Some(std::time::Duration::from_secs(30)))
.await
{
match self.lock_manager.acquire_shared(&args).await {
Ok(result) => Ok(tonic::Response::new(GenerallyLockResponse {
success: result,
success: result.success,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not rlock, resource: {resource}, owner: {owner}, err: {err}")),
error_info: Some(format!(
"can not rlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn r_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: serde_json::Value = match serde_json::from_str(&request.args) {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
@@ -1651,31 +1623,24 @@ impl Node for NodeService {
}
};
let resource = args["resources"][0].as_str().unwrap_or("");
let owner = args["owner"].as_str().unwrap_or("");
if resource.is_empty() {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some("No resource specified".to_string()),
}));
}
match self.lock_manager.runlock_batch(&[resource.to_string()], owner).await {
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not runlock, resource: {resource}, owner: {owner}, err: {err}")),
error_info: Some(format!(
"can not runlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn force_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let args: serde_json::Value = match serde_json::from_str(&request.args) {
let args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
@@ -1685,31 +1650,24 @@ impl Node for NodeService {
}
};
let resource = args["resources"][0].as_str().unwrap_or("");
let owner = args["owner"].as_str().unwrap_or("");
if resource.is_empty() {
return Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some("No resource specified".to_string()),
}));
}
match self.lock_manager.unlock_batch(&[resource.to_string()], owner).await {
match self.lock_manager.release(&args.lock_id).await {
Ok(_) => Ok(tonic::Response::new(GenerallyLockResponse {
success: true,
error_info: None,
})),
Err(err) => Ok(tonic::Response::new(GenerallyLockResponse {
success: false,
error_info: Some(format!("can not force_unlock, resource: {resource}, owner: {owner}, err: {err}")),
error_info: Some(format!(
"can not force_unlock, resource: {0}, owner: {1}, err: {2}",
args.resource, args.owner, err
)),
})),
}
}
async fn refresh(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
let request = request.into_inner();
let _args: serde_json::Value = match serde_json::from_str(&request.args) {
let _args: LockRequest = match serde_json::from_str(&request.args) {
Ok(args) => args,
Err(err) => {
return Ok(tonic::Response::new(GenerallyLockResponse {
crates/ecstore/src/set_disk.rs
+16 -22
View File
@@ -83,7 +83,7 @@ use rustfs_filemeta::{
headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
merge_file_meta_versions,
};
use rustfs_lock::{NamespaceLockManager, NsLockMap};
use rustfs_lock::NamespaceLockManager;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader};
use rustfs_utils::{
@@ -123,9 +123,8 @@ pub const MAX_PARTS_COUNT: usize = 10000;
#[derive(Clone, Debug)]
pub struct SetDisks {
pub lockers: Vec<Arc<rustfs_lock::NamespaceLock>>,
pub namespace_lock: Arc<rustfs_lock::NamespaceLock>,
pub locker_owner: String,
pub ns_mutex: Arc<NsLockMap>,
pub disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
pub set_endpoints: Vec<Endpoint>,
pub set_drive_count: usize,
@@ -138,9 +137,8 @@ pub struct SetDisks {
impl SetDisks {
#[allow(clippy::too_many_arguments)]
pub async fn new(
lockers: Vec<Arc<rustfs_lock::NamespaceLock>>,
namespace_lock: Arc<rustfs_lock::NamespaceLock>,
locker_owner: String,
ns_mutex: Arc<NsLockMap>,
disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
set_drive_count: usize,
default_parity_count: usize,
@@ -150,9 +148,8 @@ impl SetDisks {
format: FormatV3,
) -> Arc<Self> {
Arc::new(SetDisks {
lockers,
namespace_lock,
locker_owner,
ns_mutex,
disks,
set_drive_count,
default_parity_count,
@@ -4066,25 +4063,21 @@ impl ObjectIO for SetDisks {
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
let disks = self.disks.read().await;
let mut _ns = None;
if !opts.no_lock {
let paths = vec![object.to_string()];
let ns_lock = self
.ns_mutex
.new_nslock(None)
.await
.map_err(|err| Error::other(err.to_string()))?;
let lock_acquired = ns_lock
.lock_batch(&paths, &self.locker_owner, std::time::Duration::from_secs(5))
.await
.map_err(|err| Error::other(err.to_string()))?;
let lock_acquired = self
.namespace_lock
.lock_batch(
&paths,
&self.locker_owner,
std::time::Duration::from_secs(5),
std::time::Duration::from_secs(10),
)
.await?;
if !lock_acquired {
return Err(Error::other("can not get lock. please retry".to_string()));
}
_ns = Some(ns_lock);
}
let mut user_defined = opts.user_defined.clone();
@@ -4291,9 +4284,10 @@ impl ObjectIO for SetDisks {
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
if let Some(ns_lock) = _ns {
// Release lock if it was acquired
if !opts.no_lock {
let paths = vec![object.to_string()];
if let Err(err) = ns_lock.unlock_batch(&paths, &self.locker_owner).await {
if let Err(err) = self.namespace_lock.unlock_batch(&paths, &self.locker_owner).await {
error!("Failed to unlock object {}: {}", object, err);
}
}
crates/ecstore/src/sets.rs
+8 -30
View File
@@ -40,11 +40,10 @@ use crate::{
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
use futures::future::join_all;
use futures_util::FutureExt;
use http::HeaderMap;
use rustfs_common::globals::GLOBAL_Local_Node_Name;
use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLock;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use tokio::sync::RwLock;
@@ -56,12 +55,13 @@ use tokio::time::Duration;
use tracing::warn;
use tracing::{error, info};
use crate::lock_utils::create_unique_clients;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
// pub sets: Vec<Objects>,
// pub disk_set: Vec<Vec<Option<DiskStore>>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub lockers: Vec<Vec<Arc<NamespaceLock>>>,
pub disk_set: Vec<Arc<SetDisks>>, // [set_count_idx][set_drive_count_idx] = disk_idx
pub pool_idx: usize,
pub endpoints: PoolEndpoints,
@@ -95,45 +95,24 @@ impl Sets {
let set_drive_count = fm.erasure.sets[0].len();
let mut unique: Vec<Vec<String>> = (0..set_count).map(|_| vec![]).collect();
let mut lockers: Vec<Vec<Arc<NamespaceLock>>> = (0..set_count).map(|_| vec![]).collect();
for (idx, endpoint) in endpoints.endpoints.as_ref().iter().enumerate() {
let set_idx = idx / set_drive_count;
if endpoint.is_local && !unique[set_idx].contains(&"local".to_string()) {
unique[set_idx].push("local".to_string());
let local_manager = rustfs_lock::NsLockMap::new(false, None);
let local_lock = Arc::new(local_manager.new_nslock(None).await.unwrap_or_else(|_| {
// If creation fails, create an empty lock manager
rustfs_lock::NsLockMap::new(false, None)
.new_nslock(None)
.now_or_never()
.unwrap()
.unwrap()
}));
lockers[set_idx].push(local_lock);
}
if !endpoint.is_local {
let host_port = format!("{}:{}", endpoint.url.host_str().unwrap(), endpoint.url.port().unwrap());
if !unique[set_idx].contains(&host_port) {
unique[set_idx].push(host_port);
let dist_manager = rustfs_lock::NsLockMap::new(true, None);
let dist_lock = Arc::new(dist_manager.new_nslock(Some(endpoint.url.clone())).await.unwrap_or_else(|_| {
// If creation fails, create an empty lock manager
rustfs_lock::NsLockMap::new(true, None)
.new_nslock(Some(endpoint.url.clone()))
.now_or_never()
.unwrap()
.unwrap()
}));
lockers[set_idx].push(dist_lock);
}
}
}
let mut disk_set = Vec::with_capacity(set_count);
for (i, locker) in lockers.iter().enumerate().take(set_count) {
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
let mut set_endpoints = Vec::with_capacity(set_drive_count);
for j in 0..set_drive_count {
@@ -141,7 +120,6 @@ impl Sets {
let mut disk = disks[idx].clone();
let endpoint = endpoints.endpoints.as_ref()[idx].clone();
// let endpoint = endpoints.endpoints.as_ref().get(idx).cloned();
set_endpoints.push(endpoint);
if disk.is_none() {
@@ -185,12 +163,13 @@ impl Sets {
}
}
// warn!("sets new set_drive {:?}", &set_drive);
let lock_clients = create_unique_clients(&set_endpoints).await?;
let namespace_lock = rustfs_lock::NamespaceLock::with_clients(format!("set-{i}"), lock_clients);
let set_disks = SetDisks::new(
locker.clone(),
Arc::new(namespace_lock),
GLOBAL_Local_Node_Name.read().await.to_string(),
Arc::new(rustfs_lock::NsLockMap::new(is_dist_erasure().await, None)),
Arc::new(RwLock::new(set_drive)),
set_drive_count,
parity_count,
@@ -210,7 +189,6 @@ impl Sets {
id: fm.id,
// sets: todo!(),
disk_set,
lockers,
pool_idx,
endpoints: endpoints.clone(),
format: fm.clone(),