gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 14:23:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: NamespaceLock (nslock), AHM→Heal Crate, and Lock/Clippy Fixes (#1664)

Browse Source 
Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: weisd <2057561+weisd@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
weisd
2026-01-30 13:13:41 +08:00
committed by GitHub
parent 1c085590ca
commit dce117840c
80 changed files with 3787 additions and 16746 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/disk/local.rs
+5
View File
@@ -2813,6 +2813,11 @@ mod test {
let disk_info = disk.disk_info(&disk_info_opts).await.unwrap();
// Basic checks on disk info
// Note: On macOS and some other Unix systems, fs_type may be empty
// because statvfs does not provide filesystem type information.
// This is a platform limitation, not a bug.
#[cfg(not(target_os = "macos"))]
assert!(!disk_info.fs_type.is_empty(), "fs_type should not be empty on this platform");
assert!(disk_info.total > 0);
assert!(disk_info.free <= disk_info.total);
assert!(!disk_info.mount_path.is_empty());
crates/ecstore/src/global.rs
+51
View File
@@ -21,6 +21,7 @@ use crate::{
tier::tier::TierConfigMgr,
};
use lazy_static::lazy_static;
use rustfs_lock::client::LockClient;
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
@@ -55,6 +56,8 @@ lazy_static! {
pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
pub static ref GLOBAL_NodeNamesHex: HashMap<String, ()> = HashMap::new();
pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
pub static ref GLOBAL_LOCAL_LOCK_CLIENT: OnceLock<Arc<dyn rustfs_lock::client::LockClient>> = OnceLock::new();
pub static ref GLOBAL_LOCK_CLIENTS: OnceLock<HashMap<String, Arc<dyn LockClient>>> = OnceLock::new();
}
/// Global cancellation token for background services (data scanner and auto heal)
@@ -275,3 +278,51 @@ pub fn shutdown_background_services() {
cancel_token.cancel();
}
}
/// Set the global lock client (first LocalClient created)
///
/// # Arguments
/// * `client` - The LockClient instance to set globally
///
/// # Returns
/// * `Ok(())` if successful
/// * `Err(Arc<dyn LockClient>)` if setting fails (client already set)
///
pub fn set_global_lock_client(
client: Arc<dyn rustfs_lock::client::LockClient>,
) -> Result<(), Arc<dyn rustfs_lock::client::LockClient>> {
GLOBAL_LOCAL_LOCK_CLIENT.set(client)
}
/// Get the global lock client
///
/// # Returns
/// * `Option<Arc<dyn LockClient>>` - The global lock client, if set
///
pub fn get_global_lock_client() -> Option<Arc<dyn rustfs_lock::client::LockClient>> {
GLOBAL_LOCAL_LOCK_CLIENT.get().cloned()
}
/// Set the global lock clients map
///
/// # Arguments
/// * `clients` - The HashMap of lock clients to set globally
///
/// # Returns
/// * `Ok(())` if successful
/// * `Err(HashMap<String, Arc<dyn LockClient>>)` if setting fails (clients already set)
///
pub fn set_global_lock_clients(
clients: HashMap<String, Arc<dyn LockClient>>,
) -> Result<(), HashMap<String, Arc<dyn LockClient>>> {
GLOBAL_LOCK_CLIENTS.set(clients)
}
/// Get the global lock clients map
///
/// # Returns
/// * `Option<&HashMap<String, Arc<dyn LockClient>>>` - The global lock clients map, if set
///
pub fn get_global_lock_clients() -> Option<&'static HashMap<String, Arc<dyn LockClient>>> {
GLOBAL_LOCK_CLIENTS.get()
}
crates/ecstore/src/lib.rs
+1
View File
@@ -56,6 +56,7 @@ pub mod tier;
pub use global::new_object_layer_fn;
pub use global::set_global_endpoints;
pub use global::update_erasure_type;
pub use global::{get_global_lock_client, get_global_lock_clients, set_global_lock_client, set_global_lock_clients};
pub use global::GLOBAL_Endpoints;
pub use store_api::StorageAPI;
crates/ecstore/src/rpc/remote_locker.rs
+52 -132
View File
@@ -20,53 +20,33 @@ use rustfs_lock::{
};
use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
use rustfs_protos::proto_gen::node_service::{GenerallyLockRequest, PingRequest};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::info;
use tracing::{info, warn};
/// Remote lock client implementation
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct RemoteClient {
addr: String,
// Track active locks with their original owner information
active_locks: Arc<RwLock<HashMap<LockId, String>>>, // lock_id -> owner
}
impl Clone for RemoteClient {
fn clone(&self) -> Self {
Self {
addr: self.addr.clone(),
active_locks: self.active_locks.clone(),
}
}
}
impl RemoteClient {
pub fn new(endpoint: String) -> Self {
Self {
addr: endpoint,
active_locks: Arc::new(RwLock::new(HashMap::new())),
}
Self { addr: endpoint }
}
pub fn from_url(url: url::Url) -> Self {
Self {
addr: url.to_string(),
active_locks: Arc::new(RwLock::new(HashMap::new())),
}
Self { addr: url.to_string() }
}
/// Create a minimal LockRequest for unlock operations
fn create_unlock_request(&self, lock_id: &LockId, owner: &str) -> LockRequest {
/// Create a minimal LockRequest for unlock operations using only lock_id
fn create_unlock_request(lock_id: &LockId) -> LockRequest {
LockRequest {
lock_id: lock_id.clone(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // Type doesn't matter for unlock
owner: owner.to_string(),
owner: String::new(), // Owner not needed, server uses lock_id
acquire_timeout: std::time::Duration::from_secs(30),
ttl: std::time::Duration::from_secs(300),
metadata: LockMetadata::default(),
@@ -84,13 +64,14 @@ impl RemoteClient {
#[async_trait]
impl LockClient for RemoteClient {
async fn acquire_exclusive(&self, request: &LockRequest) -> Result<LockResponse> {
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
info!("remote acquire_exclusive for {}", request.resource);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.lock(req)
.await
@@ -104,11 +85,30 @@ impl LockClient for RemoteClient {
// Check if the lock acquisition was successful
if resp.success {
// Save the lock information for later release
let mut locks = self.active_locks.write().await;
locks.insert(request.lock_id.clone(), request.owner.clone());
Ok(LockResponse::success(
// Try to deserialize lock_info from response
let lock_info = if let Some(lock_info_json) = resp.lock_info {
match serde_json::from_str::<LockInfo>(&lock_info_json) {
Ok(info) => info,
Err(e) => {
// If deserialization fails, fall back to constructing from request
warn!("Failed to deserialize lock_info from response: {}, using request data", e);
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
}
}
}
} else {
// If lock_info is not provided, construct from request
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
@@ -121,9 +121,10 @@ impl LockClient for RemoteClient {
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
}
};
Ok(LockResponse::success(lock_info, std::time::Duration::ZERO))
} else {
// Lock acquisition failed
Ok(LockResponse::failure(
@@ -133,109 +134,28 @@ impl LockClient for RemoteClient {
}
}
async fn acquire_shared(&self, request: &LockRequest) -> Result<LockResponse> {
info!("remote acquire_shared for {}", request.resource);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
});
let resp = client
.r_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
// Check for explicit error first
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
// Check if the lock acquisition was successful
if resp.success {
// Save the lock information for later release
let mut locks = self.active_locks.write().await;
locks.insert(request.lock_id.clone(), request.owner.clone());
Ok(LockResponse::success(
LockInfo {
id: request.lock_id.clone(),
resource: request.resource.clone(),
lock_type: request.lock_type,
status: LockStatus::Acquired,
owner: request.owner.clone(),
acquired_at: std::time::SystemTime::now(),
expires_at: std::time::SystemTime::now() + request.ttl,
last_refreshed: std::time::SystemTime::now(),
metadata: request.metadata.clone(),
priority: request.priority,
wait_start_time: None,
},
std::time::Duration::ZERO,
))
} else {
// Lock acquisition failed
Ok(LockResponse::failure(
"Shared lock acquisition failed on remote server".to_string(),
std::time::Duration::ZERO,
))
}
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote release for {}", lock_id);
// Get the original owner for this lock
let owner = {
let locks = self.active_locks.read().await;
locks.get(lock_id).cloned().unwrap_or_else(|| "remote".to_string())
};
let unlock_request = self.create_unlock_request(lock_id, &owner);
let unlock_request = Self::create_unlock_request(lock_id);
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
let mut client = self.get_client().await?;
// Try UnLock first (for exclusive locks)
let req = Request::new(GenerallyLockRequest {
args: request_string.clone(),
});
let resp = client.un_lock(req).await;
let success = if resp.is_err() {
// If that fails, try RUnLock (for shared locks)
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = client
.r_un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
resp.success
} else {
let resp = resp.map_err(|e| LockError::internal(e.to_string()))?.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
resp.success
};
// Remove the lock from our tracking if successful
if success {
let mut locks = self.active_locks.write().await;
locks.remove(lock_id);
let req = Request::new(GenerallyLockRequest { args: request_string });
let resp = client
.un_lock(req)
.await
.map_err(|e| LockError::internal(e.to_string()))?
.into_inner();
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
Ok(success)
Ok(resp.success)
}
async fn refresh(&self, lock_id: &LockId) -> Result<bool> {
info!("remote refresh for {}", lock_id);
let refresh_request = self.create_unlock_request(lock_id, "remote");
let refresh_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&refresh_request)
@@ -254,7 +174,7 @@ impl LockClient for RemoteClient {
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote force_release for {}", lock_id);
let force_request = self.create_unlock_request(lock_id, "remote");
let force_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
let req = Request::new(GenerallyLockRequest {
args: serde_json::to_string(&force_request)
@@ -276,7 +196,7 @@ impl LockClient for RemoteClient {
// Since there's no direct status query in the gRPC service,
// we attempt a non-blocking lock acquisition to check if the resource is available
let status_request = self.create_unlock_request(lock_id, "remote");
let status_request = Self::create_unlock_request(lock_id);
let mut client = self.get_client().await?;
// Try to acquire a very short-lived lock to test availability
@@ -307,7 +227,7 @@ impl LockClient for RemoteClient {
// We can't determine the exact details remotely, so return a generic status
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.as_str().to_string(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive, // We can't know the exact type
status: LockStatus::Acquired,
owner: "unknown".to_string(), // Remote client can't determine owner
@@ -324,7 +244,7 @@ impl LockClient for RemoteClient {
// Communication error or lock is held
Ok(Some(LockInfo {
id: lock_id.clone(),
resource: lock_id.as_str().to_string(),
resource: lock_id.resource.clone(),
lock_type: LockType::Exclusive,
status: LockStatus::Acquired,
owner: "unknown".to_string(),
crates/ecstore/src/set_disk.rs
+254 -245
View File
@@ -73,8 +73,10 @@ use rustfs_filemeta::{
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
RawFileInfo, ReplicationStatusType, VersionPurgeStatusType, file_info_from_raw, merge_file_meta_versions,
};
use rustfs_lock::FastLockGuard;
use rustfs_lock::LockClient;
use rustfs_lock::fast_lock::types::LockResult;
use rustfs_lock::local_lock::LocalLock;
use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey};
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
use rustfs_utils::http::RUSTFS_BUCKET_REPLICATION_SSEC_CHECKSUM;
@@ -118,9 +120,14 @@ pub const MAX_PARTS_COUNT: usize = 10000;
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
/// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds)
/// Defaults to 30 seconds if not set or invalid
fn get_lock_acquire_timeout() -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
}
#[derive(Clone, Debug)]
pub struct SetDisks {
pub fast_lock_manager: Arc<rustfs_lock::FastObjectLockManager>,
pub locker_owner: String,
pub disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
pub set_endpoints: Vec<Endpoint>,
@@ -130,6 +137,7 @@ pub struct SetDisks {
pub pool_index: usize,
pub format: FormatV3,
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
pub lockers: Vec<Arc<dyn LockClient>>,
}
#[derive(Clone, Debug)]
@@ -151,7 +159,6 @@ impl DiskHealthEntry {
impl SetDisks {
#[allow(clippy::too_many_arguments)]
pub async fn new(
fast_lock_manager: Arc<rustfs_lock::FastObjectLockManager>,
locker_owner: String,
disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
set_drive_count: usize,
@@ -160,9 +167,9 @@ impl SetDisks {
pool_index: usize,
set_endpoints: Vec<Endpoint>,
format: FormatV3,
lockers: Vec<Arc<dyn LockClient>>,
) -> Arc<Self> {
Arc::new(SetDisks {
fast_lock_manager,
locker_owner,
disks,
set_drive_count,
@@ -172,6 +179,7 @@ impl SetDisks {
format,
set_endpoints,
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
lockers,
})
}
@@ -237,6 +245,27 @@ impl SetDisks {
LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"),
}
}
fn format_lock_error_from_error(
&self,
bucket: &str,
object: &str,
mode: &str,
err: &rustfs_lock::error::LockError,
) -> String {
match err {
rustfs_lock::error::LockError::Timeout { .. } => {
format!(
"ns_loc: {mode} lock acquisition timed out on {bucket}/{object} (owner={})",
self.locker_owner
)
}
rustfs_lock::error::LockError::AlreadyLocked { owner, .. } => {
format!("ns_loc: {mode} lock conflicted on {bucket}/{object}: held by {owner}")
}
_ => format!("ns_loc: {mode} lock acquisition failed on {bucket}/{object}: {}", err),
}
}
async fn get_disks_internal(&self) -> Vec<Option<DiskStore>> {
let rl = self.disks.read().await;
@@ -602,8 +631,10 @@ impl SetDisks {
Ok(())
}
#[tracing::instrument(skip(disks))]
async fn cleanup_multipart_path(disks: &[Option<DiskStore>], paths: &[String]) {
#[tracing::instrument(skip(self))]
async fn cleanup_multipart_path(&self, paths: &[String]) {
let disks = self.get_disks_internal().await;
let mut errs = Vec::with_capacity(disks.len());
// Use improved simple batch processor instead of join_all for better performance
@@ -831,7 +862,9 @@ impl SetDisks {
}
#[tracing::instrument(skip(disks, meta))]
#[allow(clippy::too_many_arguments)]
async fn rename_part(
&self,
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
@@ -878,7 +911,8 @@ impl SetDisks {
if let Some(err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
warn!("rename_part errs {:?}", &errs);
Self::cleanup_multipart_path(disks, &[dst_object.to_string(), format!("{dst_object}.meta")]).await;
self.cleanup_multipart_path(&[dst_object.to_string(), format!("{dst_object}.meta")])
.await;
return Err(err);
}
@@ -2627,63 +2661,15 @@ impl SetDisks {
..Default::default()
};
let _write_lock_guard = if !opts.no_lock {
info!("Acquiring write lock for object: {}, owner: {}", object, self.locker_owner);
// let fast_lock_guard = self.new_ns_lock(bucket, object).await?;
// Some(fast_lock_guard)
// Check if lock is already held
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
let mut reuse_existing_lock = false;
if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key) {
if lock_info.owner.as_ref() == self.locker_owner.as_str()
&& matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive)
{
reuse_existing_lock = true;
debug!("Reusing existing exclusive lock for object {} held by {}", object, self.locker_owner);
} else {
warn!("Lock already exists for object {}: {:?}", object, lock_info);
}
} else {
info!("No existing lock found for object {}", object);
}
if reuse_existing_lock {
None
} else {
let mut lock_result = None;
for i in 0..3 {
let start_time = Instant::now();
match self
.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.await
{
Ok(res) => {
let elapsed = start_time.elapsed();
info!(duration = ?elapsed, attempt = i + 1, "Write lock acquired");
lock_result = Some(res);
break;
}
Err(e) => {
let elapsed = start_time.elapsed();
info!(error = ?e, attempt = i + 1, duration = ?elapsed, "Lock acquisition failed, retrying");
if i < 2 {
tokio::time::sleep(Duration::from_millis(50 * (i as u64 + 1))).await;
} else {
let message = self.format_lock_error(bucket, object, "write", &e);
error!("Failed to acquire write lock after retries: {}", message);
return Err(DiskError::other(message));
}
}
}
}
lock_result
}
let write_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
} else {
info!("Skipping lock acquisition (no_lock=true)");
None
};
@@ -3330,11 +3316,15 @@ impl SetDisks {
remove: bool,
) -> Result<(HealResultItem, Option<DiskError>)> {
let _write_lock_guard = self
.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.new_ns_lock(bucket, object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| {
let message = self.format_lock_error(bucket, object, "write", &e);
let message = format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
);
DiskError::other(message)
})?;
@@ -3632,10 +3622,16 @@ impl ObjectIO for SetDisks {
// Acquire a shared read-lock early to protect read consistency
let read_lock_guard = if !opts.no_lock {
Some(
self.fast_lock_manager
.acquire_read_lock(bucket, object, self.locker_owner.as_str())
self.new_ns_lock(bucket, object)
.await?
.get_read_lock(get_lock_acquire_timeout())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "read", &e)))?,
.map_err(|e| {
Error::other(format!(
"Failed to acquire read lock: {}",
self.format_lock_error_from_error(bucket, object, "read", &e)
))
})?,
)
} else {
None
@@ -3718,24 +3714,23 @@ impl ObjectIO for SetDisks {
#[tracing::instrument(level = "debug", skip(self, data,))]
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
let disks = self.get_disks_internal().await;
// let (disks, filtered_online) = self.filter_online_disks(disks_snapshot).await;
// Acquire per-object exclusive lock via RAII guard. It auto-releases asynchronously on drop.
let _object_lock_guard = if !opts.no_lock {
Some(
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
)
} else {
None
};
let mut object_lock_guard = None;
if let Some(http_preconditions) = opts.http_preconditions.clone()
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err);
if let Some(http_preconditions) = opts.http_preconditions.clone() {
if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
let mut user_defined = opts.user_defined.clone();
@@ -3937,6 +3932,16 @@ impl ObjectIO for SetDisks {
drop(writers); // drop writers to close all files, this is to prevent FileAccessDenied errors when renaming data
if !opts.no_lock && object_lock_guard.is_none() {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
}
let (online_disks, _, op_old_dir) = Self::rename_data(
&shuffle_disks,
RUSTFS_META_TMP_BUCKET,
@@ -3953,6 +3958,8 @@ impl ObjectIO for SetDisks {
.await?;
}
drop(object_lock_guard); // drop object lock guard to release the lock
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
for (i, op_disk) in online_disks.iter().enumerate() {
@@ -3975,11 +3982,30 @@ impl ObjectIO for SetDisks {
#[async_trait::async_trait]
impl StorageAPI for SetDisks {
#[tracing::instrument(skip(self))]
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<FastLockGuard> {
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
let set_lock = if is_dist_erasure().await {
// Calculate quorum based on lockers count (majority)
let lockers_count = self.lockers.len();
let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };
NamespaceLock::with_clients_and_quorum(
format!("set-{}-{}", self.pool_index, self.set_index),
self.lockers.clone(),
write_quorum,
)
} else {
NamespaceLock::Local(LocalLock::new(
format!("set-{}-{}", self.pool_index, self.set_index),
Arc::new(rustfs_lock::GlobalLockManager::new()),
))
};
let resource = ObjectKey {
bucket: Arc::from(bucket),
object: Arc::from(object),
version: None,
};
Ok(NamespaceLockWrapper::new(set_lock, resource, self.locker_owner.clone()))
}
#[tracing::instrument(skip(self))]
@@ -4041,10 +4067,16 @@ impl StorageAPI for SetDisks {
// Guard lock for source object metadata update
let _lock_guard = self
.fast_lock_manager
.acquire_write_lock(src_bucket, src_object, self.locker_owner.as_str())
.new_ns_lock(src_bucket, src_object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| Error::other(self.format_lock_error(src_bucket, src_object, "write", &e)))?;
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(src_bucket, src_object, "write", &e)
))
})?;
let disks = self.get_disks_internal().await;
@@ -4139,12 +4171,6 @@ impl StorageAPI for SetDisks {
}
#[tracing::instrument(skip(self))]
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> {
// // Guard lock for single object delete-version
// let _lock_guard = self
// .fast_lock_manager
// .acquire_write_lock("", object, self.locker_owner.as_str())
// .await
// .map_err(|_| Error::other("can not get lock. please retry".to_string()))?;
let disks = self.get_disks(0, 0).await?;
let write_quorum = disks.len() / 2 + 1;
@@ -4206,34 +4232,41 @@ impl StorageAPI for SetDisks {
let mut unique_objects: HashSet<String> = HashSet::new();
for dobj in &objects {
if unique_objects.insert(dobj.object_name.clone()) {
batch = batch.add_write_lock(bucket, dobj.object_name.clone());
batch = batch.add_write_lock(rustfs_lock::ObjectKey::new(bucket, dobj.object_name.clone()));
}
}
let batch_result = self.fast_lock_manager.acquire_locks_batch(batch).await;
let locked_objects: HashSet<String> = batch_result
.successful_locks
.iter()
.map(|key| key.object.as_ref().to_string())
.collect();
let _lock_guards = batch_result.guards;
let mut failed_map = HashMap::new();
let mut batch_guards = Vec::with_capacity(batch.requests.len());
let failed_map: HashMap<(String, String), LockResult> = batch_result
.failed_locks
.into_iter()
.map(|(key, err)| ((key.bucket.as_ref().to_string(), key.object.as_ref().to_string()), err))
.collect();
let mut locked_objects = HashSet::new();
for req in batch.requests.iter() {
let ns_lock = match self.new_ns_lock(req.key.bucket.as_ref(), req.key.object.as_ref()).await {
Ok(ns_lock) => ns_lock,
Err(e) => {
failed_map.insert((req.key.bucket.as_ref().to_string(), req.key.object.as_ref().to_string()), e.to_string());
continue;
}
};
let _lock_guard = match ns_lock.get_write_lock(get_lock_acquire_timeout()).await {
Ok(lock_guard) => lock_guard,
Err(e) => {
failed_map.insert((req.key.bucket.as_ref().to_string(), req.key.object.as_ref().to_string()), e.to_string());
continue;
}
};
batch_guards.push(_lock_guard);
locked_objects.insert(req.key.object.as_ref().to_string());
}
// Mark failures for objects that could not be locked
for (i, dobj) in objects.iter().enumerate() {
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {
let message = self.format_lock_error(bucket, dobj.object_name.as_str(), "write", err);
del_errs[i] = Some(Error::other(message));
del_errs[i] = Some(Error::other(err.to_string()));
}
}
// let mut del_fvers = Vec::with_capacity(objects.len());
let ver_cfg = BucketVersioningSys::get(bucket).await.unwrap_or_default();
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
@@ -4398,10 +4431,16 @@ impl StorageAPI for SetDisks {
// Guard lock for single object delete
let _lock_guard = if !opts.delete_prefix {
Some(
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
self.new_ns_lock(bucket, object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
)
} else {
None
@@ -4578,10 +4617,16 @@ impl StorageAPI for SetDisks {
// Acquire a shared read-lock to protect consistency during info fetch
let _read_lock_guard = if !opts.no_lock {
Some(
self.fast_lock_manager
.acquire_read_lock(bucket, object, self.locker_owner.as_str())
self.new_ns_lock(bucket, object)
.await?
.get_read_lock(get_lock_acquire_timeout())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "read", &e)))?,
.map_err(|e| {
Error::other(format!(
"Failed to acquire read lock: {}",
self.format_lock_error_from_error(bucket, object, "read", &e)
))
})?,
)
} else {
None
@@ -4628,10 +4673,16 @@ impl StorageAPI for SetDisks {
// Guard lock for metadata update
let _lock_guard = if !opts.no_lock {
Some(
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
self.new_ns_lock(bucket, object)
.await?
.get_write_lock(get_lock_acquire_timeout())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
.map_err(|e| {
Error::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?,
)
} else {
None
@@ -4640,8 +4691,8 @@ impl StorageAPI for SetDisks {
let disks = self.get_disks_internal().await;
let (metas, errs) = {
if let Some(vid) = opts.version_id.as_ref() {
Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), false, false).await?
if let Some(version_id) = &opts.version_id {
Self::read_all_fileinfo(&disks, "", bucket, object, version_id.to_string().as_str(), false, false).await?
} else {
Self::read_all_xl(&disks, bucket, object, false, false).await
}
@@ -5156,16 +5207,17 @@ impl StorageAPI for SetDisks {
drop(writers); // drop writers to close all files
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
let _ = Self::rename_part(
&disks,
RUSTFS_META_TMP_BUCKET,
&tmp_part_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
part_info_buff.into(),
write_quorum,
)
.await?;
let _ = self
.rename_part(
&disks,
RUSTFS_META_TMP_BUCKET,
&tmp_part_path,
RUSTFS_META_MULTIPART_BUCKET,
&part_path,
part_info_buff.into(),
write_quorum,
)
.await?;
let ret: PartInfo = PartInfo {
etag: Some(etag.clone()),
@@ -5453,6 +5505,24 @@ impl StorageAPI for SetDisks {
#[tracing::instrument(skip(self))]
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult> {
if let Some(http_preconditions) = opts.http_preconditions.clone() {
let object_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
} else {
None
};
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
let disks = self.disks.read().await;
let disks = disks.clone();
@@ -5608,16 +5678,24 @@ impl StorageAPI for SetDisks {
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let _object_lock_guard = if !opts.no_lock {
Some(
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
)
} else {
None
};
let mut object_lock_guard = None;
// Acquire per-object exclusive lock via RAII guard. It auto-releases asynchronously on drop.
if let Some(http_preconditions) = opts.http_preconditions.clone() {
if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
}
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
let (mut fi, files_metas) = self.check_upload_id_exists(bucket, object, upload_id, true).await?;
let upload_id_path = Self::get_upload_id_dir(bucket, object, upload_id);
@@ -5629,25 +5707,6 @@ impl StorageAPI for SetDisks {
let disks = disks.clone();
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
// Acquire per-object exclusive lock via RAII guard. It auto-releases asynchronously on drop.
if let Some(http_preconditions) = opts.http_preconditions.clone() {
// if !opts.no_lock {
// let guard_opt = self
// .namespace_lock
// .lock_guard(object, &self.locker_owner, Duration::from_secs(5), Duration::from_secs(10))
// .await?;
// if guard_opt.is_none() {
// return Err(Error::other("can not get lock. please retry".to_string()));
// }
// _object_lock_guard = guard_opt;
// }
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
return Err(err);
}
}
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
let part_meta_paths = uploaded_parts
@@ -5671,11 +5730,16 @@ impl StorageAPI for SetDisks {
return Err(Error::other("checksum type not found"));
};
checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct);
if let Some(want) = opts.want_checksum.as_ref()
&& !want.checksum_type.is(checksum_type)
if let Some(want) = &opts.want_checksum
&& !want
.checksum_type
.is(rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct))
{
return Err(Error::other(format!("checksum type mismatch, got {:?}, want {:?}", want, checksum_type)));
return Err(Error::other(format!(
"checksum type mismatch, got {:?}, want {:?}",
want,
rustfs_rio::ChecksumType::from_string_with_obj_type(cs, ct)
)));
}
}
@@ -5962,11 +6026,18 @@ impl StorageAPI for SetDisks {
}
}
{
let disks = self.get_disks_internal().await;
Self::cleanup_multipart_path(&disks, &parts).await;
if !opts.no_lock && object_lock_guard.is_none() {
let ns_lock = self.new_ns_lock(bucket, object).await?;
object_lock_guard = Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?);
}
self.cleanup_multipart_path(&parts).await;
let (online_disks, versions, op_old_dir) = Self::rename_data(
&shuffle_disks,
RUSTFS_META_MULTIPART_BUCKET,
@@ -5982,6 +6053,9 @@ impl StorageAPI for SetDisks {
self.commit_rename_data_dir(&shuffle_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
drop(object_lock_guard); // drop object lock guard to release the lock
if let Some(versions) = versions {
let _ =
rustfs_common::heal_channel::send_heal_request(rustfs_common::heal_channel::create_heal_request_with_options(
@@ -6048,79 +6122,14 @@ impl StorageAPI for SetDisks {
version_id: &str,
opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> {
// let mut effective_object = object.to_string();
//
// // Optimization: Only attempt correction if the name looks suspicious (quotes or URL encoded)
// // and the original object does NOT exist.
// let has_quotes = (effective_object.starts_with('\'') && effective_object.ends_with('\''))
// || (effective_object.starts_with('"') && effective_object.ends_with('"'));
// let has_percent = effective_object.contains('%');
//
// if has_quotes || has_percent {
// let disks = self.disks.read().await;
// // 1. Check if the original object exists (lightweight check)
// let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, &effective_object, version_id, false, false).await?;
//
// if DiskError::is_all_not_found(&errs) {
// // Original not found. Try candidates.
// let mut candidates = Vec::new();
//
// // Candidate 1: URL Decoded (Priority for web access issues)
// if has_percent {
// if let Ok(decoded) = urlencoding::decode(&effective_object) {
// if decoded != effective_object {
// candidates.push(decoded.to_string());
// }
// }
// }
//
// // Candidate 2: Quote Stripped (For shell copy-paste issues)
// if has_quotes && effective_object.len() >= 2 {
// candidates.push(effective_object[1..effective_object.len() - 1].to_string());
// }
//
// // Check candidates
// for candidate in candidates {
// let (_, errs_cand) =
// Self::read_all_fileinfo(&disks, "", bucket, &candidate, version_id, false, false).await?;
//
// if !DiskError::is_all_not_found(&errs_cand) {
// info!(
// "Heal request for object '{}' failed (not found). Auto-corrected to '{}'.",
// effective_object, candidate
// );
// effective_object = candidate;
// break; // Found a match, stop searching
// }
// }
// }
// }
// let object = effective_object.as_str();
let _write_lock_guard = if !opts.no_lock {
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
let mut skip_lock = false;
if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key)
&& lock_info.owner.as_ref() == self.locker_owner.as_str()
&& matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive)
{
debug!(
"Reusing existing exclusive lock for heal operation on {}/{} held by {}",
bucket, object, self.locker_owner
);
skip_lock = true;
}
if skip_lock {
None
} else {
info!(?opts, "Starting heal_object");
Some(
self.fast_lock_manager
.acquire_write_lock(bucket, object, self.locker_owner.as_str())
.await
.map_err(|e| Error::other(self.format_lock_error(bucket, object, "write", &e)))?,
)
}
let ns_lock = self.new_ns_lock(bucket, object).await?;
Some(ns_lock.get_write_lock(get_lock_acquire_timeout()).await.map_err(|e| {
StorageError::other(format!(
"Failed to acquire write lock: {}",
self.format_lock_error_from_error(bucket, object, "write", &e)
))
})?)
} else {
None
};
crates/ecstore/src/sets.rs
+19 -25
View File
@@ -25,7 +25,7 @@ use crate::{
},
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, is_dist_erasure},
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
set_disk::SetDisks,
store_api::{
BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec,
@@ -42,7 +42,8 @@ use rustfs_common::{
heal_channel::{DriveState, HealItemType},
};
use rustfs_filemeta::FileInfo;
use rustfs_lock::FastLockGuard;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_lock::client::LockClient;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use std::{collections::HashMap, sync::Arc};
@@ -91,35 +92,30 @@ impl Sets {
let set_count = fm.erasure.sets.len();
let set_drive_count = fm.erasure.sets[0].len();
let mut unique: Vec<Vec<String>> = (0..set_count).map(|_| vec![]).collect();
for (idx, endpoint) in endpoints.endpoints.as_ref().iter().enumerate() {
let set_idx = idx / set_drive_count;
if endpoint.is_local && !unique[set_idx].contains(&"local".to_string()) {
unique[set_idx].push("local".to_string());
}
if !endpoint.is_local {
let host_port = format!("{}:{}", endpoint.url.host_str().unwrap(), endpoint.url.port().unwrap());
if !unique[set_idx].contains(&host_port) {
unique[set_idx].push(host_port);
}
}
}
let mut disk_set = Vec::with_capacity(set_count);
// Create fast lock manager for high performance
let fast_lock_manager = Arc::new(rustfs_lock::FastObjectLockManager::new());
// Get lock clients from global storage
let lock_clients = get_global_lock_clients();
for i in 0..set_count {
let mut set_drive = Vec::with_capacity(set_drive_count);
let mut set_endpoints = Vec::with_capacity(set_drive_count);
let mut set_lock_clients: HashMap<String, Arc<dyn LockClient>> = HashMap::new();
for j in 0..set_drive_count {
let idx = i * set_drive_count + j;
let mut disk = disks[idx].clone();
let endpoint = endpoints.endpoints.as_ref()[idx].clone();
if let Some(lock_clients_map) = lock_clients {
let host_port = endpoint.host_port();
if let Some(lock_client) = lock_clients_map.get(&host_port)
&& !set_lock_clients.contains_key(&host_port)
{
set_lock_clients.insert(host_port, lock_client.clone());
}
}
set_endpoints.push(endpoint);
if disk.is_none() {
@@ -163,11 +159,8 @@ impl Sets {
}
}
// Note: write_quorum was used for the old lock system, no longer needed with FastLock
let _write_quorum = set_drive_count - parity_count;
let lockers = set_lock_clients.values().cloned().collect::<Vec<Arc<dyn LockClient>>>();
let set_disks = SetDisks::new(
fast_lock_manager.clone(),
GLOBAL_LOCAL_NODE_NAME.read().await.to_string(),
Arc::new(RwLock::new(set_drive)),
set_drive_count,
@@ -176,6 +169,7 @@ impl Sets {
pool_idx,
set_endpoints,
fm.clone(),
lockers,
)
.await;
@@ -365,7 +359,7 @@ impl ObjectIO for Sets {
#[async_trait::async_trait]
impl StorageAPI for Sets {
#[tracing::instrument(skip(self))]
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<FastLockGuard> {
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
self.disk_set[0].new_ns_lock(bucket, object).await
}
#[tracing::instrument(skip(self))]
crates/ecstore/src/store.rs
+45 -15
View File
@@ -45,6 +45,7 @@ use crate::global::{
use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::store_api::{
ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO, ObjectInfoOrErr, WalkOptions,
};
@@ -69,7 +70,7 @@ use rand::Rng as _;
use rustfs_common::heal_channel::{HealItemType, HealOpts};
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_HOST, GLOBAL_RUSTFS_PORT};
use rustfs_filemeta::FileInfo;
use rustfs_lock::FastLockGuard;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
@@ -1125,6 +1126,45 @@ pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()>
Ok(())
}
/// create unique lock clients for the endpoints and store them globally
pub fn init_lock_clients(endpoint_pools: EndpointServerPools) {
let mut unique_endpoints: HashMap<String, &Endpoint> = HashMap::new();
for pool_eps in endpoint_pools.as_ref().iter() {
for ep in pool_eps.endpoints.as_ref().iter() {
unique_endpoints.insert(ep.host_port(), ep);
}
}
let mut clients = HashMap::new();
let mut first_local_client_set = false;
for (key, endpoint) in unique_endpoints {
if endpoint.is_local {
let local_client = Arc::new(LocalClient::new()) as Arc<dyn LockClient>;
// Store the first LocalClient globally for use by other modules
if !first_local_client_set {
if let Err(e) = crate::global::set_global_lock_client(local_client.clone()) {
// If already set, ignore the error (another thread may have set it)
warn!("set_global_lock_client error: {:?}", e);
} else {
first_local_client_set = true;
}
}
clients.insert(key, local_client);
} else {
clients.insert(key, Arc::new(RemoteClient::new(endpoint.url.to_string())) as Arc<dyn LockClient>);
}
}
// Store the lock clients map globally
if crate::global::set_global_lock_clients(clients).is_err() {
error!("init_lock_clients: error setting lock clients");
}
}
#[derive(Debug, Default)]
struct PoolErr {
index: Option<usize>,
@@ -1245,7 +1285,7 @@ lazy_static! {
#[async_trait::async_trait]
impl StorageAPI for ECStore {
#[instrument(skip(self))]
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<FastLockGuard> {
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
self.pools[0].new_ns_lock(bucket, object).await
}
#[instrument(skip(self))]
@@ -2352,20 +2392,10 @@ impl StorageAPI for ECStore {
let mut futures = Vec::with_capacity(self.pools.len());
for pool in self.pools.iter() {
//TODO: IsSuspended
if self.is_suspended(pool.pool_idx).await {
continue;
}
futures.push(pool.heal_object(bucket, &object, version_id, opts));
// futures.push(async move {
// match pool.heal_object(bucket, &object, version_id, opts).await {
// Ok((mut result, err)) => {
// result.object = utils::path::decode_dir_object(&result.object);
// results.write().await.insert(idx, result);
// errs.write().await[idx] = err;
// }
// Err(err) => {
// errs.write().await[idx] = Some(err);
// }
// }
// });
}
let results = join_all(futures).await;
crates/ecstore/src/store_api.rs
+2 -5
View File
@@ -31,7 +31,7 @@ use rustfs_filemeta::{
ReplicationStatusType, RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::FastLockGuard;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
@@ -1349,10 +1349,7 @@ pub trait ObjectIO: Send + Sync + Debug + 'static {
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait StorageAPI: ObjectIO + Debug {
// NewNSLock TODO:
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<FastLockGuard>;
// Shutdown TODO:
// NSScanner TODO:
async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper>;
async fn backend_info(&self) -> rustfs_madmin::BackendInfo;
async fn storage_info(&self) -> rustfs_madmin::StorageInfo;