refactor: NamespaceLock (nslock), AHM→Heal Crate, and Lock/Clippy Fixes (#1664)

Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
Co-authored-by: weisd <2057561+weisd@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

rustfs/Cargo.toml

@@ -39,7 +39,7 @@ workspace = true
 
 [dependencies]
 # RustFS Internal Crates
-rustfs-ahm = { workspace = true }
+rustfs-heal = { workspace = true }
 rustfs-appauth = { workspace = true }
 rustfs-audit = { workspace = true }
 rustfs-common = { workspace = true }

rustfs/src/main.rs

@@ -36,9 +36,9 @@ use crate::server::{
 };
 use clap::Parser;
 use license::init_license;
-use rustfs_ahm::{create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services};
 use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
 use rustfs_credentials::init_global_action_credentials;
+use rustfs_ecstore::store::init_lock_clients;
 use rustfs_ecstore::{
     StorageAPI,
     bucket::metadata_sys::init_bucket_metadata_sys,
@@ -54,6 +54,9 @@ use rustfs_ecstore::{
     store_api::BucketOptions,
     update_erasure_type,
 };
+use rustfs_heal::{
+    create_ahm_services_cancel_token, heal::storage::ECStoreHealStorage, init_heal_manager, shutdown_ahm_services,
+};
 use rustfs_iam::init_iam_sys;
 use rustfs_obs::{init_obs, set_global_guard};
 use rustfs_scanner::init_data_scanner;
@@ -183,6 +186,14 @@ async fn run(opt: config::Opt) -> Result<()> {
         .await
         .map_err(Error::other)?;
 
+    set_global_endpoints(endpoint_pools.as_ref().clone());
+    update_erasure_type(setup_type).await;
+
+    // Initialize the local disk
+    init_local_disks(endpoint_pools.clone()).await.map_err(Error::other)?;
+    // Initialize the lock clients
+    init_lock_clients(endpoint_pools.clone());
+
     for (i, eps) in endpoint_pools.as_ref().iter().enumerate() {
         info!(
             target: "rustfs::main::run",
@@ -236,12 +247,6 @@ async fn run(opt: config::Opt) -> Result<()> {
         None
     };
 
-    set_global_endpoints(endpoint_pools.as_ref().clone());
-    update_erasure_type(setup_type).await;
-
-    // Initialize the local disk
-    init_local_disks(endpoint_pools.clone()).await.map_err(Error::other)?;
-
     let ctx = CancellationToken::new();
 
     // init store

rustfs/src/storage/ecfs.rs

@@ -111,6 +111,8 @@ use rustfs_targets::{
     EventName,
     arn::{ARN, TargetIDError},
 };
+use rustfs_utils::http::RUSTFS_FORCE_DELETE;
+use rustfs_utils::string::parse_bool;
 use rustfs_utils::{
     CompressionAlgorithm, extract_params_header, extract_resp_elements, get_request_host, get_request_port,
     get_request_user_agent,
@@ -1357,19 +1359,37 @@ impl S3 for FS {
 
     /// Delete a bucket
     #[instrument(level = "debug", skip(self, req))]
-    async fn delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
+    async fn delete_bucket(&self, mut req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
         let helper = OperationHelper::new(&req, EventName::BucketRemoved, "s3:DeleteBucket");
-        let input = req.input;
+        let input = req.input.clone();
         // TODO: DeleteBucketInput doesn't have force parameter?
         let Some(store) = new_object_layer_fn() else {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
         };
 
+        // get value from header, support mc style
+        let force_str = req
+            .headers
+            .get(RUSTFS_FORCE_DELETE)
+            .map(|v| v.to_str().unwrap_or_default())
+            .unwrap_or(
+                req.headers
+                    .get("x-minio-force-delete")
+                    .map(|v| v.to_str().unwrap_or_default())
+                    .unwrap_or_default(),
+            );
+
+        let force = parse_bool(force_str).unwrap_or_default();
+
+        if force {
+            authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
+        }
+
         store
             .delete_bucket(
                 &input.bucket,
                 &DeleteBucketOptions {
-                    force: false,
+                    force,
                     ..Default::default()
                 },
             )
@@ -4504,8 +4524,6 @@ impl S3 for FS {
             sse_customer_key_md5,
             ssekms_key_id,
             content_md5,
-            if_match,
-            if_none_match,
             ..
         } = input;
 
@@ -4540,46 +4558,6 @@ impl S3 for FS {
             }
         }
 
-        if if_match.is_some() || if_none_match.is_some() {
-            let Some(store) = new_object_layer_fn() else {
-                return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
-            };
-
-            match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
-                Ok(info) => {
-                    if !info.delete_marker {
-                        if let Some(ifmatch) = if_match
-                            && let Some(strong_etag) = ifmatch.into_etag()
-                            && info
-                                .etag
-                                .as_ref()
-                                .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
-                        {
-                            return Err(s3_error!(PreconditionFailed));
-                        }
-                        if let Some(ifnonematch) = if_none_match
-                            && let Some(strong_etag) = ifnonematch.into_etag()
-                            && info
-                                .etag
-                                .as_ref()
-                                .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
-                        {
-                            return Err(s3_error!(PreconditionFailed));
-                        }
-                    }
-                }
-                Err(err) => {
-                    if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
-                        return Err(ApiError::from(err).into());
-                    }
-
-                    if if_match.is_some() && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
-                        return Err(ApiError::from(err).into());
-                    }
-                }
-            }
-        }
-
         let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };
 
         let mut size = match content_length {

rustfs/src/storage/tonic_service.rs

@@ -24,6 +24,7 @@ use rustfs_ecstore::{
         DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts,
         error::DiskError,
     },
+    get_global_lock_client,
     metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
     new_object_layer_fn,
     rpc::{LocalPeerS3Client, PeerS3Client},
@@ -74,16 +75,11 @@ type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send>>;
 #[derive(Debug)]
 pub struct NodeService {
     local_peer: LocalPeerS3Client,
-    lock_manager: Arc<rustfs_lock::LocalClient>,
 }
 
 pub fn make_server() -> NodeService {
     let local_peer = LocalPeerS3Client::new(None, None);
-    let lock_manager = Arc::new(rustfs_lock::LocalClient::new());
-    NodeService {
-        local_peer,
-        lock_manager,
-    }
+    NodeService { local_peer }
 }
 
 impl NodeService {
@@ -94,6 +90,12 @@ impl NodeService {
     async fn all_disk(&self) -> Vec<String> {
         all_local_disk_path().await
     }
+
+    /// Get the global lock client, returning an error if not initialized
+    fn get_lock_client(&self) -> Result<Arc<dyn LockClient>, Status> {
+        get_global_lock_client()
+            .ok_or_else(|| Status::internal("Lock client not initialized. Please ensure storage is initialized first."))
+    }
 }
 
 #[tonic::async_trait]
@@ -1458,21 +1460,29 @@ impl Node for NodeService {
                 return Ok(Response::new(GenerallyLockResponse {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
+                    lock_info: None,
                 }));
             }
         };
 
-        match self.lock_manager.acquire_exclusive(&args).await {
-            Ok(result) => Ok(Response::new(GenerallyLockResponse {
-                success: result.success,
-                error_info: None,
-            })),
+        let lock_client = self.get_lock_client()?;
+        match lock_client.acquire_lock(&args).await {
+            Ok(result) => {
+                // Serialize lock_info if available
+                let lock_info_json = result.lock_info.as_ref().and_then(|info| serde_json::to_string(info).ok());
+                Ok(Response::new(GenerallyLockResponse {
+                    success: result.success,
+                    error_info: None,
+                    lock_info: lock_info_json,
+                }))
+            }
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
                 error_info: Some(format!(
                     "can not lock, resource: {0}, owner: {1}, err: {2}",
                     args.resource, args.owner, err
                 )),
+                lock_info: None,
             })),
         }
     }
@@ -1485,14 +1495,17 @@ impl Node for NodeService {
                 return Ok(Response::new(GenerallyLockResponse {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
+                    lock_info: None,
                 }));
             }
         };
 
-        match self.lock_manager.release(&args.lock_id).await {
+        let lock_client = self.get_lock_client()?;
+        match lock_client.release(&args.lock_id).await {
             Ok(_) => Ok(Response::new(GenerallyLockResponse {
                 success: true,
                 error_info: None,
+                lock_info: None,
             })),
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
@@ -1500,60 +1513,7 @@ impl Node for NodeService {
                     "can not unlock, resource: {0}, owner: {1}, err: {2}",
                     args.resource, args.owner, err
                 )),
-            })),
-        }
-    }
-
-    async fn r_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
-        let request = request.into_inner();
-        let args: LockRequest = match serde_json::from_str(&request.args) {
-            Ok(args) => args,
-            Err(err) => {
-                return Ok(Response::new(GenerallyLockResponse {
-                    success: false,
-                    error_info: Some(format!("can not decode args, err: {err}")),
-                }));
-            }
-        };
-
-        match self.lock_manager.acquire_shared(&args).await {
-            Ok(result) => Ok(Response::new(GenerallyLockResponse {
-                success: result.success,
-                error_info: None,
-            })),
-            Err(err) => Ok(Response::new(GenerallyLockResponse {
-                success: false,
-                error_info: Some(format!(
-                    "can not rlock, resource: {0}, owner: {1}, err: {2}",
-                    args.resource, args.owner, err
-                )),
-            })),
-        }
-    }
-
-    async fn r_un_lock(&self, request: Request<GenerallyLockRequest>) -> Result<Response<GenerallyLockResponse>, Status> {
-        let request = request.into_inner();
-        let args: LockRequest = match serde_json::from_str(&request.args) {
-            Ok(args) => args,
-            Err(err) => {
-                return Ok(Response::new(GenerallyLockResponse {
-                    success: false,
-                    error_info: Some(format!("can not decode args, err: {err}")),
-                }));
-            }
-        };
-
-        match self.lock_manager.release(&args.lock_id).await {
-            Ok(_) => Ok(Response::new(GenerallyLockResponse {
-                success: true,
-                error_info: None,
-            })),
-            Err(err) => Ok(Response::new(GenerallyLockResponse {
-                success: false,
-                error_info: Some(format!(
-                    "can not runlock, resource: {0}, owner: {1}, err: {2}",
-                    args.resource, args.owner, err
-                )),
+                lock_info: None,
             })),
         }
     }
@@ -1566,14 +1526,17 @@ impl Node for NodeService {
                 return Ok(Response::new(GenerallyLockResponse {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
+                    lock_info: None,
                 }));
             }
         };
 
-        match self.lock_manager.release(&args.lock_id).await {
+        let lock_client = self.get_lock_client()?;
+        match lock_client.release(&args.lock_id).await {
             Ok(_) => Ok(Response::new(GenerallyLockResponse {
                 success: true,
                 error_info: None,
+                lock_info: None,
             })),
             Err(err) => Ok(Response::new(GenerallyLockResponse {
                 success: false,
@@ -1581,6 +1544,7 @@ impl Node for NodeService {
                     "can not force_unlock, resource: {0}, owner: {1}, err: {2}",
                     args.resource, args.owner, err
                 )),
+                lock_info: None,
             })),
         }
     }
@@ -1593,6 +1557,7 @@ impl Node for NodeService {
                 return Ok(Response::new(GenerallyLockResponse {
                     success: false,
                     error_info: Some(format!("can not decode args, err: {err}")),
+                    lock_info: None,
                 }));
             }
         };
@@ -1600,6 +1565,7 @@ impl Node for NodeService {
         Ok(Response::new(GenerallyLockResponse {
             success: true,
             error_info: None,
+            lock_info: None,
         }))
     }
 
@@ -3209,38 +3175,6 @@ mod tests {
         assert!(unlock_response.error_info.is_some());
     }
 
-    #[tokio::test]
-    async fn test_r_lock_invalid_args() {
-        let service = create_test_node_service();
-
-        let request = Request::new(GenerallyLockRequest {
-            args: "invalid json".to_string(),
-        });
-
-        let response = service.r_lock(request).await;
-        assert!(response.is_ok());
-
-        let rlock_response = response.unwrap().into_inner();
-        assert!(!rlock_response.success);
-        assert!(rlock_response.error_info.is_some());
-    }
-
-    #[tokio::test]
-    async fn test_r_un_lock_invalid_args() {
-        let service = create_test_node_service();
-
-        let request = Request::new(GenerallyLockRequest {
-            args: "invalid json".to_string(),
-        });
-
-        let response = service.r_un_lock(request).await;
-        assert!(response.is_ok());
-
-        let runlock_response = response.unwrap().into_inner();
-        assert!(!runlock_response.success);
-        assert!(runlock_response.error_info.is_some());
-    }
-
     #[tokio::test]
     async fn test_force_un_lock_invalid_args() {
         let service = create_test_node_service();