fix(ecstore): tighten object copy rename handling (#3131)

* fix(ecstore): tighten object copy rename handling

* fix(ecstore): narrow copy lock lifetime

* test(ecstore): cover reverse copy concurrency

* fix(multipart): ignore preconditions for internal lookup

* fix(ecstore): clean precondition lock bindings

---------

Co-authored-by: houseme <housemecn@gmail.com>

rustfs/src/app/lifecycle_transition_api_test.rs

@@ -18,8 +18,8 @@ use crate::storage::ecfs::FS;
 use bytes::Bytes;
 use futures::FutureExt;
 use futures::stream;
-use http::{Extensions, HeaderMap, Method, Uri};
-use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
+use http::{Extensions, HeaderMap, HeaderValue, Method, Uri, header::IF_NONE_MATCH};
+use rustfs_config::{ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT};
 use rustfs_ecstore::{
     bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
     bucket::metadata_sys,
@@ -53,7 +53,7 @@ use std::{
 };
 use tokio::fs;
 use tokio::io::AsyncReadExt;
-use tokio::sync::Mutex;
+use tokio::sync::{Barrier, Mutex};
 use tokio_util::sync::CancellationToken;
 use uuid::Uuid;
 
@@ -484,6 +484,203 @@ fn streaming_blob_from_bytes(data: &[u8]) -> StreamingBlob {
     StreamingBlob::wrap::<_, Infallible>(stream::once(async move { Ok(body) }))
 }
 
+async fn read_object_bytes(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> Vec<u8> {
+    let mut reader = (**ecstore)
+        .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
+        .await
+        .expect("Failed to read object");
+    let mut buf = Vec::new();
+    reader
+        .stream
+        .read_to_end(&mut buf)
+        .await
+        .expect("Failed to drain object reader");
+    buf
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "requires isolated global object layer state"]
+async fn put_object_if_none_match_existing_object_returns_precondition_failed() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let fs = FS::new();
+    let usecase = DefaultObjectUsecase::without_context();
+
+    let bucket = format!("test-put-if-none-match-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let object = "test/object.txt";
+    let initial_payload = b"initial conditional put payload";
+    let replacement_payload = b"replacement conditional put payload";
+
+    create_test_bucket(&ecstore, bucket.as_str()).await;
+
+    let initial_input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(initial_payload)))
+        .content_length(Some(initial_payload.len() as i64))
+        .build()
+        .unwrap();
+    Box::pin(usecase.execute_put_object(&fs, build_request(initial_input, Method::PUT)))
+        .await
+        .expect("Failed to upload initial object through usecase");
+
+    let existing_info = ecstore
+        .get_object_info(bucket.as_str(), object, &ObjectOptions::default())
+        .await
+        .expect("Failed to fetch existing object info");
+    let existing_etag = existing_info.etag.expect("existing object should have an ETag");
+
+    let replacement_input = PutObjectInput::builder()
+        .bucket(bucket.clone())
+        .key(object.to_string())
+        .body(Some(streaming_blob_from_bytes(replacement_payload)))
+        .content_length(Some(replacement_payload.len() as i64))
+        .build()
+        .unwrap();
+    let mut req = build_request(replacement_input, Method::PUT);
+    req.headers
+        .insert(IF_NONE_MATCH, HeaderValue::from_str(existing_etag.as_str()).unwrap());
+
+    let err = Box::pin(usecase.execute_put_object(&fs, req)).await.unwrap_err();
+
+    assert_eq!(err.code(), &s3s::S3ErrorCode::PreconditionFailed);
+    assert_eq!(
+        read_object_bytes(&ecstore, bucket.as_str(), object).await,
+        initial_payload,
+        "failed conditional PutObject must not overwrite the current object"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[serial]
+#[ignore = "requires isolated global object layer state"]
+async fn copy_object_if_none_match_existing_destination_returns_precondition_failed() {
+    let (_disk_paths, ecstore) = setup_test_env().await;
+    let usecase = DefaultObjectUsecase::without_context();
+
+    let src_bucket = format!("test-copy-if-none-match-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let dst_bucket = format!("test-copy-if-none-match-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
+    let src_object = "test/source.txt";
+    let dst_object = "test/destination.txt";
+    let src_payload = b"conditional copy source payload";
+    let dst_payload = b"conditional copy destination payload";
+
+    create_test_bucket(&ecstore, src_bucket.as_str()).await;
+    create_test_bucket(&ecstore, dst_bucket.as_str()).await;
+    let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, src_payload).await;
+    let _ = upload_test_object(&ecstore, dst_bucket.as_str(), dst_object, dst_payload).await;
+
+    let dst_info = ecstore
+        .get_object_info(dst_bucket.as_str(), dst_object, &ObjectOptions::default())
+        .await
+        .expect("Failed to fetch destination object info");
+    let dst_etag = dst_info.etag.expect("destination object should have an ETag");
+
+    let copy_input = CopyObjectInput::builder()
+        .copy_source(CopySource::Bucket {
+            bucket: src_bucket.clone().into(),
+            key: src_object.to_string().into(),
+            version_id: None,
+        })
+        .bucket(dst_bucket.clone())
+        .key(dst_object.to_string())
+        .build()
+        .unwrap();
+    let mut req = build_request(copy_input, Method::PUT);
+    req.headers
+        .insert(IF_NONE_MATCH, HeaderValue::from_str(dst_etag.as_str()).unwrap());
+
+    let err = Box::pin(usecase.execute_copy_object(req)).await.unwrap_err();
+
+    assert_eq!(err.code(), &s3s::S3ErrorCode::PreconditionFailed);
+    assert_eq!(
+        read_object_bytes(&ecstore, dst_bucket.as_str(), dst_object).await,
+        dst_payload,
+        "failed conditional CopyObject must not overwrite the current destination object"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+#[ignore = "requires isolated global object layer state"]
+async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
+    temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
+        let (_disk_paths, ecstore) = setup_test_env().await;
+
+        let bucket = format!("test-reverse-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_a = "test/a.bin";
+        let object_b = "test/b.bin";
+        let payload_a = vec![b'a'; 4 * 1024 * 1024];
+        let payload_b = vec![b'b'; 4 * 1024 * 1024];
+
+        create_test_bucket(&ecstore, bucket.as_str()).await;
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object_a, &payload_a).await;
+        let _ = upload_test_object(&ecstore, bucket.as_str(), object_b, &payload_b).await;
+
+        let a_to_b_input = CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object_a.to_string().into(),
+                version_id: None,
+            })
+            .bucket(bucket.clone())
+            .key(object_b.to_string())
+            .build()
+            .unwrap();
+        let b_to_a_input = CopyObjectInput::builder()
+            .copy_source(CopySource::Bucket {
+                bucket: bucket.clone().into(),
+                key: object_b.to_string().into(),
+                version_id: None,
+            })
+            .bucket(bucket.clone())
+            .key(object_a.to_string())
+            .build()
+            .unwrap();
+
+        let start = Arc::new(Barrier::new(3));
+        let start_a = start.clone();
+        let a_to_b = tokio::spawn(async move {
+            start_a.wait().await;
+            let usecase = DefaultObjectUsecase::without_context();
+            Box::pin(usecase.execute_copy_object(build_request(a_to_b_input, Method::PUT))).await
+        });
+        let start_b = start.clone();
+        let b_to_a = tokio::spawn(async move {
+            start_b.wait().await;
+            let usecase = DefaultObjectUsecase::without_context();
+            Box::pin(usecase.execute_copy_object(build_request(b_to_a_input, Method::PUT))).await
+        });
+
+        start.wait().await;
+
+        let (a_to_b_result, b_to_a_result) = tokio::time::timeout(Duration::from_secs(10), async {
+            let (a_to_b_result, b_to_a_result) = tokio::join!(a_to_b, b_to_a);
+            (
+                a_to_b_result.expect("A-to-B CopyObject task should not panic"),
+                b_to_a_result.expect("B-to-A CopyObject task should not panic"),
+            )
+        })
+        .await
+        .expect("reverse CopyObject operations should not deadlock");
+
+        a_to_b_result.expect("A-to-B CopyObject should succeed");
+        b_to_a_result.expect("B-to-A CopyObject should succeed");
+
+        let final_a = read_object_bytes(&ecstore, bucket.as_str(), object_a).await;
+        let final_b = read_object_bytes(&ecstore, bucket.as_str(), object_b).await;
+        assert!(
+            final_a == payload_a || final_a == payload_b,
+            "object A must contain a complete copied payload"
+        );
+        assert!(
+            final_b == payload_a || final_b == payload_b,
+            "object B must contain a complete copied payload"
+        );
+    })
+    .await;
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
 #[serial]
 #[ignore = "requires isolated global object layer state"]

rustfs/src/app/multipart_usecase.rs

@@ -148,6 +148,11 @@ fn has_complete_multipart_object_lock_headers(headers: &HeaderMap) -> bool {
         || has_bypass_governance_header(headers)
 }
 
+fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
+    opts.http_preconditions = None;
+    opts
+}
+
 fn encode_s3_path(path: &str) -> String {
     path.split('/')
         .map(|part| encode(part).to_string())
@@ -336,9 +341,11 @@ impl DefaultMultipartUsecase {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
         };
 
-        let current_opts = get_opts(&bucket, &key, None, None, &req.headers)
-            .await
-            .map_err(ApiError::from)?;
+        let current_opts = internal_object_info_lookup_opts(
+            get_opts(&bucket, &key, None, None, &req.headers)
+                .await
+                .map_err(ApiError::from)?,
+        );
         let previous_current_size = match store.get_object_info(&bucket, &key, &current_opts).await {
             Ok(existing_obj_info) => {
                 validate_existing_object_lock_for_write(&existing_obj_info)?;
@@ -1317,6 +1324,26 @@ mod tests {
         assert_eq!(location, "/bucket/nested/object");
     }
 
+    #[test]
+    fn internal_object_info_lookup_opts_drops_http_preconditions() {
+        let opts = ObjectOptions {
+            version_id: Some(Uuid::new_v4().to_string()),
+            no_lock: true,
+            http_preconditions: Some(rustfs_ecstore::store_api::HTTPPreconditions {
+                if_none_match: Some("*".to_string()),
+                if_match: Some("\"etag\"".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let lookup_opts = internal_object_info_lookup_opts(opts);
+
+        assert!(lookup_opts.http_preconditions.is_none());
+        assert!(lookup_opts.no_lock);
+        assert!(lookup_opts.version_id.is_some());
+    }
+
     #[test]
     fn merge_part_encryption_metadata_keeps_source_metadata_unchanged() {
         let multipart_metadata = HashMap::from([

rustfs/src/app/object_usecase.rs

@@ -70,9 +70,9 @@ use rustfs_ecstore::config::storageclass;
 use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
 use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
 use rustfs_ecstore::new_object_layer_fn;
-use rustfs_ecstore::set_disk::is_valid_storage_class;
+use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
 use rustfs_ecstore::store_api::{
-    HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
+    HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, StorageAPI,
 };
 use rustfs_filemeta::{
     REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
@@ -80,6 +80,7 @@ use rustfs_filemeta::{
     version_purge_statuses_map,
 };
 use rustfs_io_metrics;
+use rustfs_lock::NamespaceLockGuard;
 use rustfs_notify::EventArgsBuilder;
 use rustfs_policy::policy::action::{Action, S3Action};
 use rustfs_rio::{CompressReader, DynReader, EncryptReader, HashReader, wrap_reader};
@@ -103,7 +104,7 @@ use rustfs_utils::http::{
     },
     insert_str, remove_str,
 };
-use rustfs_utils::path::{is_dir_object, path_join_buf};
+use rustfs_utils::path::{encode_dir_object, is_dir_object, path_join_buf};
 use rustfs_zip::CompressionFormat;
 use s3s::dto::*;
 use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH};
@@ -679,6 +680,36 @@ fn should_use_existing_delete_replication_info(opts: &ObjectOptions) -> bool {
     opts.version_id.is_some() && !opts.delete_marker
 }
 
+fn internal_object_info_lookup_opts(mut opts: ObjectOptions) -> ObjectOptions {
+    opts.http_preconditions = None;
+    opts
+}
+
+fn copy_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
+    match err {
+        rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
+            mode,
+            bucket: bucket.to_owned(),
+            object: object.to_owned(),
+            required,
+            achieved,
+        },
+        other => StorageError::other(format!("Failed to acquire {mode} lock on {bucket}/{object}: {other}")),
+    }
+}
+
+async fn acquire_self_copy_namespace_lock<S: StorageAPI + ?Sized>(
+    store: &S,
+    bucket: &str,
+    object: &str,
+) -> S3Result<NamespaceLockGuard> {
+    let object = encode_dir_object(object);
+    let lock = store.new_ns_lock(bucket, &object).await.map_err(ApiError::from)?;
+    lock.get_write_lock(get_lock_acquire_timeout())
+        .await
+        .map_err(|err| ApiError::from(copy_namespace_lock_error(bucket, &object, "write", err)).into())
+}
+
 fn delete_replication_state_source<'a>(
     opts: &ObjectOptions,
     existing_object_info: Option<&'a ObjectInfo>,
@@ -1812,9 +1843,11 @@ impl DefaultObjectUsecase {
         )
         .await?;
 
-        let current_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
-            .await
-            .map_err(ApiError::from)?;
+        let current_opts: ObjectOptions = internal_object_info_lookup_opts(
+            get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
+                .await
+                .map_err(ApiError::from)?,
+        );
         let previous_current_size = match store.get_object_info(&bucket, &key, &current_opts).await {
             Ok(existing_obj_info) => {
                 validate_existing_object_lock_for_write(&existing_obj_info)?;
@@ -2671,23 +2704,34 @@ impl DefaultObjectUsecase {
             ..Default::default()
         };
 
-        let dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, HashMap::new())
+        let mut dst_opts = copy_dst_opts(&bucket, &key, version_id, &req.headers, HashMap::new())
             .await
             .map_err(ApiError::from)?;
 
         let cp_src_dst_same = path_join_buf(&[&src_bucket, &src_key]) == path_join_buf(&[&bucket, &key]);
 
-        if cp_src_dst_same {
-            src_get_opts.no_lock = true;
-        }
-
         let Some(store) = new_object_layer_fn() else {
             return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
         };
 
-        let current_opts: ObjectOptions = get_opts(&bucket, &key, dest_version_id.clone(), None, &req.headers)
-            .await
-            .map_err(ApiError::from)?;
+        let _self_copy_lock_guard = if cp_src_dst_same {
+            let guard = acquire_self_copy_namespace_lock(store.as_ref(), &bucket, &key).await?;
+            src_opts.no_lock = true;
+            src_get_opts.no_lock = true;
+            dst_opts.no_lock = true;
+            Some(guard)
+        } else {
+            None
+        };
+
+        let mut current_opts: ObjectOptions = internal_object_info_lookup_opts(
+            get_opts(&bucket, &key, dest_version_id.clone(), None, &req.headers)
+                .await
+                .map_err(ApiError::from)?,
+        );
+        if cp_src_dst_same {
+            current_opts.no_lock = true;
+        }
         let previous_current_size = match store.get_object_info(&bucket, &key, &current_opts).await {
             Ok(existing_obj_info) => {
                 validate_existing_object_lock_for_write(&existing_obj_info)?;
@@ -4446,6 +4490,27 @@ mod tests {
         }
     }
 
+    #[test]
+    fn internal_object_info_lookup_opts_drops_http_preconditions() {
+        let version_id = Uuid::new_v4().to_string();
+        let opts = ObjectOptions {
+            version_id: Some(version_id.clone()),
+            no_lock: true,
+            http_preconditions: Some(rustfs_ecstore::store_api::HTTPPreconditions {
+                if_none_match: Some("\"etag\"".to_string()),
+                if_match: Some("\"other\"".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let lookup_opts = internal_object_info_lookup_opts(opts);
+
+        assert!(lookup_opts.http_preconditions.is_none());
+        assert_eq!(lookup_opts.version_id.as_deref(), Some(version_id.as_str()));
+        assert!(lookup_opts.no_lock);
+    }
+
     #[tokio::test]
     async fn build_put_like_object_lock_metadata_rejects_mode_without_retain_until_date() {
         let err = build_put_like_object_lock_metadata(