fix(ecstore): fence deletes against decommission commits (#6363)

* fix(ecstore): fence deletes against decommission commits

* fix(ecstore): preserve decommission target write locks

* fix(ecstore): preserve delete markers during source cleanup

* fix(ecstore): route batch delete markers to active pools

* fix(ecstore): preserve batch delete pool errors

* fix(ecstore): retain source-set lock during cleanup

* test(ecstore): exercise decommission delete fences

* test(ecstore): finish decommission delete fence scenario

* fix(ecstore): reuse fixed fence for reverse decommission

* fix(ecstore): fence decommission commit loss

* fix(ecstore): annotate batch delete fallback

* test(ecstore): fix decommission fence fixtures

* fix(ecstore): unblock decommission delete fences

* fix(ecstore): preserve distributed decommission set locks

* fix(ecstore): match decommission lock backend domain

* test(ecstore): align decommission fence barriers

* fix(ecstore): satisfy delete fence lint checks

* fix(rebalance): preserve access-denied delete errors

crates/ecstore/src/store/multipart.rs

@@ -400,7 +400,7 @@ impl ECStore {
         object: &str,
         opts: &ObjectOptions,
     ) -> Result<MultipartUploadResult> {
-        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
+        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
             .await
             .map(|(res, _, _)| res)
     }
@@ -410,20 +410,22 @@ impl ECStore {
         bucket: &str,
         object: &str,
         opts: &ObjectOptions,
+        mutation_fence: Option<&ObjectLockDiagGuard>,
     ) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
         check_new_multipart_args(bucket, object)?;
-        let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
-        let opts = &opts;
+        let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
 
         if self.single_pool() {
+            self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
+                .await;
             return self.pools[0]
-                .new_multipart_upload(bucket, object, opts)
+                .new_multipart_upload(bucket, object, &opts)
                 .await
                 .map(|res| (res, 0, opts.expected_bucket_incarnation_id));
         }
 
         if opts.data_movement && opts.version_id.is_some() {
-            let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
+            let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
             if idx == opts.src_pool_idx {
                 return Err(StorageError::DataMovementOverwriteErr(
                     bucket.to_owned(),
@@ -431,7 +433,9 @@ impl ECStore {
                     opts.version_id.clone().unwrap_or_default(),
                 ));
             }
-            let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+            self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                .await;
+            let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
             return Ok((res, idx, opts.expected_bucket_incarnation_id));
         }
 
@@ -454,7 +458,9 @@ impl ECStore {
             .await?;
 
             if !res.uploads.is_empty() {
-                let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+                self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+                    .await;
+                let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
                 return Ok((res, idx, opts.expected_bucket_incarnation_id));
             }
         }
@@ -467,7 +473,9 @@ impl ECStore {
             ));
         }
 
-        let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+        self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
+            .await;
+        let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
         Ok((res, idx, opts.expected_bucket_incarnation_id))
     }
 
@@ -704,13 +712,14 @@ impl ECStore {
 
     pub(crate) async fn complete_multipart_upload_for_data_movement(
         self: Arc<Self>,
-        target_pool_idx: usize,
+        target: (usize, Option<&ObjectLockDiagGuard>),
         bucket: &str,
         object: &str,
         upload_id: &str,
         uploaded_parts: Vec<CompletePart>,
         opts: &ObjectOptions,
     ) -> Result<ObjectInfo> {
+        let (target_pool_idx, mutation_fence) = target;
         check_complete_multipart_args(bucket, object, upload_id)?;
         if !opts.data_movement {
             return Err(Error::other("targeted multipart completion requires data_movement options"));
@@ -739,6 +748,8 @@ impl ECStore {
             snapshot.add_lock_fences(&mut opts);
             opts.object_lock_config_snapshot = Some(snapshot);
         }
+        self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
+            .await;
         #[cfg(test)]
         pause_data_movement_multipart_before_selected_completion(bucket).await;
         let pool = self