fix(ecstore): fence rebalance and decommission activation (#6400)

* fix(ecstore): fence rebalance and decommission activation

* fix(ecstore): fence lost activation locks

* fix(ecstore): bind rebalance workers to activation id

* fix(ecstore): close rebalance activation races

* test(ecstore): exercise lost rebalance commit fence

* fix(ecstore): repair rebalance fence test wiring

* test(ecstore): reuse rebalance metadata fixture

* fix(ecstore): satisfy rebalance activation clippy checks

* fix(ecstore): fence stale rebalance workers

* fix(ecstore): commit rebalance activation after persistence

* fix(ecstore): fence rebalance commits and unblock stop

* test(ecstore): exercise real rebalance fences

* fix(rebalance): cancel admin stop before activation wait

* fix(ecstore): fence multipart staging on rebalance lock loss

* fix(ecstore): adopt activations after durable commit

* fix(rebalance): preserve committed activation recovery

* fix(rebalance): make prepared stop terminal-safe

* fix(ecstore): repair rebalance test imports

* fix(ecstore): repair rebalance entry runtime failures

* test(ecstore): fix activation fence synchronization

* test(ecstore): scope rebalance disk trait import

* test(ecstore): observe decommission lock attempt

* fix(ecstore): align activation fence test imports

* fix(ecstore): remove duplicate activation test import

* fix(ecstore): resolve CI clippy failures

* fix: satisfy activation merge lint gates

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/set_disk/ops/mod.rs

@@ -25,3 +25,6 @@ pub(crate) mod list;
 pub(crate) mod locking;
 pub(crate) mod multipart;
 pub(crate) mod object;
+
+#[cfg(test)]
+pub(crate) use object::hermetic_set_disks_support::hermetic_set_disks_isolated;

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -86,6 +86,8 @@ struct MultipartCommitBarrierState {
     pause: MultipartCommitPause,
     expected_arrivals: usize,
     arrivals: AtomicUsize,
+    #[cfg(test)]
+    committed: AtomicBool,
     arrived: tokio::sync::Notify,
     release: tokio::sync::Semaphore,
 }
@@ -113,6 +115,8 @@ impl MultipartCommitBarrier {
             pause,
             expected_arrivals,
             arrivals: AtomicUsize::new(0),
+            #[cfg(test)]
+            committed: AtomicBool::new(false),
             arrived: tokio::sync::Notify::new(),
             release: tokio::sync::Semaphore::new(0),
         });
@@ -143,6 +147,11 @@ impl MultipartCommitBarrier {
     pub fn release(&self) {
         self.state.release.add_permits(self.state.expected_arrivals);
     }
+
+    #[cfg(test)]
+    pub(crate) fn commit_observed(&self) -> bool {
+        self.state.committed.load(Ordering::Acquire)
+    }
 }
 
 #[cfg(any(test, feature = "test-util"))]
@@ -263,6 +272,20 @@ async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartComm
     }
 }
 
+#[cfg(test)]
+fn observe_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
+    let slot = MULTIPART_COMMIT_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("multipart commit barrier mutex should not poison");
+    if let Some(barrier) = slot
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause == pause)
+    {
+        barrier.committed.store(true, Ordering::Release);
+    }
+}
+
 fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err: DiskError) -> Error {
     if err == DiskError::FileNotFound {
         return StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
@@ -1337,6 +1360,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
             pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
             fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
             fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
+            if opts
+                .namespace_lock_fence
+                .as_ref()
+                .is_some_and(NamespaceLockFence::is_lock_lost)
+            {
+                return Err(StorageError::NamespaceLockQuorumUnavailable {
+                    mode: "put_object_part_outer_lock",
+                    bucket: bucket.to_string(),
+                    object: object.to_string(),
+                    required: 1,
+                    achieved: 0,
+                });
+            }
             ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
 
             let _ = self
@@ -1357,6 +1393,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                     }),
                 )
                 .await?;
+            #[cfg(test)]
+            observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
 
             #[cfg(test)]
             pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
@@ -1737,7 +1775,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         .await
         .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
         #[cfg(test)]
-        observe_new_multipart_upload_commit(bucket, object);
+        {
+            observe_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost);
+            observe_new_multipart_upload_commit(bucket, object);
+        }
 
         // evalDisks
 

crates/ecstore/src/set_disk/ops/object.rs

@@ -6662,6 +6662,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
         };
 
         let find_vid = Uuid::new_v4();
+        #[cfg(test)]
+        pause_delete_object_commit(bucket, object).await;
 
         if mark_delete && (opts.versioned || opts.version_suspended) {
             if !delete_marker {
@@ -6730,8 +6732,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             dfi.set_skip_tier_free_version();
         }
 
-        #[cfg(test)]
-        pause_delete_object_commit(bucket, object).await;
         ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
         self.delete_object_version(bucket, object, &dfi, opts.delete_marker)
             .await
@@ -7904,9 +7904,7 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
     /// for tests that never touch context-resolved services registered on the
     /// ambient context (tier config manager, expiry state, ...), because the
     /// isolated context starts every one of those cells fresh.
-    pub(in crate::set_disk::ops) async fn hermetic_set_disks_isolated(
-        disk_count: usize,
-    ) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
+    pub(crate) async fn hermetic_set_disks_isolated(disk_count: usize) -> (Vec<TempDir>, Vec<DiskStore>, Arc<SetDisks>) {
         hermetic_set_disks_for_pool_with_default_parity_isolated(disk_count, 0, disk_count / 2).await
     }
 
@@ -12603,6 +12601,7 @@ mod transition_upload_integrity_tests {
                 crate::data_movement::SourceCleanupBucketFence {
                     expected_incarnation_id: None,
                     lifecycle_guard: Some(&bucket_guard),
+                    namespace_lock_lost_signal: None,
                     ..Default::default()
                 },
                 "test_data_movement",