fix: resolve release-blocking integration failures (#6320)

* fix: resolve release-blocking integration failures

* fix: satisfy stable clippy lints

* fix: satisfy Rust 1.98 CI lints

crates/ecstore/src/set_disk/replication.rs

@@ -18,6 +18,78 @@ use rustfs_filemeta::RestoreStatusOps;
 use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
 use s3s::dto::{RestoreStatus, Timestamp};
 
+#[cfg(all(test, feature = "test-util"))]
+struct RestoreFinalizeBarrierState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+static RESTORE_FINALIZE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<RestoreFinalizeBarrierState>>>> =
+    std::sync::OnceLock::new();
+
+#[cfg(all(test, feature = "test-util"))]
+pub(in crate::set_disk) struct RestoreFinalizeBarrier {
+    state: Arc<RestoreFinalizeBarrierState>,
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl RestoreFinalizeBarrier {
+    pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(RestoreFinalizeBarrierState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        assert!(slot.is_none(), "restore finalize barrier must be installed by one test at a time");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(in crate::set_disk) async fn wait_until_paused(&self) {
+        self.state.arrived.notified().await;
+    }
+
+    pub(in crate::set_disk) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+impl Drop for RestoreFinalizeBarrier {
+    fn drop(&mut self) {
+        let mut slot = RESTORE_FINALIZE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("restore finalize barrier mutex should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(all(test, feature = "test-util"))]
+async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
+    let barrier = RESTORE_FINALIZE_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("restore finalize barrier mutex should not poison")
+        .as_ref()
+        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
+        .cloned();
+    if let Some(barrier) = barrier {
+        barrier.arrived.notify_one();
+        barrier.release.notified().await;
+    }
+}
+
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 struct RestoreCleanupIdentity {
     version_id: Option<Uuid>,
@@ -80,7 +152,7 @@ impl SetDisks {
             .clone()
             .unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
         let version_id = expected.version_id.map(|v| v.to_string());
-        let _lock_guard = if !opts.no_lock {
+        let lock_guard = if !opts.no_lock {
             Some(
                 self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
                     .await?,
@@ -99,13 +171,16 @@ impl SetDisks {
             .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
             .await?
             .into_owned();
-        if let Some(expected_operation_id) = expected_operation_id {
-            require_restore_operation_id(&fi.metadata, expected_operation_id)?;
+        if let Some(expected_operation_id) = expected_operation_id
+            && restore_operation_id_from_metadata(&fi.metadata)?.is_some_and(|actual| actual != expected_operation_id)
+        {
+            return Err(Error::other("restore operation id changed before metadata finalization"));
         }
         if !expected.matches_file_info(&fi, &expected_etag) {
             return Err(Error::other("restored object changed before restore metadata finalization"));
         }
-        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        #[cfg(all(test, feature = "test-util"))]
+        maybe_pause_restore_finalize(bucket, object).await;
         let restore_expiry =
             lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
         fi.metadata.insert(
@@ -117,6 +192,10 @@ impl SetDisks {
             .to_string(),
         );
         self.invalidate_get_object_metadata_cache(bucket, object).await;
+        ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
+        if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
+            return Err(Error::other("restore finalization lock lost before metadata update"));
+        }
         self.update_object_meta_with_opts(
             bucket,
             object,