fix(ecstore): migrate tier free versions during decommission (#6393)

crates/ecstore/src/store/object.rs

@@ -490,6 +490,82 @@ fn decommission_mutation_fence_for_test(
         .map(|hook| hook.fence.clone())
 }
 
+#[cfg(test)]
+struct DecommissionFreeVersionSourceRaceState {
+    bucket: String,
+    object: String,
+    arrived: tokio::sync::Notify,
+    release: tokio::sync::Notify,
+}
+
+#[cfg(test)]
+pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
+    state: Arc<DecommissionFreeVersionSourceRaceState>,
+}
+
+#[cfg(test)]
+static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
+    std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
+> = std::sync::OnceLock::new();
+
+#[cfg(test)]
+impl DecommissionFreeVersionSourceRaceBarrier {
+    pub(crate) fn install(bucket: &str, object: &str) -> Self {
+        let state = Arc::new(DecommissionFreeVersionSourceRaceState {
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            arrived: tokio::sync::Notify::new(),
+            release: tokio::sync::Notify::new(),
+        });
+        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("decommission free-version source race barrier should not poison");
+        assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
+        *slot = Some(Arc::clone(&state));
+        Self { state }
+    }
+
+    pub(crate) async fn wait_until_paused(&self) {
+        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
+            .await
+            .expect("decommission should pause before acquiring the free-version source lock");
+    }
+
+    pub(crate) fn release(&self) {
+        self.state.release.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for DecommissionFreeVersionSourceRaceBarrier {
+    fn drop(&mut self) {
+        self.state.release.notify_one();
+        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
+            .get_or_init(|| std::sync::Mutex::new(None))
+            .lock()
+            .expect("decommission free-version source race barrier should not poison");
+        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *slot = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
+    let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
+        .get_or_init(|| std::sync::Mutex::new(None))
+        .lock()
+        .expect("decommission free-version source race barrier should not poison")
+        .as_ref()
+        .filter(|state| state.bucket == bucket && state.object == object)
+        .cloned();
+    if let Some(state) = state {
+        state.arrived.notify_one();
+        state.release.notified().await;
+    }
+}
+
 pub(crate) struct SourceCleanupMutationFence {
     guard: ObjectLockDiagGuard,
     source_lock_covered: bool,
@@ -1495,13 +1571,15 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
         && source_actual_size == target_actual_size
 }
 
-fn tiered_data_movement_source_matches(
+pub(crate) fn tiered_data_movement_source_matches(
     expected: &rustfs_filemeta::FileInfo,
     current: &rustfs_filemeta::FileInfo,
 ) -> Result<bool> {
     let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
     let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&current.metadata)?;
     Ok(expected.version_id == current.version_id
+        && expected.deleted == current.deleted
+        && expected.tier_free_version() == current.tier_free_version()
         && expected.data_dir == current.data_dir
         && expected.mod_time == current.mod_time
         && expected.size == current.size
@@ -1515,6 +1593,14 @@ fn tiered_data_movement_source_matches(
         && expected_backend == current_backend)
 }
 
+fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
+    StorageError::DataMovementOverwriteErr(
+        bucket.to_owned(),
+        object.to_owned(),
+        version_id.map(|id| id.to_string()).unwrap_or_default(),
+    )
+}
+
 fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
     target_pool_idx != src_pool_idx
 }
@@ -2222,6 +2308,23 @@ impl ECStore {
         )
     }
 
+    async fn has_equivalent_data_movement_tier_free_version(
+        &self,
+        bucket: &str,
+        object: &str,
+        source: &rustfs_filemeta::FileInfo,
+        opts: &ObjectOptions,
+        target_pool_idx: usize,
+    ) -> Result<bool> {
+        let pool = self
+            .pools
+            .get(target_pool_idx)
+            .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
+        pool.get_disks_by_key(object)
+            .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
+            .await
+    }
+
     fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
         result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
     }
@@ -2241,6 +2344,10 @@ impl ECStore {
         check_put_object_args(bucket, object)?;
 
         let mut opts = opts.clone();
+        let is_free_version = fi.tier_free_version();
+        if is_free_version {
+            opts.incl_free_versions = true;
+        }
         let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
             None
         } else {
@@ -2278,6 +2385,10 @@ impl ECStore {
                 &object,
             )?
         };
+        #[cfg(test)]
+        if is_free_version {
+            pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
+        }
         let _object_guards = self
             .acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts)
             .await?;
@@ -2295,7 +2406,7 @@ impl ECStore {
                 versions
                     .versions
                     .iter()
-                    .find(|current| current.version_id == fi.version_id && !current.tier_free_version())
+                    .find(|current| current.version_id == fi.version_id && current.tier_free_version() == is_free_version)
             })
             .ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
         if !tiered_data_movement_source_matches(fi, current_source)? {
@@ -2310,24 +2421,34 @@ impl ECStore {
                 .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
                 .await;
             let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
-            if self
-                .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
-                .await?
-            {
+            if is_free_version && target_pool_idx == opts.src_pool_idx {
+                return Err(Error::DiskFull);
+            }
+            let equivalent = if is_free_version {
+                self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
+                    .await?
+            } else {
+                self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
+                    .await?
+            };
+            if equivalent {
                 return Ok(());
             }
 
-            return Err(StorageError::DataMovementOverwriteErr(
-                bucket.to_owned(),
-                object.to_owned(),
-                opts.version_id.clone().unwrap_or_default(),
-            ));
+            return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
         }
 
-        let result = self.pools[idx]
-            .get_disks_by_key(&object)
-            .decommission_tiered_object(bucket, &object, &fi, &opts)
-            .await;
+        let result = if is_free_version {
+            self.pools[idx]
+                .get_disks_by_key(&object)
+                .decommission_tier_free_version(bucket, &object, &fi, &opts)
+                .await
+        } else {
+            self.pools[idx]
+                .get_disks_by_key(&object)
+                .decommission_tiered_object(bucket, &object, &fi, &opts)
+                .await
+        };
         if matches!(result, Err(Error::PreconditionFailed)) {
             if self
                 .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)