fix(ecstore): fence free-version decommission replay

crates/ecstore/src/set_disk/mod.rs

@@ -227,15 +227,15 @@ pub(super) fn restore_commit_operation_id_from_metadata(metadata: &HashMap<Strin
     restore_operation_id_from_metadata(metadata)
 }
 
-async fn check_decommission_tier_free_version_target(
+async fn inspect_decommission_tier_free_version_target(
     disk: &DiskStore,
     bucket: &str,
     object: &str,
     source: &FileInfo,
-) -> Result<()> {
+) -> Result<bool> {
     let raw = match disk.read_xl(bucket, object, false).await {
         Ok(raw) => raw,
-        Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => return Ok(()),
+        Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => return Ok(false),
         Err(err) => return Err(err.into()),
     };
     let meta = FileMeta::load(&raw.buf)?;
@@ -253,8 +253,11 @@ async fn check_decommission_tier_free_version_target(
             all_matching_versions_equivalent = false;
         }
     }
-    if matching_count == 0 || (matching_count == 1 && all_matching_versions_equivalent) {
-        return Ok(());
+    if matching_count == 0 {
+        return Ok(false);
+    }
+    if matching_count == 1 && all_matching_versions_equivalent {
+        return Ok(true);
     }
 
     Err(StorageError::DataMovementOverwriteErr(
@@ -265,6 +268,28 @@ async fn check_decommission_tier_free_version_target(
     .into())
 }
 
+fn ensure_decommission_tier_free_version_commit_fence(bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
+    if opts
+        .namespace_lock_fence
+        .as_ref()
+        .is_some_and(NamespaceLockFence::is_lock_lost)
+        || opts
+            .bucket_lifecycle_lock_fence
+            .as_ref()
+            .is_some_and(NamespaceLockFence::is_lock_lost)
+    {
+        return Err(StorageError::NamespaceLockQuorumUnavailable {
+            mode: "decommission_tier_free_version_commit",
+            bucket: bucket.to_string(),
+            object: object.to_string(),
+            required: 1,
+            achieved: 0,
+        });
+    }
+
+    Ok(())
+}
+
 impl SetDisks {
     pub(super) async fn require_current_restore_operation_id(
         &self,
@@ -4718,26 +4743,11 @@ impl SetDisks {
         if !fi.deleted || !fi.tier_free_version() {
             return Err(Error::other("decommission tier free-version write requires a free version record"));
         }
-        if opts
-            .namespace_lock_fence
-            .as_ref()
-            .is_some_and(NamespaceLockFence::is_lock_lost)
-            || opts
-                .bucket_lifecycle_lock_fence
-                .as_ref()
-                .is_some_and(NamespaceLockFence::is_lock_lost)
-        {
-            return Err(StorageError::NamespaceLockQuorumUnavailable {
-                mode: "decommission_tier_free_version_commit",
-                bucket: bucket.to_string(),
-                object: object.to_string(),
-                required: 1,
-                achieved: 0,
-            });
-        }
+        ensure_decommission_tier_free_version_commit_fence(bucket, object, opts)?;
 
         self.validate_decommission_tier_free_version_target(bucket, object, fi)
             .await?;
+        ensure_decommission_tier_free_version_commit_fence(bucket, object, opts)?;
 
         let disks = self.disks.read().await.clone();
         let write_quorum = self.default_write_quorum();
@@ -4760,6 +4770,8 @@ impl SetDisks {
             }
         }
 
+        ensure_decommission_tier_free_version_commit_fence(bucket, object, opts)?;
+
         resolve_tiered_decommission_write_quorum_result(&errs, write_quorum, bucket, object)
     }
 
@@ -4776,13 +4788,38 @@ impl SetDisks {
         let preflight = disks
             .iter()
             .flatten()
-            .map(|disk| check_decommission_tier_free_version_target(disk, bucket, object, fi));
+            .map(|disk| inspect_decommission_tier_free_version_target(disk, bucket, object, fi));
         for result in join_all(preflight).await {
             result?;
         }
         Ok(())
     }
 
+    pub(crate) async fn has_decommission_tier_free_version_write_quorum(
+        &self,
+        bucket: &str,
+        object: &str,
+        fi: &FileInfo,
+        opts: &ObjectOptions,
+    ) -> Result<bool> {
+        ensure_decommission_tier_free_version_commit_fence(bucket, object, opts)?;
+        let disks = self.disks.read().await.clone();
+        let preflight = disks.iter().map(|disk| async {
+            match disk {
+                Some(disk) => inspect_decommission_tier_free_version_target(disk, bucket, object, fi).await,
+                None => Ok(false),
+            }
+        });
+        let mut equivalent = 0;
+        for result in join_all(preflight).await {
+            if result? {
+                equivalent += 1;
+            }
+        }
+        ensure_decommission_tier_free_version_commit_fence(bucket, object, opts)?;
+        Ok(equivalent >= self.default_write_quorum())
+    }
+
     #[tracing::instrument(skip(self, fi, opts))]
     pub(crate) async fn decommission_tiered_object(
         &self,
@@ -10056,6 +10093,72 @@ mod tests {
         assert_eq!(migrated.transitioned_objname, "remote/object");
     }
 
+    #[tokio::test]
+    async fn decommission_tier_free_version_resume_requires_write_quorum() {
+        let set_disks = make_local_bucket_test_set_disks_with_drive_count(4).await;
+        let bucket = "free-version-decommission-resume";
+        let object = "object.txt";
+        let mut free_version = FileInfo {
+            name: object.to_string(),
+            volume: bucket.to_string(),
+            version_id: Some(Uuid::new_v4()),
+            mod_time: Some(time::OffsetDateTime::now_utc()),
+            deleted: true,
+            transition_tier: "WARM-TIER".to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            ..Default::default()
+        };
+        free_version.set_tier_free_version();
+        let opts = ObjectOptions::default();
+
+        let disks = set_disks.get_disks_internal().await;
+        for disk in disks.iter().take(2).flatten() {
+            disk.write_metadata("", bucket, object, free_version.clone())
+                .await
+                .expect("partial first attempt should leave equivalent metadata");
+        }
+        assert!(
+            !set_disks
+                .has_decommission_tier_free_version_write_quorum(bucket, object, &free_version, &opts)
+                .await
+                .expect("partial target metadata should remain valid"),
+            "write-quorum-minus-one must not be accepted as an idempotent migration"
+        );
+
+        disks[2]
+            .as_ref()
+            .expect("third target disk should be online")
+            .write_metadata("", bucket, object, free_version.clone())
+            .await
+            .expect("third equivalent target write should complete quorum");
+        assert!(
+            set_disks
+                .has_decommission_tier_free_version_write_quorum(bucket, object, &free_version, &opts)
+                .await
+                .expect("write-quorum target metadata should remain valid")
+        );
+    }
+
+    #[test]
+    fn decommission_tier_free_version_commit_rejects_lost_fence() {
+        let opts = ObjectOptions {
+            namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
+            ..Default::default()
+        };
+
+        let err = ensure_decommission_tier_free_version_commit_fence("bucket", "object", &opts)
+            .expect_err("lost target lock must fail the free-version commit");
+        assert!(matches!(
+            err,
+            Error::NamespaceLockQuorumUnavailable {
+                mode: "decommission_tier_free_version_commit",
+                required: 1,
+                achieved: 0,
+                ..
+            }
+        ));
+    }
+
     #[test]
     fn test_resolve_tiered_decommission_write_quorum_result_allows_successful_quorum() {
         let errs = vec![None, None, Some(DiskError::DiskNotFound), None];