fix(ecstore): complete decommission capacity recovery (#6949)

crates/ecstore/src/store/init.rs

@@ -837,8 +837,9 @@ mod tests {
     use crate::{
         bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
         core::pools::{
-            POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus,
-            pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
+            DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
+            PoolDecommissionInfo, PoolMeta, PoolStatus, pool_meta_identity_initialized_for_test,
+            pool_meta_v3_commit_state_for_test, set_decommission_capacity_info_overrides_for_test,
         },
         disk::endpoint::Endpoint,
         error::{Error, Result, StorageError},
@@ -2163,12 +2164,31 @@ mod tests {
         (source_version, expected_source_versions)
     }
 
+    fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
+        let layout = DecommissionErasureLayout { data: 2, parity: 2 };
+        let source_physical_bytes = 1024 * 1024 * 1024;
+        let target_physical_bytes = source_physical_bytes * 8;
+        let capacity = store
+            .pools
+            .iter()
+            .enumerate()
+            .map(|(index, _)| {
+                if index == pool_idx {
+                    DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
+                } else {
+                    DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
+                }
+            })
+            .collect();
+        set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity]);
+    }
+
     async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
-        let mut pool_meta = store.pool_meta.write().await;
-        pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
-            start_time: Some(OffsetDateTime::now_utc()),
-            ..Default::default()
-        });
+        set_test_decommission_capacity_override(store, pool_idx);
+        store
+            .save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
+            .await
+            .expect("test decommission capacity reservation should activate");
     }
 
     const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
@@ -7325,6 +7345,7 @@ mod tests {
             .await
             .expect("legacy decommission queue should reload after restart");
         *store.pool_meta.write().await = restarted_pool_meta;
+        set_test_decommission_capacity_override(&store, 0);
         store
             .promote_queued_decommission_for_test(0)
             .await

crates/ecstore/src/store/object.rs

@@ -1204,7 +1204,11 @@ fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions
 }
 
 fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
-    pool_count > 0 && (!opts.versioned && !opts.version_suspended || opts.version_id.is_some())
+    pool_count > 0 && delete_only_releases_capacity(opts)
+}
+
+fn delete_only_releases_capacity(opts: &ObjectOptions) -> bool {
+    !opts.versioned && !opts.version_suspended || opts.version_id.is_some()
 }
 
 fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
@@ -2013,16 +2017,69 @@ impl ECStore {
         bucket: &str,
         lock_object: &str,
         target_object: &str,
-        mut opts: ObjectOptions,
+        opts: ObjectOptions,
         operation: F,
     ) -> Result<T>
     where
         F: FnOnce(ObjectOptions) -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
     {
-        let (capacity_guard, has_active_decommission) = self
-            .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation")
-            .await?;
+        self.run_external_decommission_capacity_object_operation(
+            target_pool_idx,
+            bucket,
+            (lock_object, target_object),
+            opts,
+            false,
+            operation,
+        )
+        .await
+    }
+
+    pub(super) async fn run_external_decommission_capacity_object_delete<T, F, Fut>(
+        &self,
+        target_pool_idx: usize,
+        bucket: &str,
+        lock_object: &str,
+        target_object: &str,
+        opts: ObjectOptions,
+        operation: F,
+    ) -> Result<T>
+    where
+        F: FnOnce(ObjectOptions) -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        let capacity_releasing = delete_only_releases_capacity(&opts);
+        self.run_external_decommission_capacity_object_operation(
+            target_pool_idx,
+            bucket,
+            (lock_object, target_object),
+            opts,
+            capacity_releasing,
+            operation,
+        )
+        .await
+    }
+
+    async fn run_external_decommission_capacity_object_operation<T, F, Fut>(
+        &self,
+        target_pool_idx: usize,
+        bucket: &str,
+        objects: (&str, &str),
+        mut opts: ObjectOptions,
+        capacity_releasing: bool,
+        operation: F,
+    ) -> Result<T>
+    where
+        F: FnOnce(ObjectOptions) -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        let (lock_object, target_object) = objects;
+        let (capacity_guard, has_active_decommission) = if capacity_releasing {
+            self.acquire_decommission_capacity_release_fence_with_active_source().await?
+        } else {
+            self.acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "mutation")
+                .await?
+        };
         let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
             // Active migration acquires the object namespace before its capacity
             // write. Match that order, then recheck capacity admission.
@@ -2034,9 +2091,12 @@ impl ECStore {
                 .await?;
             self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
                 .await;
-            let capacity_guard = self
-                .acquire_external_decommission_capacity_fence(&[target_pool_idx], "mutation")
-                .await?;
+            let capacity_guard = if capacity_releasing {
+                self.acquire_decommission_capacity_release_fence_with_active_source().await?.0
+            } else {
+                self.acquire_external_decommission_capacity_fence(&[target_pool_idx], "mutation")
+                    .await?
+            };
             (capacity_guard, Some(guard))
         } else {
             (capacity_guard, None)
@@ -3572,7 +3632,7 @@ impl ECStore {
             let pool_idx = pool.pool_idx;
             let pool = pool.clone();
             match self
-                .run_external_decommission_capacity_object_mutation(
+                .run_external_decommission_capacity_object_delete(
                     pool_idx,
                     bucket,
                     object,

crates/ecstore/src/store/rebalance.rs

@@ -929,7 +929,7 @@ impl ECStore {
                 results.push(RebalanceDeletePoolResult {
                     pool_idx: idx,
                     result: self
-                        .run_external_decommission_capacity_object_mutation(
+                        .run_external_decommission_capacity_object_delete(
                             idx,
                             bucket,
                             object,