fix(ecstore): finalize decommission capacity recovery (#6955)

crates/ecstore/src/store/init.rs

@@ -562,6 +562,7 @@ impl ECStore {
             decommission_cancelers,
             start_gate: Mutex::new(()),
             pool_meta_save_gate: Mutex::new(PoolMetaWriteState::for_startup(deployment_id, fresh_bootstrap_proven)),
+            decommission_capacity_entry_gate: Mutex::default(),
             // Adopt the caller's context (the process bootstrap one on the
             // legacy path) so startup writes (erasure type recorded before
             // this point) and later reads share one cell.
@@ -2427,45 +2428,6 @@ mod tests {
         );
     }
 
-    async fn assert_suspended_decommission_converged(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
-        let source_versions = store.pools[0]
-            .get_disks_by_key(object)
-            .load_file_info_versions_exact(bucket, object)
-            .await
-            .expect("source versions should remain readable after suspended convergence");
-        assert!(
-            source_versions.is_none_or(|versions| versions.versions.is_empty()),
-            "worker convergence must remove only the decommissioned source null version"
-        );
-
-        let target_versions = store.pools[1]
-            .get_disks_by_key(object)
-            .load_file_info_versions_exact(bucket, object)
-            .await
-            .expect("active target versions should be readable")
-            .expect("active target must retain the suspended DELETE marker");
-        assert!(
-            matches!(target_versions.versions.as_slice(), [marker] if marker.deleted && marker.version_id.is_none_or(|version_id| version_id.is_nil())),
-            "active target must contain only its null delete marker: {target_versions:?}"
-        );
-
-        let err = store
-            .get_object_info(
-                bucket,
-                object,
-                &ObjectOptions {
-                    version_suspended: true,
-                    ..Default::default()
-                },
-            )
-            .await
-            .expect_err("the active null delete marker must hide the migrated source generation");
-        assert!(
-            matches!(err, StorageError::ObjectNotFound(_, _)),
-            "unexpected suspended latest-object result: {err:?}"
-        );
-    }
-
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn tag_updates_skip_active_rebalance_source_pool() {
@@ -4216,13 +4178,7 @@ mod tests {
             .put_object(&bucket, &object, &mut source, &ObjectOptions::default())
             .await
             .expect("write source object to the pool being decommissioned");
-        {
-            let mut pool_meta = store.pool_meta.write().await;
-            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
-                start_time: Some(OffsetDateTime::now_utc()),
-                ..Default::default()
-            });
-        }
+        mark_test_pool_decommissioning(&store, 0).await;
         assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
 
         let barrier = crate::set_disk::PutObjectCommitBarrier::install(
@@ -4958,9 +4914,10 @@ mod tests {
                 .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
             let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
             let source_set = store.pools[0].get_disks_by_key(object);
+            let recovery_source_set = Arc::clone(&source_set);
             let worker_store = Arc::clone(&store);
             let worker_bucket = bucket.clone();
-            let worker = tokio::spawn(async move {
+            let mut worker = tokio::spawn(async move {
                 worker_store
                     .decommission_entry_for_test(
                         0,
@@ -4974,7 +4931,10 @@ mod tests {
                     .await
             });
 
-            barrier.wait_until_paused().await;
+            tokio::select! {
+                () = barrier.wait_until_paused() => {}
+                result = &mut worker => panic!("decommission multipart worker exited before the commit barrier: {result:?}"),
+            }
             loss_hook.mark_lost();
             barrier.release();
             drop(barrier);
@@ -4996,6 +4956,19 @@ mod tests {
                 .await
                 .expect("list target multipart uploads after fenced migration");
             assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
+            drop(loss_hook);
+            store
+                .decommission_entry_for_test(
+                    0,
+                    MetaCacheEntry {
+                        name: object.to_string(),
+                        ..Default::default()
+                    },
+                    bucket.clone(),
+                    recovery_source_set,
+                )
+                .await
+                .expect("same-mutation retry should recover the durable capacity intent");
         }
 
         shutdown.cancel();
@@ -5204,6 +5177,12 @@ mod tests {
             .expect("self-copy should keep using the committed active target");
         assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
 
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            pool_meta.pools[1].decommission = None;
+        }
+        mark_test_pool_decommissioning(&store, 1).await;
+
         let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
         let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
             &bucket,
@@ -6217,28 +6196,27 @@ mod tests {
         write_suspended_decommission_source(&store, &bucket, object).await;
         mark_test_pool_decommissioning(&store, 0).await;
 
-        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
-        let delete_store = Arc::clone(&store);
-        let delete_bucket = bucket.clone();
-        let delete = tokio::spawn(async move {
-            delete_store
-                .delete_object(
-                    &delete_bucket,
-                    object,
-                    ObjectOptions {
-                        version_suspended: true,
-                        ..Default::default()
-                    },
-                )
-                .await
-        });
-        delete_barrier.wait_until_paused().await;
+        let delete_err = store
+            .delete_object(
+                &bucket,
+                object,
+                ObjectOptions {
+                    version_suspended: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("capacity-reserved target must reject a concurrent suspended DELETE");
+        assert!(
+            matches!(delete_err, Error::SlowDown),
+            "unexpected suspended DELETE result: {delete_err:?}"
+        );
         assert_suspended_null_source_present(&store, &bucket, object).await;
 
         let source_set = store.pools[0].get_disks_by_key(object);
         let worker_store = Arc::clone(&store);
         let worker_bucket = bucket.clone();
-        let worker = tokio::spawn(async move {
+        tokio::spawn(async move {
             worker_store
                 .decommission_entry_for_test(
                     0,
@@ -6250,25 +6228,25 @@ mod tests {
                     source_set,
                 )
                 .await
-        });
+        })
+        .await
+        .expect("suspended decommission worker should join")
+        .expect("worker must migrate the fenced suspended source");
 
-        delete_barrier.release();
-        let marker = delete
-            .await
-            .expect("suspended DELETE task should join")
-            .expect("suspended DELETE should commit its active-pool marker");
-        drop(delete_barrier);
-        assert!(marker.delete_marker, "suspended DELETE must create a marker");
-        assert!(
-            marker.version_id.is_none_or(|version_id| version_id.is_nil()),
-            "suspended DELETE marker must keep the null version identity"
+        assert_decommission_source_absent(
+            &store,
+            &bucket,
+            object,
+            &ObjectOptions {
+                version_suspended: true,
+                ..Default::default()
+            },
+        )
+        .await;
+        assert_eq!(
+            read_decommission_target_body(&store, &bucket, object, &ObjectOptions::default()).await,
+            b"suspended source generation"
         );
-        worker
-            .await
-            .expect("suspended decommission worker should join")
-            .expect("worker must treat the newer active null marker as a completed migration");
-
-        assert_suspended_decommission_converged(&store, &bucket, object).await;
         shutdown.cancel();
     }
 
@@ -6301,31 +6279,29 @@ mod tests {
                 },
                 None,
             ));
-        let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
-        let delete_store = Arc::clone(&store);
-        let delete_bucket = bucket.clone();
-        let delete = tokio::spawn(async move {
-            delete_store
-                .delete_objects(
-                    &delete_bucket,
-                    vec![ObjectToDelete {
-                        object_name: object.to_string(),
-                        ..Default::default()
-                    }],
-                    ObjectOptions {
-                        delete_replication_config_snapshot: Some(delete_config_snapshot),
-                        ..Default::default()
-                    },
-                )
-                .await
-        });
-        delete_barrier.wait_until_paused().await;
+        let (_deleted, errors) = store
+            .delete_objects(
+                &bucket,
+                vec![ObjectToDelete {
+                    object_name: object.to_string(),
+                    ..Default::default()
+                }],
+                ObjectOptions {
+                    delete_replication_config_snapshot: Some(delete_config_snapshot),
+                    ..Default::default()
+                },
+            )
+            .await;
+        assert!(
+            matches!(errors.as_slice(), [Some(Error::SlowDown)]),
+            "unexpected suspended batch DELETE result: {errors:?}"
+        );
         assert_suspended_null_source_present(&store, &bucket, object).await;
 
         let source_set = store.pools[0].get_disks_by_key(object);
         let worker_store = Arc::clone(&store);
         let worker_bucket = bucket.clone();
-        let worker = tokio::spawn(async move {
+        tokio::spawn(async move {
             worker_store
                 .decommission_entry_for_test(
                     0,
@@ -6337,22 +6313,25 @@ mod tests {
                     source_set,
                 )
                 .await
-        });
+        })
+        .await
+        .expect("suspended batch decommission worker should join")
+        .expect("worker must migrate the batch-fenced suspended source");
 
-        delete_barrier.release();
-        let (deleted, errors) = delete.await.expect("suspended batch DELETE task should join");
-        drop(delete_barrier);
-        assert!(errors.iter().all(Option::is_none), "suspended batch DELETE should succeed: {errors:?}");
-        assert!(
-            matches!(deleted.as_slice(), [marker] if marker.delete_marker && marker.delete_marker_version_id.is_none_or(|version_id| version_id.is_nil())),
-            "suspended batch DELETE must create one null marker: {deleted:?}"
+        assert_decommission_source_absent(
+            &store,
+            &bucket,
+            object,
+            &ObjectOptions {
+                version_suspended: true,
+                ..Default::default()
+            },
+        )
+        .await;
+        assert_eq!(
+            read_decommission_target_body(&store, &bucket, object, &ObjectOptions::default()).await,
+            b"suspended source generation"
         );
-        worker
-            .await
-            .expect("suspended batch decommission worker should join")
-            .expect("worker must treat the newer batch null marker as a completed migration");
-
-        assert_suspended_decommission_converged(&store, &bucket, object).await;
         shutdown.cancel();
     }