fix(replication): persist force-delete handoff state (#5641)

* fix(replication): persist force-delete handoff state

* fix(arch): route force-delete config access through boundary

* style: format force-delete imports

---------

Co-authored-by: Zhengchao An <anzhengchao@gmail.com>

rustfs/src/admin/handlers/replication.rs

@@ -1125,6 +1125,7 @@ mod tests {
                     delete_marker: false,
                     delete_marker_mtime: None,
                     target_arns: vec!["arn-a".to_string(), "arn-durable-only".to_string()],
+                    ..Default::default()
                 },
                 MrfReplicateEntry {
                     bucket: "other-bucket".to_string(),
@@ -1138,6 +1139,7 @@ mod tests {
                     delete_marker: false,
                     delete_marker_mtime: None,
                     target_arns: Vec::new(),
+                    ..Default::default()
                 },
             ],
         };
@@ -1197,6 +1199,7 @@ mod tests {
                 delete_marker: false,
                 delete_marker_mtime: None,
                 target_arns: Vec::new(),
+                ..Default::default()
             }],
         };
 

rustfs/src/app/object_usecase.rs

@@ -41,11 +41,11 @@ use super::storage_api::object_usecase::bucket::{
     predict_lifecycle_expiration,
     quota::{QuotaCheckResult, QuotaError, QuotaOperation},
     replication::{
-        DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, delete_replication_state_from_config,
-        delete_replication_version_id, deleted_object_has_pending_replication_delete, has_active_delete_rule,
-        load_delete_config_snapshot, must_replicate_object, schedule_object_replication, schedule_replication_delete,
-        schedule_replication_deletes, set_deleted_object_replication_state, should_schedule_delete_replication,
-        should_use_existing_delete_replication_info,
+        DeleteReplicationConfigSnapshot, REPLICATE_INCOMING_DELETE, ReplicationStatusType, commit_force_delete_intent,
+        delete_replication_state_from_config, delete_replication_version_id, deleted_object_has_pending_replication_delete,
+        force_delete_target_set, has_active_delete_rule, load_delete_config_snapshot, must_replicate_object,
+        persist_force_delete_intent, schedule_object_replication, schedule_replication_delete, schedule_replication_deletes,
+        set_deleted_object_replication_state, should_schedule_delete_replication, should_use_existing_delete_replication_info,
     },
     tagging::decode_tags,
     validate_restore_request,
@@ -7137,6 +7137,7 @@ impl DefaultObjectUsecase {
         opts.expected_current_version_id = expected_current_version_id.clone();
 
         let replicate_force_delete = force_delete && !replica && has_active_delete_rule(&delete_config_snapshot, &key);
+        let mut force_delete_intent = None;
 
         // Check Object Lock retention before deletion
         // TODO: Future optimization (separate PR) - If performance becomes critical under high delete load:
@@ -7176,6 +7177,17 @@ impl DefaultObjectUsecase {
             let _ = invalidate_object_data_cache_before_mutation(&cache_adapter, &bucket, &key).await;
         }
 
+        if replicate_force_delete
+            && let Some((target_arns, generation)) = force_delete_target_set(&delete_config_snapshot, &key)
+            && !target_arns.is_empty()
+        {
+            let operation_id =
+                persist_force_delete_intent(store.clone(), bucket.clone(), key.clone(), target_arns.clone(), generation)
+                    .await
+                    .map_err(ApiError::from)?;
+            force_delete_intent = Some((operation_id, target_arns, generation));
+        }
+
         let obj_info = {
             match store
                 .delete_object_with_tier_delete_journal(&bucket, &key, opts.clone())
@@ -7183,6 +7195,18 @@ impl DefaultObjectUsecase {
             {
                 Ok(obj) => obj,
                 Err(err) => {
+                    if let Some((operation_id, _, _)) = force_delete_intent.as_ref()
+                        && let Err(cleanup_error) =
+                            crate::storage::storage_api::complete_force_delete_intent(store.clone(), *operation_id).await
+                    {
+                        warn!(
+                            bucket = %bucket,
+                            object = %key,
+                            operation_id = %operation_id,
+                            error = %cleanup_error,
+                            "failed to remove uncommitted force-delete intent after local delete failure"
+                        );
+                    }
                     if is_err_bucket_not_found(&err) {
                         return Err(S3Error::with_message(S3ErrorCode::NoSuchBucket, "Bucket not found".to_string()));
                     }
@@ -7211,7 +7235,31 @@ impl DefaultObjectUsecase {
         }
 
         if obj_info.name.is_empty() {
-            if replicate_force_delete {
+            if let Some((operation_id, target_arns, generation)) = force_delete_intent {
+                if let Err(error) = commit_force_delete_intent(store.clone(), operation_id).await {
+                    warn!(
+                        bucket = %bucket,
+                        object = %key,
+                        operation_id = %operation_id,
+                        error = %error,
+                        "failed to mark force-delete intent committed after local delete"
+                    );
+                }
+                let generation = i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX);
+                schedule_replication_delete(
+                    StorageDeletedObject {
+                        object_name: key.clone(),
+                        force_delete: true,
+                        force_delete_id: Some(operation_id),
+                        force_delete_target_arns: target_arns,
+                        force_delete_generation: Some(generation),
+                        ..Default::default()
+                    },
+                    bucket.clone(),
+                    REPLICATE_INCOMING_DELETE.to_string(),
+                )
+                .await;
+            } else if replicate_force_delete {
                 let mut delete_object = StorageDeletedObject {
                     object_name: key.clone(),
                     force_delete: true,

rustfs/src/app/storage_api.rs

@@ -577,6 +577,44 @@ pub(crate) mod bucket {
         #[cfg(test)]
         pub(crate) use replication_contracts::replication_statuses_map;
 
+        pub(crate) async fn persist_force_delete_intent(
+            store: Arc<crate::storage::storage_api::ECStore>,
+            bucket: String,
+            object: String,
+            target_arns: Vec<String>,
+            generation: time::OffsetDateTime,
+        ) -> crate::storage::storage_api::Result<Uuid> {
+            let operation_id = Uuid::new_v4();
+            crate::storage::storage_api::persist_force_delete_intent(
+                store,
+                replication_contracts::MrfReplicateEntry {
+                    bucket,
+                    object,
+                    version_id: None,
+                    retry_count: 0,
+                    size: 0,
+                    op: replication_contracts::MrfOpKind::Delete,
+                    force_delete: true,
+                    delete_marker_version_id: None,
+                    delete_marker: false,
+                    delete_marker_mtime: None,
+                    target_arns,
+                    force_delete_id: Some(operation_id),
+                    force_delete_generation: Some(i64::try_from(generation.unix_timestamp_nanos()).unwrap_or(i64::MAX)),
+                    force_delete_local_commit: false,
+                },
+            )
+            .await
+            .map(|_| operation_id)
+        }
+
+        pub(crate) async fn commit_force_delete_intent(
+            store: Arc<crate::storage::storage_api::ECStore>,
+            operation_id: Uuid,
+        ) -> crate::storage::storage_api::Result<()> {
+            crate::storage::storage_api::commit_force_delete_intent(store, operation_id).await
+        }
+
         /// Test-only counter of `must_replicate_object` invocations.
         ///
         /// Used by white-box regression tests to assert that a single PUT
@@ -614,6 +652,13 @@ pub(crate) mod bucket {
             ReplicationObjectBridge::has_active_delete_rule(snapshot, object)
         }
 
+        pub(crate) fn force_delete_target_set(
+            snapshot: &DeleteReplicationConfigSnapshot,
+            prefix: &str,
+        ) -> Option<(Vec<String>, time::OffsetDateTime)> {
+            ReplicationObjectBridge::force_delete_target_set(snapshot, prefix)
+        }
+
         pub(crate) fn delete_replication_version_id(
             replication_source: &crate::storage::storage_api::StorageObjectInfo,
             deleted_delete_marker_version: bool,

rustfs/src/storage/storage_api.rs

@@ -1441,6 +1441,21 @@ pub(crate) async fn get_bucket_replication_config(
     ecstore_bucket::metadata_sys::get_replication_config(bucket).await
 }
 
+pub(crate) async fn persist_force_delete_intent(
+    api: Arc<ECStore>,
+    entry: ecstore_bucket::replication::MrfReplicateEntry,
+) -> Result<()> {
+    ecstore_bucket::replication::persist_force_delete_intent(api, entry).await
+}
+
+pub(crate) async fn commit_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
+    ecstore_bucket::replication::commit_force_delete_intent(api, operation_id).await
+}
+
+pub(crate) async fn complete_force_delete_intent(api: Arc<ECStore>, operation_id: uuid::Uuid) -> Result<()> {
+    ecstore_bucket::replication::complete_force_delete_intent(api, operation_id).await
+}
+
 pub(crate) async fn get_bucket_request_payment_config(
     bucket: &str,
 ) -> Result<(s3s::dto::RequestPaymentConfiguration, time::OffsetDateTime)> {