fix(tier): sweep transitioned copies from delete handlers (#2287)

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -45,7 +45,7 @@ use lazy_static::lazy_static;
 use rustfs_common::data_usage::TierStats;
 use rustfs_common::heal_channel::rep_has_active_rules;
 use rustfs_common::metrics::{IlmAction, Metrics};
-use rustfs_filemeta::{NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
+use rustfs_filemeta::{FileInfo, NULL_VERSION_ID, RestoreStatusOps, is_restored_object_on_disk};
 use rustfs_s3_common::EventName;
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
 use s3s::Body;
@@ -393,12 +393,81 @@ impl ExpiryState {
                         //delete_object_versions(api, &v.bucket, &v.versions, v.event).await;
                     }
                     else if v.as_any().is::<Jentry>() {
-                        //transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+                        let v = v.as_any().downcast_ref::<Jentry>().expect("err!");
+                        if let Err(err) = delete_object_from_remote_tier(&v.obj_name, &v.version_id, &v.tier_name).await {
+                            warn!(
+                                object = %v.obj_name,
+                                version_id = %v.version_id,
+                                tier = %v.tier_name,
+                                error = ?err,
+                                "failed to delete transitioned object from remote tier"
+                            );
+                        }
                     }
                     else if v.as_any().is::<FreeVersionTask>() {
                         let v = v.as_any().downcast_ref::<FreeVersionTask>().expect("err!");
-                        let _oi = v.0.clone();
+                        let oi = v.0.clone();
+                        if let Err(err) = delete_object_from_remote_tier(
+                            &oi.transitioned_object.name,
+                            &oi.transitioned_object.version_id,
+                            &oi.transitioned_object.tier,
+                        )
+                        .await
+                        {
+                            warn!(
+                                bucket = %oi.bucket,
+                                object = %oi.name,
+                                remote_object = %oi.transitioned_object.name,
+                                remote_version_id = %oi.transitioned_object.version_id,
+                                tier = %oi.transitioned_object.tier,
+                                error = ?err,
+                                "failed to sweep transitioned free version from remote tier"
+                            );
+                            continue;
+                        }
 
+                        let mut fi = FileInfo {
+                            name: oi.name.clone(),
+                            version_id: oi.version_id,
+                            deleted: true,
+                            ..Default::default()
+                        };
+                        fi.set_tier_free_version();
+
+                        let mut deleted_locally = false;
+                        for pool in api.pools.iter() {
+                            let set = pool.get_disks_by_key(&oi.name);
+                            match set.delete_object_version(&oi.bucket, &oi.name, &fi, false).await {
+                                Ok(()) => {
+                                    deleted_locally = true;
+                                    break;
+                                }
+                                Err(err) if is_err_version_not_found(&err) || is_err_object_not_found(&err) => continue,
+                                Err(err) => {
+                                    warn!(
+                                        bucket = %oi.bucket,
+                                        object = %oi.name,
+                                        remote_object = %oi.transitioned_object.name,
+                                        remote_version_id = %oi.transitioned_object.version_id,
+                                        tier = %oi.transitioned_object.tier,
+                                        error = ?err,
+                                        "failed to delete transitioned free version after remote tier sweep"
+                                    );
+                                    break;
+                                }
+                            }
+                        }
+
+                        if !deleted_locally {
+                            warn!(
+                                bucket = %oi.bucket,
+                                object = %oi.name,
+                                remote_object = %oi.transitioned_object.name,
+                                remote_version_id = %oi.transitioned_object.version_id,
+                                tier = %oi.transitioned_object.tier,
+                                "transitioned free version was not found during local cleanup"
+                            );
+                        }
                     }
                     else {
                         //info!("Invalid work type - {:?}", v);

crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs

@@ -120,9 +120,9 @@ impl ObjSweeper {
 #[derive(Debug, Clone)]
 #[allow(unused_assignments)]
 pub struct Jentry {
-    obj_name: String,
-    version_id: String,
-    tier_name: String,
+    pub(crate) obj_name: String,
+    pub(crate) version_id: String,
+    pub(crate) tier_name: String,
 }
 
 impl ExpiryOp for Jentry {
@@ -147,5 +147,37 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
     w.remove(obj_name, rv_id).await
 }
 
+pub fn transitioned_delete_journal_entry(
+    version_id: Option<Uuid>,
+    versioned: bool,
+    suspended: bool,
+    transitioned: &TransitionedObject,
+) -> Option<Jentry> {
+    let sweeper = ObjSweeper {
+        version_id,
+        versioned,
+        suspended,
+        transition_status: transitioned.status.clone(),
+        transition_tier: transitioned.tier.clone(),
+        transition_version_id: transitioned.version_id.clone(),
+        remote_object: transitioned.name.clone(),
+        ..Default::default()
+    };
+
+    sweeper.should_remove_remote_object()
+}
+
+pub fn transitioned_force_delete_journal_entry(transitioned: &TransitionedObject) -> Option<Jentry> {
+    if transitioned.status != lifecycle::TRANSITION_COMPLETE {
+        return None;
+    }
+
+    Some(Jentry {
+        obj_name: transitioned.name.clone(),
+        version_id: transitioned.version_id.clone(),
+        tier_name: transitioned.tier.clone(),
+    })
+}
+
 #[cfg(test)]
 mod test {}

crates/ecstore/src/set_disk.rs

@@ -1077,7 +1077,7 @@ impl ObjectOperations for SetDisks {
 
         let (mut metas, errs) = {
             if let Some(vid) = &src_opts.version_id {
-                Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false).await?
+                Self::read_all_fileinfo(&disks, "", src_bucket, src_object, vid, true, false, false).await?
             } else {
                 Self::read_all_xl(&disks, src_bucket, src_object, true, false).await
             }
@@ -1565,8 +1565,6 @@ impl ObjectOperations for SetDisks {
             return Ok(oi);
         }
 
-        let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
-
         // Create a single object deletion request
         let mut dfi = FileInfo {
             name: object.to_string(),
@@ -1673,7 +1671,7 @@ impl ObjectOperations for SetDisks {
 
         let (metas, errs) = {
             if let Some(version_id) = &opts.version_id {
-                Self::read_all_fileinfo(&disks, "", bucket, object, version_id.to_string().as_str(), false, false).await?
+                Self::read_all_fileinfo(&disks, "", bucket, object, version_id.to_string().as_str(), false, false, false).await?
             } else {
                 Self::read_all_xl(&disks, bucket, object, false, false).await
             }
@@ -3279,7 +3277,7 @@ impl HealOperations for SetDisks {
         let disks = self.disks.read().await;
 
         let disks = disks.clone();
-        let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false).await?;
+        let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
         if DiskError::is_all_not_found(&errs) {
             warn!(
                 "heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",

crates/ecstore/src/set_disk/heal.rs

@@ -56,7 +56,8 @@ impl SetDisks {
             }
         };
 
-        let (mut parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true).await?;
+        let (mut parts_metadata, errs) =
+            Self::read_all_fileinfo(&disks, "", bucket, object, version_id, true, true, false).await?;
 
         info!(
             parts_count = parts_metadata.len(),

crates/ecstore/src/set_disk/multipart.rs

@@ -105,7 +105,8 @@ impl SetDisks {
         let disks = disks.clone();
 
         let (parts_metadata, errs) =
-            Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false).await?;
+            Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_id_path, "", false, false, false)
+                .await?;
 
         let map_err_notfound = |err: DiskError| {
             if err == DiskError::FileNotFound {

crates/ecstore/src/set_disk/read.rs

@@ -131,6 +131,7 @@ impl SetDisks {
         Ok(ret)
     }
 
+    #[allow(clippy::too_many_arguments)]
     #[tracing::instrument(level = "debug", skip(disks))]
     pub(super) async fn read_all_fileinfo(
         disks: &[Option<DiskStore>],
@@ -140,13 +141,14 @@ impl SetDisks {
         version_id: &str,
         read_data: bool,
         healing: bool,
+        incl_free_versions: bool,
     ) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>)> {
         let mut ress = Vec::with_capacity(disks.len());
         let mut errors = Vec::with_capacity(disks.len());
         let opts = Arc::new(ReadOptions {
+            incl_free_versions,
             read_data,
             healing,
-            ..Default::default()
         });
         let org_bucket = Arc::new(org_bucket.to_string());
         let bucket = Arc::new(bucket.to_string());
@@ -474,7 +476,8 @@ impl SetDisks {
         let vid = opts.version_id.clone().unwrap_or_default();
 
         // TODO: optimize concurrency and break once enough slots are available
-        let (parts_metadata, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false).await?;
+        let (parts_metadata, errs) =
+            Self::read_all_fileinfo(&disks, "", bucket, object, vid.as_str(), read_data, false, opts.incl_free_versions).await?;
         // warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
         // warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
 
@@ -541,6 +544,9 @@ impl SetDisks {
         }
 
         if fi.deleted {
+            if opts.incl_free_versions && fi.tier_free_version() && opts.version_id.is_some() {
+                return (oi, write_quorum, None);
+            }
             return if opts.version_id.is_none() || opts.delete_marker {
                 (oi, write_quorum, Some(to_object_err(StorageError::FileNotFound, vec![bucket, object])))
             } else {

crates/ecstore/src/store/object.rs

@@ -286,7 +286,9 @@ impl ECStore {
         }
 
         if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
-            return self.delete_object_from_all_pools(bucket, object, &opts, errs).await;
+            let mut obj = self.delete_object_from_all_pools(bucket, object, &opts, errs).await?;
+            obj.name = decode_dir_object(object);
+            return Ok(obj);
         }
 
         for pool in self.pools.iter() {

crates/ecstore/src/store_api/types.rs

@@ -47,6 +47,7 @@ pub struct ObjectOptions {
 
     pub versioned: bool,
     pub version_suspended: bool,
+    pub incl_free_versions: bool,
 
     pub skip_decommissioned: bool,
     pub skip_rebalancing: bool,