fix(ilm): delete historical null versions by exact identity (#7109)

crates/scanner/src/lib.rs

@@ -43,9 +43,9 @@ use storage_api::owner::{
     ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
     ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
     ecstore_invalidate_admin_data_usage_snapshot_cache, ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure,
-    ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
-    ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
-    ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
+    ecstore_is_reserved_or_invalid_bucket, ecstore_lifecycle_version_delete_target, ecstore_list_path_raw,
+    ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
+    ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
     scanner_replication_config_for_lifecycle_eval,
 };
 #[cfg(test)]

crates/scanner/src/scanner_folder.rs

@@ -65,9 +65,9 @@ use crate::{
     Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts, ReplicationConfig,
     ReplicationHealObject, ReplicationQueueAdmission, ReplicationStatusType, STORAGE_FORMAT_FILE, ScannerDiskExt as _,
     ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, TierRegistrySnapshot, apply_expiry_rule,
-    apply_transition_rule, enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object,
-    path2_bucket_object_with_base_path, queue_replication_heal, runtime_tier_registry_for_cycle, scanner_is_erasure,
-    scanner_replication_config_for_lifecycle_eval,
+    apply_transition_rule, ecstore_lifecycle_version_delete_target, enqueue_runtime_newer_noncurrent,
+    is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, path2_bucket_object_with_base_path,
+    queue_replication_heal, runtime_tier_registry_for_cycle, scanner_is_erasure, scanner_replication_config_for_lifecycle_eval,
 };
 use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete};
 

crates/scanner/src/scanner_folder/item_actions.rs

@@ -819,14 +819,15 @@ impl ScannerItem {
                                 }
                             }
                             IlmAction::DeleteVersionAction => {
-                                if let Some(opt) = object_opts.get(i) {
-                                    to_delete_objs.push(ObjectToDelete {
-                                        object_name: opt.name.clone(),
-                                        version_id: opt.version_id,
-                                        ..Default::default()
-                                    });
+                                if let Some(target) = ecstore_lifecycle_version_delete_target(oi) {
+                                    to_delete_objs.push(target);
                                     noncurrent_events.push(event.clone());
                                     noncurrent_unknown.push(oi);
+                                } else {
+                                    if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
+                                        self.heal_actions(oi, *physical, size_summary).await;
+                                    }
+                                    size_summary.actions_accounting_unknown(oi);
                                 }
                             }
                             IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
@@ -933,12 +934,8 @@ impl ScannerItem {
                         }
                     }
                     IlmAction::DeleteVersionAction => {
-                        if let Some(opt) = object_opts.get(i) {
-                            to_delete_objs.push(ObjectToDelete {
-                                object_name: opt.name.clone(),
-                                version_id: opt.version_id,
-                                ..Default::default()
-                            });
+                        if let Some(target) = ecstore_lifecycle_version_delete_target(oi) {
+                            to_delete_objs.push(target);
                             if let Some(actual_size) = known_size {
                                 noncurrent_accounting.push(PendingScannerAccounting {
                                     object: oi,
@@ -947,8 +944,8 @@ impl ScannerItem {
                                 });
                             }
                             account_now = false;
+                            noncurrent_events.push(event.clone());
                         }
-                        noncurrent_events.push(event.clone());
                     }
                     IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
                         debug!(

crates/scanner/src/storage_api.rs

@@ -32,6 +32,7 @@ pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys a
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
     apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule,
+    lifecycle_version_delete_target as ecstore_lifecycle_version_delete_target,
 };
 pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_from_object_info as ecstore_object_opts_from_object_info;
 #[cfg(test)]
@@ -142,10 +143,10 @@ pub(crate) mod owner {
         ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
         ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
         ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_reserved_or_invalid_bucket,
-        ecstore_list_path_raw, ecstore_object_opts_from_object_info, ecstore_path2_bucket_object,
-        ecstore_path2_bucket_object_with_base_path, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
-        ecstore_resolve_object_store_handle, ecstore_save_config, ecstore_send_event,
-        scanner_replication_config_for_lifecycle_eval,
+        ecstore_lifecycle_version_delete_target, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
+        ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
+        ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
+        ecstore_send_event, scanner_replication_config_for_lifecycle_eval,
     };
 
     #[cfg(test)]

crates/scanner/tests/lifecycle_integration_test.rs

@@ -40,7 +40,7 @@ use uuid::Uuid;
 mod storage_api;
 
 use storage_api::lifecycle::{
-    BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart,
+    BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart,
     DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints,
     ExpiryState, IlmAction, LcEvent, LcEventSrc, ListOperations as _, MakeBucketOptions, MockWarmBackend,
     MultipartOperations as _, ObjectIO as _, ObjectOperations as _, PoolEndpoints, STORAGE_FORMAT_FILE, TRANSITION_PENDING,
@@ -2112,6 +2112,116 @@ mod serial_tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#2198"]
+    async fn test_scanner_expires_historical_null_without_deleting_active_version() {
+        let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
+        let bucket_name = format!("test-historical-null-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "test/object.txt";
+
+        create_test_bucket(&ecstore, bucket_name.as_str()).await;
+        update_bucket_metadata(
+            bucket_name.as_str(),
+            BUCKET_VERSIONING_CONFIG,
+            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
+        )
+        .await
+        .expect("bucket versioning should first be enabled");
+        update_bucket_metadata(
+            bucket_name.as_str(),
+            BUCKET_VERSIONING_CONFIG,
+            b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
+        )
+        .await
+        .expect("bucket versioning should be suspended");
+
+        let mut null_reader = PutObjReader::from_vec(b"historical null body".to_vec());
+        let historical_null = ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut null_reader,
+                &ObjectOptions {
+                    version_suspended: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("suspended PUT should create a null version");
+        assert_eq!(historical_null.version_id, Some(Uuid::nil()));
+
+        update_bucket_metadata(
+            bucket_name.as_str(),
+            BUCKET_VERSIONING_CONFIG,
+            b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
+        )
+        .await
+        .expect("bucket versioning should be re-enabled");
+        let active_body = b"active uuid body".to_vec();
+        let mut active_reader = PutObjReader::from_vec(active_body.clone());
+        let active = ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut active_reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("re-enabled PUT should create an active UUID version");
+        let active_version_id = active.version_id.expect("the active version should have an identity");
+        assert!(!active_version_id.is_nil());
+
+        let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>expire-historical-null</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>test/</Prefix>
+        </Filter>
+        <NoncurrentVersionExpiration>
+            <NoncurrentDays>0</NoncurrentDays>
+        </NoncurrentVersionExpiration>
+    </Rule>
+</LifecycleConfiguration>"#;
+        update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
+            .await
+            .expect("noncurrent expiration should be configured");
+        init_background_expiry(ecstore.clone()).await;
+
+        assert_eq!(object_version_count(&ecstore, bucket_name.as_str(), object_name).await, 2);
+        scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
+        assert!(
+            wait_for_version_count(&ecstore, bucket_name.as_str(), object_name, 1, Duration::from_secs(3)).await,
+            "scanner should delete only the historical null generation"
+        );
+
+        let remaining = ecstore
+            .get_object_info(bucket_name.as_str(), object_name, &ObjectOptions::default())
+            .await
+            .expect("the active UUID version must remain visible");
+        assert_eq!(remaining.version_id, Some(active_version_id));
+        assert_eq!(read_object_fully(&ecstore, bucket_name.as_str(), object_name).await, active_body);
+
+        let null_error = ecstore
+            .get_object_info(
+                bucket_name.as_str(),
+                object_name,
+                &ObjectOptions {
+                    version_id: Some(Uuid::nil().to_string()),
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("the historical null generation should be gone");
+        assert!(is_err_object_not_found(&null_error) || is_err_version_not_found(&null_error));
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
     #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]

crates/scanner/tests/storage_api/mod.rs

@@ -22,7 +22,7 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::{
     },
     lifecycle::{Event as LcEvent, IlmAction, TRANSITION_PENDING, TransitionOptions},
 };
-pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
+pub(crate) use rustfs_ecstore::api::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
 pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
     get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
 };
@@ -50,13 +50,13 @@ pub(crate) mod lifecycle {
     };
 
     pub(crate) use super::{
-        BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DeleteAfterObjectLockSnapshotBarrier, DiskOption, ECStore, EcstoreError,
-        Endpoint, EndpointServerPools, Endpoints, ExpiryState, IlmAction, LcEvent, LcEventSrc, MockWarmBackend, PoolEndpoints,
-        STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier, TransitionOptions,
-        assert_transition_meta_consistent, enqueue_transition_for_existing_objects, expire_transitioned_object,
-        free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys,
-        init_local_disks, is_err_object_not_found, is_err_version_not_found, new_disk, path2_bucket_object_with_base_path,
-        recover_transition_transaction_records, recover_transition_transaction_records_at, register_mock_tier_util,
-        update_bucket_metadata, wait_for_free_version_absence,
+        BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG, BucketVersioningSys, DeleteAfterObjectLockSnapshotBarrier, DiskOption,
+        ECStore, EcstoreError, Endpoint, EndpointServerPools, Endpoints, ExpiryState, IlmAction, LcEvent, LcEventSrc,
+        MockWarmBackend, PoolEndpoints, STORAGE_FORMAT_FILE, TRANSITION_PENDING, TransitionCleanupStoreBarrier,
+        TransitionOptions, assert_transition_meta_consistent, enqueue_transition_for_existing_objects,
+        expire_transitioned_object, free_version_count, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry,
+        init_bucket_metadata_sys, init_local_disks, is_err_object_not_found, is_err_version_not_found, new_disk,
+        path2_bucket_object_with_base_path, recover_transition_transaction_records, recover_transition_transaction_records_at,
+        register_mock_tier_util, update_bucket_metadata, wait_for_free_version_absence,
     };
 }