feat(lifecycle): improve ILM compatibility and scanner runtime config (#2534)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>

crates/scanner/Cargo.toml

@@ -55,6 +55,7 @@ metrics = { workspace = true }
 [dev-dependencies]
 tracing-subscriber = { workspace = true }
 serial_test = { workspace = true }
+temp-env = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 tokio = { workspace = true, features = ["test-util"] }
 

crates/scanner/src/scanner.rs

@@ -22,10 +22,8 @@ use crate::{DataUsageInfo, ScannerError};
 use chrono::{DateTime, Utc};
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
-use rustfs_config::DEFAULT_SCANNER_SPEED;
-use rustfs_config::ENV_SCANNER_SPEED;
-use rustfs_config::ENV_SCANNER_START_DELAY_SECS;
 use rustfs_config::ScannerSpeed;
+use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
 use rustfs_ecstore::StorageAPI as _;
 use rustfs_ecstore::config::com::{read_config, save_config};
 use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
@@ -40,11 +38,15 @@ use tracing::{debug, error, info, instrument, warn};
 
 const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
 
-/// Returns the base cycle interval. If `RUSTFS_SCANNER_START_DELAY_SECS`
-/// is set (or `RUSTFS_DATA_SCANNER_START_DELAY_SECS` as deprecated alias),
-/// it takes precedence; otherwise the value is derived from the
-/// `RUSTFS_SCANNER_SPEED` preset.
+/// Returns the base cycle interval.
+/// Priority order:
+/// 1. RUSTFS_SCANNER_CYCLE (if set, overrides everything)
+/// 2. RUSTFS_SCANNER_START_DELAY_SECS (for backward compatibility)
+/// 3. RUSTFS_SCANNER_SPEED preset
 fn cycle_interval() -> Duration {
+    if let Some(secs) = rustfs_utils::get_env_opt_u64(ENV_SCANNER_CYCLE) {
+        return Duration::from_secs(secs);
+    }
     if let Some(secs) = scanner_start_delay_secs() {
         return Duration::from_secs(secs);
     }
@@ -181,6 +183,7 @@ fn get_lock_acquire_timeout() -> Duration {
 
 #[instrument(skip_all)]
 async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
+    SCANNER_SLEEPER.refresh_from_env();
     info!("Start run data scanner cycle");
     cycle_info.current = cycle_info.next;
     let now = Instant::now();
@@ -356,6 +359,7 @@ pub async fn store_data_usage_in_backend(
 mod tests {
     use super::*;
     use serial_test::serial;
+    use temp_env::{with_var, with_var_unset};
 
     #[test]
     #[serial]
@@ -376,6 +380,42 @@ mod tests {
         assert!(delay <= Duration::from_secs(132));
     }
 
+    #[test]
+    #[serial]
+    fn test_cycle_interval_prefers_explicit_cycle_override() {
+        with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
+            with_var(ENV_SCANNER_CYCLE, Some("42"), || {
+                assert_eq!(cycle_interval(), Duration::from_secs(42));
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_cycle_interval_supports_minio_speed_alias() {
+        with_var_unset(ENV_SCANNER_SPEED, || {
+            with_var_unset(ENV_SCANNER_CYCLE, || {
+                with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
+                    with_var("MINIO_SCANNER_SPEED", Some("slowest"), || {
+                        assert_eq!(cycle_interval(), Duration::from_secs(30 * 60));
+                    });
+                });
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_cycle_interval_supports_minio_cycle_alias() {
+        with_var_unset(ENV_SCANNER_CYCLE, || {
+            with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
+                with_var("MINIO_SCANNER_CYCLE", Some("90"), || {
+                    assert_eq!(cycle_interval(), Duration::from_secs(90));
+                });
+            });
+        });
+    }
+
     #[test]
     #[serial]
     fn test_randomized_cycle_delay_handles_small_start_delay() {

crates/scanner/src/scanner_folder.rs

@@ -30,7 +30,7 @@ use rustfs_common::heal_channel::{
 };
 use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
 use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
-use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule;
+use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
 use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
 use rustfs_ecstore::bucket::lifecycle::{
     bucket_lifecycle_ops::apply_transition_rule,
@@ -444,8 +444,14 @@ impl ScannerItem {
             }
         }
 
-        if !to_delete_objs.is_empty() {
-            // TODO: enqueueNoncurrentVersions
+        if !to_delete_objs.is_empty()
+            && let Some(event) = noncurrent_events.first().cloned()
+        {
+            GLOBAL_ExpiryState
+                .write()
+                .await
+                .enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event)
+                .await;
         }
         self.alert_excessive_versions(remaining_versions, cumulative_size);
     }

crates/scanner/src/sleeper.rs

@@ -21,6 +21,13 @@ use tokio::time::Duration;
 
 const MIN_SLEEP: Duration = Duration::from_millis(1);
 
+fn scanner_env_config() -> (ScannerSpeed, bool) {
+    let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
+    let speed = ScannerSpeed::from_env_str(&speed_str);
+    let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
+    (speed, idle_mode)
+}
+
 /// When `true` (default), the scanner throttles itself between operations.
 /// When `false`, all sleeps are skipped and the scanner runs at full speed.
 pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
@@ -28,10 +35,7 @@ pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_
 /// Global scanner sleeper initialized from the `RUSTFS_SCANNER_SPEED` and
 /// `RUSTFS_SCANNER_IDLE_MODE` environment variables.
 pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
-    let speed_str = rustfs_utils::get_env_str(ENV_SCANNER_SPEED, DEFAULT_SCANNER_SPEED);
-    let speed = ScannerSpeed::from_env_str(&speed_str);
-
-    let idle_mode = rustfs_utils::get_env_bool(ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE);
+    let (speed, idle_mode) = scanner_env_config();
     SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
 
     DynamicSleeper::new(speed)
@@ -104,6 +108,13 @@ impl DynamicSleeper {
         let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner());
         *m = speed.max_sleep();
     }
+
+    /// Reload speed and idle-mode settings from the current environment.
+    pub fn refresh_from_env(&self) {
+        let (speed, idle_mode) = scanner_env_config();
+        self.update(speed);
+        SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
+    }
 }
 
 /// A timer returned by [`DynamicSleeper::timer`].  Records the instant it
@@ -138,6 +149,7 @@ impl SleepTimer {
 mod tests {
     use super::*;
     use serial_test::serial;
+    use temp_env::with_var;
 
     #[test]
     fn test_scanner_speed_presets() {
@@ -166,6 +178,26 @@ mod tests {
         assert_eq!(max_sleep, Duration::from_secs(15));
     }
 
+    #[test]
+    #[serial]
+    fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
+        let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
+        SCANNER_IDLE_MODE.store(true, Ordering::Relaxed);
+
+        let s = DynamicSleeper::new(ScannerSpeed::Fastest);
+        with_var(ENV_SCANNER_SPEED, Some("slow"), || {
+            with_var(ENV_SCANNER_IDLE_MODE, Some("false"), || {
+                s.refresh_from_env();
+                let (factor, max_sleep) = s.read_params();
+                assert_eq!(factor, 10.0);
+                assert_eq!(max_sleep, Duration::from_secs(15));
+                assert!(!SCANNER_IDLE_MODE.load(Ordering::Relaxed));
+            });
+        });
+
+        SCANNER_IDLE_MODE.store(prev_mode, Ordering::Relaxed);
+    }
+
     #[tokio::test(start_paused = true)]
     #[serial]
     async fn test_fastest_never_sleeps() {

crates/scanner/tests/lifecycle_integration_test.rs

@@ -487,6 +487,89 @@ async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usi
         .len()
 }
 
+async fn object_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
+    let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
+    let disk = new_disk(
+        &endpoint,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await
+    .expect("failed to open local disk");
+    let data = disk
+        .read_metadata(bucket, &path_join_buf(&[object, STORAGE_FORMAT_FILE]))
+        .await
+        .expect("failed to read object metadata");
+    let meta = FileMeta::load(&data).expect("failed to load file metadata");
+    meta.get_file_info_versions(bucket, object, false)
+        .expect("failed to decode file info versions")
+        .versions
+        .len()
+}
+
+async fn wait_for_version_count(disk_path: &Path, bucket: &str, object: &str, expected: usize, timeout: Duration) -> bool {
+    let deadline = tokio::time::Instant::now() + timeout;
+
+    loop {
+        if object_version_count(disk_path, bucket, object).await == expected {
+            return true;
+        }
+
+        if tokio::time::Instant::now() >= deadline {
+            return false;
+        }
+
+        tokio::time::sleep(Duration::from_millis(50)).await;
+    }
+}
+
+async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str) {
+    let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
+    endpoint.set_pool_index(0);
+    endpoint.set_set_index(0);
+    endpoint.set_disk_index(0);
+    let disk = new_disk(
+        &endpoint,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await
+    .expect("failed to open local disk");
+    let metadata_path = disk_path.join(bucket).join(object).join(STORAGE_FORMAT_FILE);
+    let relative_path = metadata_path.to_string_lossy().to_string();
+    let (_, scanner_path) = path2_bucket_object_with_base_path(disk_path.to_string_lossy().as_ref(), relative_path.as_str());
+    let file_type = fs::metadata(&metadata_path)
+        .await
+        .expect("failed to stat object metadata")
+        .file_type();
+    let lifecycle = metadata_sys::get(bucket)
+        .await
+        .expect("failed to load bucket metadata")
+        .lifecycle_config
+        .clone()
+        .map(Arc::new);
+    let item = ScannerItem {
+        path: scanner_path.clone(),
+        bucket: bucket.to_string(),
+        prefix: object.to_string(),
+        object_name: STORAGE_FORMAT_FILE.to_string(),
+        file_type,
+        lifecycle,
+        replication: None,
+        heal_enabled: false,
+        heal_bitrot: false,
+        debug: false,
+    };
+    disk.get_size(item).await.expect("scanner get_size should succeed");
+}
+
 async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
     let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
     endpoint.set_pool_index(0);
@@ -1131,4 +1214,255 @@ mod serial_tests {
             "deleted object should remain absent after scanner cleanup"
         );
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_scanner_expires_zero_day_current_version() {
+        let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
+
+        let bucket_name = format!("test-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "test/object.txt";
+
+        create_test_bucket(&ecstore, bucket_name.as_str()).await;
+        set_bucket_lifecycle(bucket_name.as_str())
+            .await
+            .expect("Failed to set lifecycle configuration");
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+
+        assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
+
+        scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
+
+        assert!(
+            wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(3)).await,
+            "scanner should delete zero-day current version after enqueueing expiry"
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_put_object_immediately_enqueues_zero_day_current_expiry() {
+        let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
+
+        let bucket_name = format!("test-put-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "expire-now.txt";
+
+        create_test_bucket(&ecstore, bucket_name.as_str()).await;
+
+        let lifecycle_xml = format!(
+            r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>test-rule</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>{object_name}</Prefix>
+        </Filter>
+        <Expiration>
+            <Days>0</Days>
+        </Expiration>
+    </Rule>
+</LifecycleConfiguration>"#
+        );
+        metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
+            .await
+            .expect("Failed to set lifecycle configuration");
+
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+
+        assert!(
+            wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await,
+            "put_object should enqueue zero-day current expiry without waiting for scanner"
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_scanner_expires_zero_day_noncurrent_version() {
+        let (disk_paths, ecstore) = setup_isolated_test_env(false).await;
+
+        let bucket_name = format!("test-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "test/object.txt";
+
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+
+        let mut reader = PutObjReader::from_vec(b"v1".to_vec());
+        ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("failed to upload v1");
+        let mut reader = PutObjReader::from_vec(b"v2".to_vec());
+        ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("failed to upload v2");
+
+        assert_eq!(object_version_count(&disk_paths[0], bucket_name.as_str(), object_name).await, 2);
+
+        let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>test-rule</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>test/</Prefix>
+        </Filter>
+        <NoncurrentVersionExpiration>
+            <NoncurrentDays>0</NoncurrentDays>
+        </NoncurrentVersionExpiration>
+    </Rule>
+</LifecycleConfiguration>"#;
+        metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
+            .await
+            .expect("Failed to set noncurrent lifecycle configuration");
+
+        rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
+
+        scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
+
+        assert!(
+            wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(3)).await,
+            "scanner should delete zero-day noncurrent versions after enqueueing expiry"
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_put_object_immediately_enqueues_zero_day_noncurrent_expiry() {
+        let (disk_paths, ecstore) = setup_isolated_test_env(true).await;
+
+        let bucket_name = format!("test-put-zero-day-noncurrent-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "test/object.txt";
+
+        create_test_lock_bucket(&ecstore, bucket_name.as_str()).await;
+
+        let lifecycle_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>test-rule</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>test/</Prefix>
+        </Filter>
+        <NoncurrentVersionExpiration>
+            <NoncurrentDays>0</NoncurrentDays>
+        </NoncurrentVersionExpiration>
+    </Rule>
+</LifecycleConfiguration>"#;
+        metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
+            .await
+            .expect("Failed to set noncurrent lifecycle configuration");
+
+        let mut reader = PutObjReader::from_vec(b"v1".to_vec());
+        ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("failed to upload v1");
+        let mut reader = PutObjReader::from_vec(b"v2".to_vec());
+        ecstore
+            .put_object(
+                bucket_name.as_str(),
+                object_name,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("failed to upload v2");
+
+        assert!(
+            wait_for_version_count(&disk_paths[0], bucket_name.as_str(), object_name, 1, Duration::from_secs(2)).await,
+            "put_object should enqueue zero-day noncurrent expiry without waiting for scanner"
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_background_scanner_expires_zero_day_current_version() {
+        let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
+
+        let bucket_name = format!("test-bg-zero-day-expire-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "test/object.txt";
+
+        create_test_bucket(&ecstore, bucket_name.as_str()).await;
+        set_bucket_lifecycle(bucket_name.as_str())
+            .await
+            .expect("Failed to set lifecycle configuration");
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+
+        let ctx = CancellationToken::new();
+        init_data_scanner(ctx.clone(), ecstore.clone()).await;
+
+        let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
+
+        ctx.cancel();
+
+        assert!(deleted, "background scanner should delete zero-day current version after startup delay");
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[serial]
+    async fn test_background_scanner_expires_zero_day_current_version_for_exact_key_prefix() {
+        let (_disk_paths, ecstore) = setup_isolated_test_env(true).await;
+
+        let bucket_name = format!("test-bg-zero-day-exact-{}", &Uuid::new_v4().simple().to_string()[..8]);
+        let object_name = "expire-now.txt";
+
+        create_test_bucket(&ecstore, bucket_name.as_str()).await;
+
+        let lifecycle_xml = format!(
+            r#"<?xml version="1.0" encoding="UTF-8"?>
+<LifecycleConfiguration>
+    <Rule>
+        <ID>test-rule</ID>
+        <Status>Enabled</Status>
+        <Filter>
+            <Prefix>{object_name}</Prefix>
+        </Filter>
+        <Expiration>
+            <Days>0</Days>
+        </Expiration>
+    </Rule>
+</LifecycleConfiguration>"#
+        );
+        metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
+            .await
+            .expect("Failed to set lifecycle configuration");
+        upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+
+        let ctx = CancellationToken::new();
+        init_data_scanner(ctx.clone(), ecstore.clone()).await;
+
+        let deleted = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(12)).await;
+
+        ctx.cancel();
+
+        assert!(deleted, "background scanner should delete zero-day exact-key lifecycle targets");
+    }
 }