fix(scanner): preserve background heal compatibility (#3041)

crates/scanner/src/scanner.rs

@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
-use crate::scanner_folder::data_usage_update_dir_cycles;
+use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
 use crate::scanner_io::ScannerIO;
 use crate::sleeper::SCANNER_SLEEPER;
 use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
@@ -23,7 +23,10 @@ use chrono::{DateTime, Utc};
 use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::{CurrentCycle, Metric, Metrics, emit_scan_cycle_complete, global_metrics};
 use rustfs_config::ScannerSpeed;
-use rustfs_config::{DEFAULT_SCANNER_SPEED, ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
+use rustfs_config::{
+    DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_SPEED, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CYCLE,
+    ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
+};
 use rustfs_ecstore::StorageAPI as _;
 use rustfs_ecstore::config::com::{read_config, save_config};
 use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
@@ -80,7 +83,7 @@ fn initial_scanner_delay() -> Duration {
 fn initial_scanner_delay_for(start_delay_secs: Option<u64>) -> Duration {
     start_delay_secs
         .map(|secs| randomized_cycle_delay_for(Duration::from_secs(secs)))
-        .unwrap_or_else(|| Duration::from_secs(rand::random::<u64>() % 5))
+        .unwrap_or_else(randomized_cycle_delay)
 }
 
 pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
@@ -111,9 +114,60 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
     });
 }
 
-fn get_cycle_scan_mode(_current_cycle: u64, _bitrot_start_cycle: u64, _bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
-    // TODO: from config
-    HealScanMode::Normal
+fn bitrot_scan_cycle() -> Option<Duration> {
+    let Ok(value) = std::env::var(ENV_SCANNER_BITROT_CYCLE_SECS) else {
+        return Some(Duration::from_secs(DEFAULT_SCANNER_BITROT_CYCLE_SECS));
+    };
+
+    match value.trim().to_ascii_lowercase().as_str() {
+        "0" | "true" | "on" | "yes" => Some(Duration::ZERO),
+        "false" | "off" | "no" | "disabled" => None,
+        value => value.parse::<u64>().ok().map(Duration::from_secs).or_else(|| {
+            warn!(
+                env = ENV_SCANNER_BITROT_CYCLE_SECS,
+                value,
+                default_secs = DEFAULT_SCANNER_BITROT_CYCLE_SECS,
+                "Invalid scanner bitrot cycle, using default"
+            );
+            Some(Duration::from_secs(DEFAULT_SCANNER_BITROT_CYCLE_SECS))
+        }),
+    }
+}
+
+fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
+    let Some(bitrot_cycle) = bitrot_scan_cycle() else {
+        return HealScanMode::Normal;
+    };
+
+    if bitrot_cycle.is_zero() {
+        return HealScanMode::Deep;
+    }
+
+    if current_cycle.saturating_sub(bitrot_start_cycle) < heal_object_select_prob() as u64 {
+        return HealScanMode::Deep;
+    }
+
+    let Some(bitrot_start_time) = bitrot_start_time else {
+        return HealScanMode::Deep;
+    };
+
+    let elapsed = Utc::now()
+        .signed_duration_since(bitrot_start_time)
+        .to_std()
+        .unwrap_or(Duration::ZERO);
+    if elapsed >= bitrot_cycle {
+        HealScanMode::Deep
+    } else {
+        HealScanMode::Normal
+    }
+}
+
+fn retain_recent_cycle_completions(cycle_completed: &mut Vec<DateTime<Utc>>) {
+    let keep = data_usage_update_dir_cycles() as usize;
+    if cycle_completed.len() > keep {
+        let drop_count = cycle_completed.len() - keep;
+        cycle_completed.drain(..drop_count);
+    }
 }
 
 /// Background healing information
@@ -238,9 +292,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
 
     info!(duration = ?now.elapsed(), cycles_total=cycle_info.cycle_completed.len(), "Success run data scanner cycle");
 
-    if cycle_info.cycle_completed.len() >= data_usage_update_dir_cycles() as usize {
-        cycle_info.cycle_completed = cycle_info.cycle_completed.split_off(data_usage_update_dir_cycles() as usize);
-    }
+    retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
 
     global_metrics().set_cycle(Some(cycle_info.clone())).await;
 
@@ -382,6 +434,16 @@ mod tests {
         assert!(delay <= Duration::from_secs(132));
     }
 
+    #[test]
+    #[serial]
+    fn test_initial_scanner_delay_uses_cycle_without_explicit_start_delay() {
+        with_var(ENV_SCANNER_CYCLE, Some("120"), || {
+            let delay = initial_scanner_delay_for(None);
+            assert!(delay >= Duration::from_secs(108));
+            assert!(delay <= Duration::from_secs(132));
+        });
+    }
+
     #[test]
     #[serial]
     fn test_cycle_interval_prefers_explicit_cycle_override() {
@@ -426,4 +488,46 @@ mod tests {
         assert!(delay >= Duration::from_secs(1), "expected delay >= 1s");
         assert!(delay < Duration::from_secs(2), "expected delay < 2s");
     }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
+            let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()));
+            assert_eq!(mode, HealScanMode::Deep);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_respects_elapsed_bitrot_cycle() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
+            let recent = Utc::now() - chrono::Duration::minutes(30);
+            let old = Utc::now() - chrono::Duration::hours(2);
+
+            assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal);
+            assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep);
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
+        with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
+            assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal);
+        });
+    }
+
+    #[test]
+    fn test_retain_recent_cycle_completions_keeps_last_entries() {
+        let base = Utc::now();
+        let keep = data_usage_update_dir_cycles() as usize;
+        let mut completed: Vec<_> = (0..keep + 2).map(|i| base + chrono::Duration::seconds(i as i64)).collect();
+
+        retain_recent_cycle_completions(&mut completed);
+
+        assert_eq!(completed.len(), keep);
+        assert_eq!(completed.first().copied(), Some(base + chrono::Duration::seconds(2)));
+        assert_eq!(completed.last().copied(), Some(base + chrono::Duration::seconds((keep + 1) as i64)));
+    }
 }

crates/scanner/src/scanner_folder.rs

@@ -69,9 +69,13 @@ const ENV_FAILED_OBJECTS_MAX: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX";
 const DEFAULT_FAILED_OBJECT_TTL_SECS: u32 = 86_400;
 const DEFAULT_FAILED_OBJECTS_MAX: u32 = 10_000;
 const METRIC_SCANNER_INLINE_HEAL_TOTAL: &str = "rustfs_scanner_inline_heal_total";
+const METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL: &str = "rustfs_scanner_excess_object_versions_total";
+const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_excess_object_version_size_total";
+const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total";
 
 static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
 static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
+static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
 
 pub fn data_usage_update_dir_cycles() -> u32 {
     rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
@@ -102,6 +106,50 @@ fn ensure_scanner_inline_heal_metric_registered() {
     });
 }
 
+fn ensure_scanner_alert_metrics_registered() {
+    SCANNER_ALERT_METRICS_ONCE.call_once(|| {
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL,
+            "Total scanner alerts for objects with too many retained versions."
+        );
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL,
+            "Total scanner alerts for objects whose retained versions exceed the cumulative size threshold."
+        );
+        describe_counter!(
+            METRIC_SCANNER_EXCESS_FOLDERS_TOTAL,
+            "Total scanner alerts for folders with too many direct subfolders."
+        );
+    });
+}
+
+fn scanner_excess_versions_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS,
+    )
+}
+
+fn scanner_excess_version_size_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE,
+    )
+}
+
+fn scanner_excess_folders_threshold() -> u64 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS,
+        rustfs_config::DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS,
+    )
+}
+
+fn should_alert_excessive_versions(remaining_versions: usize, cumulative_size: i64) -> (bool, bool) {
+    let too_many_versions = remaining_versions as u64 >= scanner_excess_versions_threshold();
+    let too_large_versions = cumulative_size > 0 && cumulative_size as u64 >= scanner_excess_version_size_threshold();
+    (too_many_versions, too_large_versions)
+}
+
 fn warn_inline_heal_compat_requested() {
     if !scanner_inline_heal_enabled() {
         return;
@@ -503,7 +551,7 @@ impl ScannerItem {
         };
 
         let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
-        if oi.delete_marker || oi.version_purge_status.is_empty() {
+        if !Self::should_account_replication_stats(oi) {
             return;
         }
 
@@ -545,6 +593,10 @@ impl ScannerItem {
         }
     }
 
+    fn should_account_replication_stats(oi: &ObjectInfo) -> bool {
+        !oi.delete_marker && oi.version_purge_status.is_empty()
+    }
+
     async fn enqueue_heal(&mut self, oi: &ObjectInfo) {
         let done_heal = Metrics::time(Metric::HealAbandonedObject);
         debug!(
@@ -588,8 +640,38 @@ impl ScannerItem {
         done_heal();
     }
 
-    fn alert_excessive_versions(&self, _object_infos_length: usize, _cumulative_size: i64) {
-        // TODO: Implement alerting for excessive versions
+    fn alert_excessive_versions(&self, remaining_versions: usize, cumulative_size: i64) {
+        ensure_scanner_alert_metrics_registered();
+        let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size);
+        if too_many_versions {
+            counter!(
+                METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL,
+                "bucket" => self.bucket.clone()
+            )
+            .increment(1);
+            warn!(
+                bucket = %self.bucket,
+                object = %self.object_path(),
+                versions = remaining_versions,
+                threshold = scanner_excess_versions_threshold(),
+                "scanner detected object with excessive retained versions"
+            );
+        }
+        if too_large_versions {
+            counter!(
+                METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL,
+                "bucket" => self.bucket.clone()
+            )
+            .increment(1);
+            warn!(
+                bucket = %self.bucket,
+                object = %self.object_path(),
+                versions = remaining_versions,
+                cumulative_size,
+                threshold = scanner_excess_version_size_threshold(),
+                "scanner detected object with excessive retained version size"
+            );
+        }
     }
 }
 
@@ -694,6 +776,27 @@ impl FolderScanner {
         }
     }
 
+    fn alert_excessive_folders(&self, folder: &str, total_folders: usize) {
+        let threshold = scanner_excess_folders_threshold();
+        if total_folders as u64 <= threshold {
+            return;
+        }
+
+        ensure_scanner_alert_metrics_registered();
+        counter!(
+            METRIC_SCANNER_EXCESS_FOLDERS_TOTAL,
+            "root" => self.root.clone()
+        )
+        .increment(1);
+        warn!(
+            root = %self.root,
+            folder,
+            folders = total_folders,
+            threshold,
+            "scanner detected folder with excessive direct subfolders"
+        );
+    }
+
     pub async fn should_heal(&self) -> bool {
         if self.skip_heal.load(std::sync::atomic::Ordering::Relaxed) {
             return false;
@@ -1011,7 +1114,8 @@ impl FolderScanner {
                 && existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
                 || existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
 
-            // TODO: Check for excess folders and send events
+            let total_folders = existing_folders.len() + new_folders.len();
+            self.alert_excessive_folders(&folder.name, total_folders);
 
             if !into.compacted && should_compact {
                 into.compacted = true;
@@ -1557,10 +1661,12 @@ mod tests {
 
     use super::*;
     use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
+    use rustfs_filemeta::VersionPurgeStatusType;
     use serial_test::serial;
     #[cfg(unix)]
     use std::os::unix::fs::{PermissionsExt, symlink};
     use std::sync::atomic::AtomicBool;
+    use temp_env::with_var;
     use uuid::Uuid;
 
     async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
@@ -1663,6 +1769,45 @@ mod tests {
         assert!(!scanner.should_skip_failed("path2"));
     }
 
+    #[test]
+    fn test_should_account_replication_stats_only_for_live_object_versions() {
+        let live = ObjectInfo::default();
+        assert!(ScannerItem::should_account_replication_stats(&live));
+
+        let delete_marker = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        assert!(!ScannerItem::should_account_replication_stats(&delete_marker));
+
+        let purge_version = ObjectInfo {
+            version_purge_status: VersionPurgeStatusType::Pending,
+            ..Default::default()
+        };
+        assert!(!ScannerItem::should_account_replication_stats(&purge_version));
+    }
+
+    #[test]
+    #[serial]
+    fn test_excessive_version_alert_thresholds_use_env() {
+        with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSIONS, Some("3"), || {
+            with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, Some("100"), || {
+                assert_eq!(should_alert_excessive_versions(2, 99), (false, false));
+                assert_eq!(should_alert_excessive_versions(3, 99), (true, false));
+                assert_eq!(should_alert_excessive_versions(2, 100), (false, true));
+                assert_eq!(should_alert_excessive_versions(3, 100), (true, true));
+            });
+        });
+    }
+
+    #[test]
+    #[serial]
+    fn test_excessive_folders_threshold_uses_env() {
+        with_var(rustfs_config::ENV_SCANNER_ALERT_EXCESS_FOLDERS, Some("3"), || {
+            assert_eq!(scanner_excess_folders_threshold(), 3);
+        });
+    }
+
     #[tokio::test]
     #[serial]
     async fn test_record_failed_prunes_to_max_entries() {