feat(scanner): expose cycle observability controls (#3147)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>

crates/obs/src/metrics/collectors/scanner.rs

@@ -21,8 +21,10 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::scanner::{
-    SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
-    SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
+    SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
+    SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD,
+    SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
 };
 
 /// Scanner statistics.
@@ -42,6 +44,20 @@ pub struct ScannerStats {
     pub last_activity_seconds: u64,
     /// Number of scanner paths currently being processed
     pub active_paths: u64,
+    /// Current scanner cycle number, or zero when idle
+    pub current_cycle: u64,
+    /// Number of scanner cycles completed since server start
+    pub completed_cycles: u64,
+    /// Seconds elapsed since the current scanner cycle started
+    pub current_cycle_age_seconds: u64,
+    /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
+    pub current_scan_mode: u64,
+    /// Last scanner cycle result: 0 unknown, 1 success, 2 error
+    pub last_cycle_result: u64,
+    /// Duration in seconds of the last finished scanner cycle
+    pub last_cycle_duration_seconds: f64,
+    /// Number of scanner cycles that failed since server start
+    pub failed_cycles: u64,
 }
 
 /// Collects scanner metrics from the given stats.
@@ -57,6 +73,13 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
         PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
         PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
         PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
+        PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
     ]
 }
 
@@ -75,12 +98,19 @@ mod tests {
             versions_scanned: 1500000,
             last_activity_seconds: 30,
             active_paths: 4,
+            current_cycle: 12,
+            completed_cycles: 11,
+            current_cycle_age_seconds: 90,
+            current_scan_mode: 2,
+            last_cycle_result: 1,
+            last_cycle_duration_seconds: 42.5,
+            failed_cycles: 3,
         };
 
         let metrics = collect_scanner_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 7);
+        assert_eq!(metrics.len(), 14);
 
         let objects = metrics.iter().find(|m| m.value == 1000000.0);
         assert!(objects.is_some());
@@ -92,6 +122,41 @@ mod tests {
             .iter()
             .find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
         assert_eq!(active_paths.map(|m| m.value), Some(4.0));
+
+        let current_cycle = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
+        assert_eq!(current_cycle.map(|m| m.value), Some(12.0));
+
+        let completed_cycles = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_COMPLETED_CYCLES_MD.get_full_metric_name());
+        assert_eq!(completed_cycles.map(|m| m.value), Some(11.0));
+
+        let current_cycle_age = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD.get_full_metric_name());
+        assert_eq!(current_cycle_age.map(|m| m.value), Some(90.0));
+
+        let current_scan_mode = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
+        assert_eq!(current_scan_mode.map(|m| m.value), Some(2.0));
+
+        let last_cycle_result = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_RESULT_MD.get_full_metric_name());
+        assert_eq!(last_cycle_result.map(|m| m.value), Some(1.0));
+
+        let last_cycle_duration = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_DURATION_SECONDS_MD.get_full_metric_name());
+        assert_eq!(last_cycle_duration.map(|m| m.value), Some(42.5));
+
+        let failed_cycles = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
+        assert_eq!(failed_cycles.map(|m| m.value), Some(3.0));
     }
 
     #[test]
@@ -99,7 +164,7 @@ mod tests {
         let stats = ScannerStats::default();
         let metrics = collect_scanner_metrics(&stats);
 
-        assert_eq!(metrics.len(), 7);
+        assert_eq!(metrics.len(), 14);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -279,6 +279,13 @@ pub enum MetricName {
     ScannerVersionsScanned,
     ScannerLastActivitySeconds,
     ScannerActivePaths,
+    ScannerCurrentCycle,
+    ScannerCompletedCycles,
+    ScannerCurrentCycleAgeSeconds,
+    ScannerCurrentScanMode,
+    ScannerLastCycleResult,
+    ScannerLastCycleDurationSeconds,
+    ScannerFailedCycles,
 
     // CPU system-related metrics
     SysCPUAvgIdle,
@@ -620,6 +627,13 @@ impl MetricName {
             Self::ScannerVersionsScanned => "versions_scanned".to_string(),
             Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
             Self::ScannerActivePaths => "active_paths".to_string(),
+            Self::ScannerCurrentCycle => "current_cycle".to_string(),
+            Self::ScannerCompletedCycles => "completed_cycles".to_string(),
+            Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
+            Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
+            Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
+            Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
+            Self::ScannerFailedCycles => "failed_cycles".to_string(),
 
             // CPU system-related metrics
             Self::SysCPUAvgIdle => "avg_idle".to_string(),

crates/obs/src/metrics/schema/scanner.rs

@@ -79,3 +79,66 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
         subsystems::SCANNER,
     )
 });
+
+pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycle,
+        "Current scanner cycle number, or zero when no cycle is running.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_COMPLETED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCompletedCycles,
+        "Total number of scanner cycles completed since server start.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleAgeSeconds,
+        "Time elapsed (in seconds) since the current scanner cycle started, or zero when no cycle is running.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentScanMode,
+        "Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleResult,
+        "Last scanner cycle result: 0 unknown, 1 success, 2 error.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleDurationSeconds,
+        "Duration in seconds of the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_counter_md(
+        MetricName::ScannerFailedCycles,
+        "Total number of scanner cycles that failed since server start.",
+        &[],
+        subsystems::SCANNER,
+    )
+});

crates/obs/src/metrics/stats_collector.rs

@@ -27,6 +27,7 @@ use crate::metrics::collectors::{
     ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
 };
 use chrono::Utc;
+use rustfs_common::heal_channel::HealScanMode;
 use rustfs_common::metrics::global_metrics;
 use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState};
 use rustfs_ecstore::bucket::metadata_sys::get_quota_config;
@@ -44,6 +45,26 @@ use std::time::Duration;
 use sysinfo::{Networks, System};
 use tracing::{instrument, warn};
 
+fn current_scanner_cycle_age_seconds(
+    current_cycle: u64,
+    current_started: chrono::DateTime<Utc>,
+    now: chrono::DateTime<Utc>,
+) -> u64 {
+    if current_cycle == 0 {
+        0
+    } else {
+        now.signed_duration_since(current_started).num_seconds().max(0) as u64
+    }
+}
+
+fn scanner_scan_mode_code(scan_mode: &str) -> u64 {
+    match scan_mode {
+        mode if mode == HealScanMode::Normal.as_str() => HealScanMode::Normal as u8 as u64,
+        mode if mode == HealScanMode::Deep.as_str() => HealScanMode::Deep as u8 as u64,
+        _ => HealScanMode::Unknown as u8 as u64,
+    }
+}
+
 const DRIVE_STATE_OK: &str = "ok";
 const DRIVE_STATE_ONLINE: &str = "online";
 const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
@@ -869,13 +890,17 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
 /// rustfs-obs scanner collector shape.
 pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     let metrics = global_metrics().report().await;
+    let now = Utc::now();
     let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
+    let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
     let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
     let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
     let versions_scanned = metrics.life_time_ilm.values().copied().sum();
     let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
-    let last_activity_seconds = Utc::now().signed_duration_since(reference_time).num_seconds().max(0) as u64;
+    let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
     let active_paths = metrics.active_scan_paths as u64;
+    let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
+    let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
 
     Some(ScannerStats {
         bucket_scans_finished,
@@ -888,6 +913,13 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
         versions_scanned,
         last_activity_seconds,
         active_paths,
+        current_cycle: metrics.current_cycle,
+        completed_cycles,
+        current_cycle_age_seconds,
+        current_scan_mode,
+        last_cycle_result: metrics.last_cycle_result_code,
+        last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
+        failed_cycles: metrics.failed_cycles,
     })
 }
 
@@ -957,4 +989,36 @@ mod tests {
         assert_eq!(stats.write_health, 1);
         assert_eq!(stats.health, 1);
     }
+
+    #[test]
+    fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
+        let now = Utc::now();
+
+        assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
+    }
+
+    #[test]
+    fn current_scanner_cycle_age_seconds_clamps_future_start() {
+        let now = Utc::now();
+
+        assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
+    }
+
+    #[test]
+    fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
+        let now = Utc::now();
+
+        assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
+    }
+
+    #[test]
+    fn scanner_scan_mode_code_maps_known_modes() {
+        assert_eq!(scanner_scan_mode_code(HealScanMode::Normal.as_str()), HealScanMode::Normal as u8 as u64);
+        assert_eq!(scanner_scan_mode_code(HealScanMode::Deep.as_str()), HealScanMode::Deep as u8 as u64);
+    }
+
+    #[test]
+    fn scanner_scan_mode_code_maps_unknown_mode() {
+        assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
+    }
 }