feat(scanner): add scanner budgets and progress metrics (#3145)

* fix(scanner): preserve maintenance scan cadence

* feat(scanner): add scanner concurrency budget

* feat(scanner): expose scanner runtime progress

* fix(scanner): address scanner review feedback

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/obs/src/metrics/collectors/scanner.rs

@@ -21,7 +21,7 @@
 
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::scanner::{
-    SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_DIRECTORIES_SCANNED_MD,
     SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
 };
 
@@ -40,6 +40,8 @@ pub struct ScannerStats {
     pub versions_scanned: u64,
     /// Seconds since last scanner activity
     pub last_activity_seconds: u64,
+    /// Number of scanner paths currently being processed
+    pub active_paths: u64,
 }
 
 /// Collects scanner metrics from the given stats.
@@ -54,6 +56,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
         PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
         PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
         PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
     ]
 }
 
@@ -71,18 +74,24 @@ mod tests {
             objects_scanned: 1000000,
             versions_scanned: 1500000,
             last_activity_seconds: 30,
+            active_paths: 4,
         };
 
         let metrics = collect_scanner_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 6);
+        assert_eq!(metrics.len(), 7);
 
         let objects = metrics.iter().find(|m| m.value == 1000000.0);
         assert!(objects.is_some());
 
         let last_activity = metrics.iter().find(|m| m.value == 30.0);
         assert!(last_activity.is_some());
+
+        let active_paths = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
+        assert_eq!(active_paths.map(|m| m.value), Some(4.0));
     }
 
     #[test]
@@ -90,7 +99,7 @@ mod tests {
         let stats = ScannerStats::default();
         let metrics = collect_scanner_metrics(&stats);
 
-        assert_eq!(metrics.len(), 6);
+        assert_eq!(metrics.len(), 7);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -278,6 +278,7 @@ pub enum MetricName {
     ScannerObjectsScanned,
     ScannerVersionsScanned,
     ScannerLastActivitySeconds,
+    ScannerActivePaths,
 
     // CPU system-related metrics
     SysCPUAvgIdle,
@@ -618,6 +619,7 @@ impl MetricName {
             Self::ScannerObjectsScanned => "objects_scanned".to_string(),
             Self::ScannerVersionsScanned => "versions_scanned".to_string(),
             Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
+            Self::ScannerActivePaths => "active_paths".to_string(),
 
             // CPU system-related metrics
             Self::SysCPUAvgIdle => "avg_idle".to_string(),

crates/obs/src/metrics/schema/scanner.rs

@@ -70,3 +70,12 @@ pub static SCANNER_LAST_ACTIVITY_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLo
         subsystems::SCANNER,
     )
 });
+
+pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerActivePaths,
+        "Current number of scanner paths being processed.",
+        &[],
+        subsystems::SCANNER,
+    )
+});

crates/obs/src/metrics/stats_collector.rs

@@ -875,6 +875,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     let versions_scanned = metrics.life_time_ilm.values().copied().sum();
     let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
     let last_activity_seconds = Utc::now().signed_duration_since(reference_time).num_seconds().max(0) as u64;
+    let active_paths = metrics.active_scan_paths as u64;
 
     Some(ScannerStats {
         bucket_scans_finished,
@@ -886,6 +887,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
         objects_scanned,
         versions_scanned,
         last_activity_seconds,
+        active_paths,
     })
 }