feat(scanner): expose cycle progress metrics (#3152)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>

crates/common/src/metrics.rs

@@ -23,7 +23,7 @@ use std::{
     pin::Pin,
     sync::{
         Arc, Mutex, OnceLock,
-        atomic::{AtomicU8, AtomicU64, Ordering},
+        atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
     },
     time::{Duration, SystemTime},
 };
@@ -342,8 +342,15 @@ pub struct Metrics {
     current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
     cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
     current_scan_mode: AtomicU8,
+    current_scan_cycle_work_active: AtomicBool,
+    current_scan_cycle_objects_start: AtomicU64,
+    current_scan_cycle_directories_start: AtomicU64,
+    current_scan_cycle_bucket_drive_scans_start: AtomicU64,
     last_scan_cycle_result: AtomicU8,
     last_scan_cycle_duration_millis: AtomicU64,
+    last_scan_cycle_objects_scanned: AtomicU64,
+    last_scan_cycle_directories_scanned: AtomicU64,
+    last_scan_cycle_bucket_drive_scans: AtomicU64,
     failed_scan_cycles: AtomicU64,
 }
 
@@ -362,6 +369,13 @@ pub struct CurrentCycle {
     pub started: DateTime<Utc>,
 }
 
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct ScanCycleWorkSnapshot {
+    objects_scanned: u64,
+    directories_scanned: u64,
+    bucket_drive_scans: u64,
+}
+
 #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct ScannerTimedAction {
     pub count: u64,
@@ -389,9 +403,21 @@ pub struct ScannerMetricsReport {
     pub last_minute: ScannerLastMinute,
     pub active_paths: Vec<String>,
     pub current_scan_mode: String,
+    #[serde(default)]
+    pub current_cycle_objects_scanned: u64,
+    #[serde(default)]
+    pub current_cycle_directories_scanned: u64,
+    #[serde(default)]
+    pub current_cycle_bucket_drive_scans: u64,
     pub last_cycle_result: String,
     pub last_cycle_result_code: u64,
     pub last_cycle_duration_seconds: f64,
+    #[serde(default)]
+    pub last_cycle_objects_scanned: u64,
+    #[serde(default)]
+    pub last_cycle_directories_scanned: u64,
+    #[serde(default)]
+    pub last_cycle_bucket_drive_scans: u64,
     pub failed_cycles: u64,
 }
 
@@ -484,8 +510,15 @@ impl Metrics {
             current_paths: Arc::new(RwLock::new(HashMap::new())),
             cycle_info: Arc::new(RwLock::new(None)),
             current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
+            current_scan_cycle_work_active: AtomicBool::new(false),
+            current_scan_cycle_objects_start: AtomicU64::new(0),
+            current_scan_cycle_directories_start: AtomicU64::new(0),
+            current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
             last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
             last_scan_cycle_duration_millis: AtomicU64::new(0),
+            last_scan_cycle_objects_scanned: AtomicU64::new(0),
+            last_scan_cycle_directories_scanned: AtomicU64::new(0),
+            last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
             failed_scan_cycles: AtomicU64::new(0),
         }
     }
@@ -645,6 +678,59 @@ impl Metrics {
             .store(duration_millis_saturated(duration), Ordering::Relaxed);
     }
 
+    pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
+        let snapshot = self.scan_cycle_work_snapshot();
+        self.current_scan_cycle_objects_start
+            .store(snapshot.objects_scanned, Ordering::Relaxed);
+        self.current_scan_cycle_directories_start
+            .store(snapshot.directories_scanned, Ordering::Relaxed);
+        self.current_scan_cycle_bucket_drive_scans_start
+            .store(snapshot.bucket_drive_scans, Ordering::Relaxed);
+        self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
+        snapshot
+    }
+
+    pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
+        let work = self.scan_cycle_work_since(start);
+        self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans);
+        self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
+    }
+
+    fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot {
+        ScanCycleWorkSnapshot {
+            objects_scanned: self.lifetime(Metric::ScanObject),
+            directories_scanned: self.lifetime(Metric::ScanFolder),
+            bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
+        }
+    }
+
+    fn current_scan_cycle_work_start(&self) -> ScanCycleWorkSnapshot {
+        ScanCycleWorkSnapshot {
+            objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
+            directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
+            bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
+        }
+    }
+
+    fn scan_cycle_work_since(&self, start: ScanCycleWorkSnapshot) -> ScanCycleWorkSnapshot {
+        let current = self.scan_cycle_work_snapshot();
+        ScanCycleWorkSnapshot {
+            objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
+            directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
+            bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
+        }
+    }
+
+    pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) {
+        // Telemetry-only gauges: readers may observe a transient mixed snapshot
+        // while these independent atomic fields are updated.
+        self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed);
+        self.last_scan_cycle_directories_scanned
+            .store(directories_scanned, Ordering::Relaxed);
+        self.last_scan_cycle_bucket_drive_scans
+            .store(bucket_drive_scans, Ordering::Relaxed);
+    }
+
     /// Snapshot of every path currently being scanned.
     pub async fn get_current_paths(&self) -> Vec<String> {
         let paths = self.current_paths.read().await;
@@ -681,10 +767,19 @@ impl Metrics {
         m.active_paths = self.get_current_paths().await;
         m.active_scan_paths = m.active_paths.len();
         m.current_scan_mode = self.current_scan_mode().as_str().to_string();
+        if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
+            let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
+            m.current_cycle_objects_scanned = current_work.objects_scanned;
+            m.current_cycle_directories_scanned = current_work.directories_scanned;
+            m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
+        }
         let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
         m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
         m.last_cycle_result_code = last_cycle_result as u64;
         m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
+        m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
+        m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
+        m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
         m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
 
         // Lifetime operation counts
@@ -899,4 +994,45 @@ mod tests {
         assert_eq!(report.last_cycle_duration_seconds, 2.0);
         assert_eq!(report.failed_cycles, 0);
     }
+
+    #[tokio::test]
+    async fn report_includes_last_scan_cycle_work() {
+        let metrics = Metrics::new();
+        metrics.record_scan_cycle_work(11, 7, 3);
+
+        let report = metrics.report().await;
+
+        assert_eq!(report.last_cycle_objects_scanned, 11);
+        assert_eq!(report.last_cycle_directories_scanned, 7);
+        assert_eq!(report.last_cycle_bucket_drive_scans, 3);
+    }
+
+    #[tokio::test]
+    async fn report_includes_active_scan_cycle_work() {
+        let metrics = Metrics::new();
+        metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
+        metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
+        metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
+
+        let start = metrics.start_scan_cycle_work();
+        metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
+        metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
+        metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
+
+        let report = metrics.report().await;
+
+        assert_eq!(report.current_cycle_objects_scanned, 7);
+        assert_eq!(report.current_cycle_directories_scanned, 3);
+        assert_eq!(report.current_cycle_bucket_drive_scans, 2);
+
+        metrics.finish_scan_cycle_work(start);
+        let report = metrics.report().await;
+
+        assert_eq!(report.current_cycle_objects_scanned, 0);
+        assert_eq!(report.current_cycle_directories_scanned, 0);
+        assert_eq!(report.current_cycle_bucket_drive_scans, 0);
+        assert_eq!(report.last_cycle_objects_scanned, 7);
+        assert_eq!(report.last_cycle_directories_scanned, 3);
+        assert_eq!(report.last_cycle_bucket_drive_scans, 2);
+    }
 }

crates/obs/src/metrics/collectors/scanner.rs

@@ -22,9 +22,14 @@
 use crate::metrics::report::PrometheusMetric;
 use crate::metrics::schema::scanner::{
     SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
-    SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
-    SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD,
+    SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
-    SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
+    SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD,
+    SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
+    SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
+    SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
+    SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD,
+    SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
 };
 
 /// Scanner statistics.
@@ -50,12 +55,36 @@ pub struct ScannerStats {
     pub completed_cycles: u64,
     /// Seconds elapsed since the current scanner cycle started
     pub current_cycle_age_seconds: u64,
+    /// Number of objects scanned by the currently running scanner cycle
+    pub current_cycle_objects_scanned: u64,
+    /// Number of directories scanned by the currently running scanner cycle
+    pub current_cycle_directories_scanned: u64,
+    /// Number of bucket-drive scans finished by the currently running scanner cycle
+    pub current_cycle_bucket_drive_scans: u64,
+    /// Object scan rate for the currently running scanner cycle
+    pub current_cycle_objects_per_second: f64,
+    /// Directory scan rate for the currently running scanner cycle
+    pub current_cycle_directories_per_second: f64,
+    /// Bucket-drive scan rate for the currently running scanner cycle
+    pub current_cycle_bucket_drive_scans_per_second: f64,
     /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
     pub current_scan_mode: u64,
     /// Last scanner cycle result: 0 unknown, 1 success, 2 error
     pub last_cycle_result: u64,
     /// Duration in seconds of the last finished scanner cycle
     pub last_cycle_duration_seconds: f64,
+    /// Number of objects scanned by the last finished scanner cycle
+    pub last_cycle_objects_scanned: u64,
+    /// Number of directories scanned by the last finished scanner cycle
+    pub last_cycle_directories_scanned: u64,
+    /// Number of bucket-drive scans finished by the last scanner cycle
+    pub last_cycle_bucket_drive_scans: u64,
+    /// Object scan rate for the last finished scanner cycle
+    pub last_cycle_objects_per_second: f64,
+    /// Directory scan rate for the last finished scanner cycle
+    pub last_cycle_directories_per_second: f64,
+    /// Bucket-drive scan rate for the last finished scanner cycle
+    pub last_cycle_bucket_drive_scans_per_second: f64,
     /// Number of scanner cycles that failed since server start
     pub failed_cycles: u64,
 }
@@ -76,9 +105,39 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
         PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
         PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
         PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, stats.current_cycle_objects_scanned as f64),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
+            stats.current_cycle_directories_scanned as f64,
+        ),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
+            stats.current_cycle_bucket_drive_scans as f64,
+        ),
+        PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
+            stats.current_cycle_directories_per_second,
+        ),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
+            stats.current_cycle_bucket_drive_scans_per_second,
+        ),
         PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
         PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
         PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, stats.last_cycle_objects_scanned as f64),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD,
+            stats.last_cycle_directories_scanned as f64,
+        ),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
+        PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
+        PrometheusMetric::from_descriptor(
+            &SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
+            stats.last_cycle_bucket_drive_scans_per_second,
+        ),
         PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
     ]
 }
@@ -101,16 +160,28 @@ mod tests {
             current_cycle: 12,
             completed_cycles: 11,
             current_cycle_age_seconds: 90,
+            current_cycle_objects_scanned: 250,
+            current_cycle_directories_scanned: 20,
+            current_cycle_bucket_drive_scans: 2,
+            current_cycle_objects_per_second: 12.5,
+            current_cycle_directories_per_second: 1.0,
+            current_cycle_bucket_drive_scans_per_second: 0.1,
             current_scan_mode: 2,
             last_cycle_result: 1,
             last_cycle_duration_seconds: 42.5,
+            last_cycle_objects_scanned: 900,
+            last_cycle_directories_scanned: 80,
+            last_cycle_bucket_drive_scans: 6,
+            last_cycle_objects_per_second: 18.0,
+            last_cycle_directories_per_second: 1.6,
+            last_cycle_bucket_drive_scans_per_second: 0.12,
             failed_cycles: 3,
         };
 
         let metrics = collect_scanner_metrics(&stats);
         report_metrics(&metrics);
 
-        assert_eq!(metrics.len(), 14);
+        assert_eq!(metrics.len(), 26);
 
         let objects = metrics.iter().find(|m| m.value == 1000000.0);
         assert!(objects.is_some());
@@ -138,6 +209,36 @@ mod tests {
             .find(|m| m.name == SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD.get_full_metric_name());
         assert_eq!(current_cycle_age.map(|m| m.value), Some(90.0));
 
+        let current_cycle_objects = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD.get_full_metric_name());
+        assert_eq!(current_cycle_objects.map(|m| m.value), Some(250.0));
+
+        let current_cycle_directories = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD.get_full_metric_name());
+        assert_eq!(current_cycle_directories.map(|m| m.value), Some(20.0));
+
+        let current_cycle_bucket_drive_scans = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
+        assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
+
+        let current_cycle_objects_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(current_cycle_objects_rate.map(|m| m.value), Some(12.5));
+
+        let current_cycle_directories_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(current_cycle_directories_rate.map(|m| m.value), Some(1.0));
+
+        let current_cycle_bucket_drive_scans_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
+
         let current_scan_mode = metrics
             .iter()
             .find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
@@ -153,6 +254,36 @@ mod tests {
             .find(|m| m.name == SCANNER_LAST_CYCLE_DURATION_SECONDS_MD.get_full_metric_name());
         assert_eq!(last_cycle_duration.map(|m| m.value), Some(42.5));
 
+        let last_cycle_objects = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD.get_full_metric_name());
+        assert_eq!(last_cycle_objects.map(|m| m.value), Some(900.0));
+
+        let last_cycle_directories = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD.get_full_metric_name());
+        assert_eq!(last_cycle_directories.map(|m| m.value), Some(80.0));
+
+        let last_cycle_bucket_drive_scans = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
+        assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
+
+        let last_cycle_objects_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(last_cycle_objects_rate.map(|m| m.value), Some(18.0));
+
+        let last_cycle_directories_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(last_cycle_directories_rate.map(|m| m.value), Some(1.6));
+
+        let last_cycle_bucket_drive_scans_rate = metrics
+            .iter()
+            .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
+        assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
+
         let failed_cycles = metrics
             .iter()
             .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
@@ -164,7 +295,7 @@ mod tests {
         let stats = ScannerStats::default();
         let metrics = collect_scanner_metrics(&stats);
 
-        assert_eq!(metrics.len(), 14);
+        assert_eq!(metrics.len(), 26);
         for metric in &metrics {
             assert_eq!(metric.value, 0.0);
             assert!(metric.labels.is_empty());

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -282,9 +282,21 @@ pub enum MetricName {
     ScannerCurrentCycle,
     ScannerCompletedCycles,
     ScannerCurrentCycleAgeSeconds,
+    ScannerCurrentCycleObjectsScanned,
+    ScannerCurrentCycleDirectoriesScanned,
+    ScannerCurrentCycleBucketDriveScans,
+    ScannerCurrentCycleObjectsPerSecond,
+    ScannerCurrentCycleDirectoriesPerSecond,
+    ScannerCurrentCycleBucketDriveScansPerSecond,
     ScannerCurrentScanMode,
     ScannerLastCycleResult,
     ScannerLastCycleDurationSeconds,
+    ScannerLastCycleObjectsScanned,
+    ScannerLastCycleDirectoriesScanned,
+    ScannerLastCycleBucketDriveScans,
+    ScannerLastCycleObjectsPerSecond,
+    ScannerLastCycleDirectoriesPerSecond,
+    ScannerLastCycleBucketDriveScansPerSecond,
     ScannerFailedCycles,
 
     // CPU system-related metrics
@@ -630,9 +642,21 @@ impl MetricName {
             Self::ScannerCurrentCycle => "current_cycle".to_string(),
             Self::ScannerCompletedCycles => "completed_cycles".to_string(),
             Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
+            Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
+            Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
+            Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
+            Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
+            Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
+            Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
             Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
             Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
             Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
+            Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
+            Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
+            Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
+            Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
+            Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
+            Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
             Self::ScannerFailedCycles => "failed_cycles".to_string(),
 
             // CPU system-related metrics

crates/obs/src/metrics/schema/scanner.rs

@@ -107,6 +107,60 @@ pub static SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = La
     )
 });
 
+pub static SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleObjectsScanned,
+        "Number of objects scanned by the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleDirectoriesScanned,
+        "Number of directories scanned by the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleBucketDriveScans,
+        "Number of bucket-drive scans finished by the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleObjectsPerSecond,
+        "Object scan rate for the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleDirectoriesPerSecond,
+        "Directory scan rate for the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerCurrentCycleBucketDriveScansPerSecond,
+        "Bucket-drive scan rate for the currently running scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
 pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::ScannerCurrentScanMode,
@@ -134,6 +188,60 @@ pub static SCANNER_LAST_CYCLE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> =
     )
 });
 
+pub static SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleObjectsScanned,
+        "Number of objects scanned by the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleDirectoriesScanned,
+        "Number of directories scanned by the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleBucketDriveScans,
+        "Number of bucket-drive scans finished by the last scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleObjectsPerSecond,
+        "Object scan rate for the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleDirectoriesPerSecond,
+        "Directory scan rate for the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
+pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::ScannerLastCycleBucketDriveScansPerSecond,
+        "Bucket-drive scan rate for the last finished scanner cycle.",
+        &[],
+        subsystems::SCANNER,
+    )
+});
+
 pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_counter_md(
         MetricName::ScannerFailedCycles,

crates/obs/src/metrics/stats_collector.rs

@@ -65,6 +65,14 @@ fn scanner_scan_mode_code(scan_mode: &str) -> u64 {
     }
 }
 
+fn scanner_work_rate_per_second(count: u64, seconds: f64) -> f64 {
+    if seconds > 0.0 && seconds.is_finite() {
+        count as f64 / seconds
+    } else {
+        0.0
+    }
+}
+
 const DRIVE_STATE_OK: &str = "ok";
 const DRIVE_STATE_ONLINE: &str = "online";
 const DRIVE_STATE_UNFORMATTED: &str = "unformatted";
@@ -901,6 +909,8 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
     let active_paths = metrics.active_scan_paths as u64;
     let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
     let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
+    let current_cycle_age = current_cycle_age_seconds as f64;
+    let last_cycle_duration = metrics.last_cycle_duration_seconds;
 
     Some(ScannerStats {
         bucket_scans_finished,
@@ -916,9 +926,33 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
         current_cycle: metrics.current_cycle,
         completed_cycles,
         current_cycle_age_seconds,
+        current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
+        current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
+        current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
+        current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
+        current_cycle_directories_per_second: scanner_work_rate_per_second(
+            metrics.current_cycle_directories_scanned,
+            current_cycle_age,
+        ),
+        current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
+            metrics.current_cycle_bucket_drive_scans,
+            current_cycle_age,
+        ),
         current_scan_mode,
         last_cycle_result: metrics.last_cycle_result_code,
         last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
+        last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
+        last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
+        last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
+        last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
+        last_cycle_directories_per_second: scanner_work_rate_per_second(
+            metrics.last_cycle_directories_scanned,
+            last_cycle_duration,
+        ),
+        last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second(
+            metrics.last_cycle_bucket_drive_scans,
+            last_cycle_duration,
+        ),
         failed_cycles: metrics.failed_cycles,
     })
 }
@@ -1021,4 +1055,16 @@ mod tests {
     fn scanner_scan_mode_code_maps_unknown_mode() {
         assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
     }
+
+    #[test]
+    fn scanner_work_rate_per_second_reports_rate() {
+        assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
+    }
+
+    #[test]
+    fn scanner_work_rate_per_second_returns_zero_for_invalid_seconds() {
+        assert_eq!(scanner_work_rate_per_second(90, 0.0), 0.0);
+        assert_eq!(scanner_work_rate_per_second(90, f64::INFINITY), 0.0);
+        assert_eq!(scanner_work_rate_per_second(90, f64::NAN), 0.0);
+    }
 }

crates/scanner/src/scanner.rs

@@ -455,12 +455,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
 
     let done_cycle = Metrics::time(Metric::ScanCycle);
     let cycle_start = std::time::Instant::now();
+    let cycle_work_start = global_metrics().start_scan_cycle_work();
     if let Err(e) = storeapi
         .clone()
         .nsscanner(ctx.clone(), sender, cycle_info.current, scan_mode)
         .await
     {
         error!(duration = ?now.elapsed(), "Fail run data scanner cycle: {e}");
+        global_metrics().finish_scan_cycle_work(cycle_work_start);
         emit_scan_cycle_complete(false, cycle_start.elapsed());
         if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
             save_background_heal_info(storeapi.clone(), new_heal_info).await;
@@ -468,6 +470,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
         return;
     }
     done_cycle();
+    global_metrics().finish_scan_cycle_work(cycle_work_start);
     emit_scan_cycle_complete(true, cycle_start.elapsed());
     if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) {
         save_background_heal_info(storeapi.clone(), new_heal_info).await;