diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 8bdee2e1f..dd98a4e33 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -23,7 +23,7 @@ use std::{ pin::Pin, sync::{ Arc, Mutex, OnceLock, - atomic::{AtomicU8, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, }, time::{Duration, SystemTime}, }; @@ -342,8 +342,15 @@ pub struct Metrics { current_paths: Arc>>>, cycle_info: Arc>>, current_scan_mode: AtomicU8, + current_scan_cycle_work_active: AtomicBool, + current_scan_cycle_objects_start: AtomicU64, + current_scan_cycle_directories_start: AtomicU64, + current_scan_cycle_bucket_drive_scans_start: AtomicU64, last_scan_cycle_result: AtomicU8, last_scan_cycle_duration_millis: AtomicU64, + last_scan_cycle_objects_scanned: AtomicU64, + last_scan_cycle_directories_scanned: AtomicU64, + last_scan_cycle_bucket_drive_scans: AtomicU64, failed_scan_cycles: AtomicU64, } @@ -362,6 +369,13 @@ pub struct CurrentCycle { pub started: DateTime, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ScanCycleWorkSnapshot { + objects_scanned: u64, + directories_scanned: u64, + bucket_drive_scans: u64, +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ScannerTimedAction { pub count: u64, @@ -389,9 +403,21 @@ pub struct ScannerMetricsReport { pub last_minute: ScannerLastMinute, pub active_paths: Vec, pub current_scan_mode: String, + #[serde(default)] + pub current_cycle_objects_scanned: u64, + #[serde(default)] + pub current_cycle_directories_scanned: u64, + #[serde(default)] + pub current_cycle_bucket_drive_scans: u64, pub last_cycle_result: String, pub last_cycle_result_code: u64, pub last_cycle_duration_seconds: f64, + #[serde(default)] + pub last_cycle_objects_scanned: u64, + #[serde(default)] + pub last_cycle_directories_scanned: u64, + #[serde(default)] + pub last_cycle_bucket_drive_scans: u64, pub failed_cycles: u64, } @@ -484,8 +510,15 @@ impl Metrics { current_paths: Arc::new(RwLock::new(HashMap::new())), cycle_info: Arc::new(RwLock::new(None)), current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8), + current_scan_cycle_work_active: AtomicBool::new(false), + current_scan_cycle_objects_start: AtomicU64::new(0), + current_scan_cycle_directories_start: AtomicU64::new(0), + current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0), last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN), last_scan_cycle_duration_millis: AtomicU64::new(0), + last_scan_cycle_objects_scanned: AtomicU64::new(0), + last_scan_cycle_directories_scanned: AtomicU64::new(0), + last_scan_cycle_bucket_drive_scans: AtomicU64::new(0), failed_scan_cycles: AtomicU64::new(0), } } @@ -645,6 +678,59 @@ impl Metrics { .store(duration_millis_saturated(duration), Ordering::Relaxed); } + pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot { + let snapshot = self.scan_cycle_work_snapshot(); + self.current_scan_cycle_objects_start + .store(snapshot.objects_scanned, Ordering::Relaxed); + self.current_scan_cycle_directories_start + .store(snapshot.directories_scanned, Ordering::Relaxed); + self.current_scan_cycle_bucket_drive_scans_start + .store(snapshot.bucket_drive_scans, Ordering::Relaxed); + self.current_scan_cycle_work_active.store(true, Ordering::Relaxed); + snapshot + } + + pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) { + let work = self.scan_cycle_work_since(start); + self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans); + self.current_scan_cycle_work_active.store(false, Ordering::Relaxed); + } + + fn scan_cycle_work_snapshot(&self) -> ScanCycleWorkSnapshot { + ScanCycleWorkSnapshot { + objects_scanned: self.lifetime(Metric::ScanObject), + directories_scanned: self.lifetime(Metric::ScanFolder), + bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive), + } + } + + fn current_scan_cycle_work_start(&self) -> ScanCycleWorkSnapshot { + ScanCycleWorkSnapshot { + objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed), + directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed), + bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed), + } + } + + fn scan_cycle_work_since(&self, start: ScanCycleWorkSnapshot) -> ScanCycleWorkSnapshot { + let current = self.scan_cycle_work_snapshot(); + ScanCycleWorkSnapshot { + objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned), + directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned), + bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans), + } + } + + pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) { + // Telemetry-only gauges: readers may observe a transient mixed snapshot + // while these independent atomic fields are updated. + self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed); + self.last_scan_cycle_directories_scanned + .store(directories_scanned, Ordering::Relaxed); + self.last_scan_cycle_bucket_drive_scans + .store(bucket_drive_scans, Ordering::Relaxed); + } + /// Snapshot of every path currently being scanned. pub async fn get_current_paths(&self) -> Vec { let paths = self.current_paths.read().await; @@ -681,10 +767,19 @@ impl Metrics { m.active_paths = self.get_current_paths().await; m.active_scan_paths = m.active_paths.len(); m.current_scan_mode = self.current_scan_mode().as_str().to_string(); + if self.current_scan_cycle_work_active.load(Ordering::Relaxed) { + let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start()); + m.current_cycle_objects_scanned = current_work.objects_scanned; + m.current_cycle_directories_scanned = current_work.directories_scanned; + m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans; + } let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed); m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string(); m.last_cycle_result_code = last_cycle_result as u64; m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0; + m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed); + m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed); + m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed); m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed); // Lifetime operation counts @@ -899,4 +994,45 @@ mod tests { assert_eq!(report.last_cycle_duration_seconds, 2.0); assert_eq!(report.failed_cycles, 0); } + + #[tokio::test] + async fn report_includes_last_scan_cycle_work() { + let metrics = Metrics::new(); + metrics.record_scan_cycle_work(11, 7, 3); + + let report = metrics.report().await; + + assert_eq!(report.last_cycle_objects_scanned, 11); + assert_eq!(report.last_cycle_directories_scanned, 7); + assert_eq!(report.last_cycle_bucket_drive_scans, 3); + } + + #[tokio::test] + async fn report_includes_active_scan_cycle_work() { + let metrics = Metrics::new(); + metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed); + metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed); + metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed); + + let start = metrics.start_scan_cycle_work(); + metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed); + metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed); + metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed); + + let report = metrics.report().await; + + assert_eq!(report.current_cycle_objects_scanned, 7); + assert_eq!(report.current_cycle_directories_scanned, 3); + assert_eq!(report.current_cycle_bucket_drive_scans, 2); + + metrics.finish_scan_cycle_work(start); + let report = metrics.report().await; + + assert_eq!(report.current_cycle_objects_scanned, 0); + assert_eq!(report.current_cycle_directories_scanned, 0); + assert_eq!(report.current_cycle_bucket_drive_scans, 0); + assert_eq!(report.last_cycle_objects_scanned, 7); + assert_eq!(report.last_cycle_directories_scanned, 3); + assert_eq!(report.last_cycle_bucket_drive_scans, 2); + } } diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index fdf56802c..ffdeb164b 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -22,9 +22,14 @@ use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::scanner::{ SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD, - SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD, - SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, - SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD, + SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, + SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, + SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, + SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD, + SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, + SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, + SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, + SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD, }; /// Scanner statistics. @@ -50,12 +55,36 @@ pub struct ScannerStats { pub completed_cycles: u64, /// Seconds elapsed since the current scanner cycle started pub current_cycle_age_seconds: u64, + /// Number of objects scanned by the currently running scanner cycle + pub current_cycle_objects_scanned: u64, + /// Number of directories scanned by the currently running scanner cycle + pub current_cycle_directories_scanned: u64, + /// Number of bucket-drive scans finished by the currently running scanner cycle + pub current_cycle_bucket_drive_scans: u64, + /// Object scan rate for the currently running scanner cycle + pub current_cycle_objects_per_second: f64, + /// Directory scan rate for the currently running scanner cycle + pub current_cycle_directories_per_second: f64, + /// Bucket-drive scan rate for the currently running scanner cycle + pub current_cycle_bucket_drive_scans_per_second: f64, /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan pub current_scan_mode: u64, /// Last scanner cycle result: 0 unknown, 1 success, 2 error pub last_cycle_result: u64, /// Duration in seconds of the last finished scanner cycle pub last_cycle_duration_seconds: f64, + /// Number of objects scanned by the last finished scanner cycle + pub last_cycle_objects_scanned: u64, + /// Number of directories scanned by the last finished scanner cycle + pub last_cycle_directories_scanned: u64, + /// Number of bucket-drive scans finished by the last scanner cycle + pub last_cycle_bucket_drive_scans: u64, + /// Object scan rate for the last finished scanner cycle + pub last_cycle_objects_per_second: f64, + /// Directory scan rate for the last finished scanner cycle + pub last_cycle_directories_per_second: f64, + /// Bucket-drive scan rate for the last finished scanner cycle + pub last_cycle_bucket_drive_scans_per_second: f64, /// Number of scanner cycles that failed since server start pub failed_cycles: u64, } @@ -76,9 +105,39 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64), PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, stats.current_cycle_objects_scanned as f64), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, + stats.current_cycle_directories_scanned as f64, + ), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, + stats.current_cycle_bucket_drive_scans as f64, + ), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, + stats.current_cycle_directories_per_second, + ), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, + stats.current_cycle_bucket_drive_scans_per_second, + ), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, stats.last_cycle_objects_scanned as f64), + PrometheusMetric::from_descriptor( + &SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, + stats.last_cycle_directories_scanned as f64, + ), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second), + PrometheusMetric::from_descriptor( + &SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, + stats.last_cycle_bucket_drive_scans_per_second, + ), PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64), ] } @@ -101,16 +160,28 @@ mod tests { current_cycle: 12, completed_cycles: 11, current_cycle_age_seconds: 90, + current_cycle_objects_scanned: 250, + current_cycle_directories_scanned: 20, + current_cycle_bucket_drive_scans: 2, + current_cycle_objects_per_second: 12.5, + current_cycle_directories_per_second: 1.0, + current_cycle_bucket_drive_scans_per_second: 0.1, current_scan_mode: 2, last_cycle_result: 1, last_cycle_duration_seconds: 42.5, + last_cycle_objects_scanned: 900, + last_cycle_directories_scanned: 80, + last_cycle_bucket_drive_scans: 6, + last_cycle_objects_per_second: 18.0, + last_cycle_directories_per_second: 1.6, + last_cycle_bucket_drive_scans_per_second: 0.12, failed_cycles: 3, }; let metrics = collect_scanner_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 14); + assert_eq!(metrics.len(), 26); let objects = metrics.iter().find(|m| m.value == 1000000.0); assert!(objects.is_some()); @@ -138,6 +209,36 @@ mod tests { .find(|m| m.name == SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD.get_full_metric_name()); assert_eq!(current_cycle_age.map(|m| m.value), Some(90.0)); + let current_cycle_objects = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD.get_full_metric_name()); + assert_eq!(current_cycle_objects.map(|m| m.value), Some(250.0)); + + let current_cycle_directories = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD.get_full_metric_name()); + assert_eq!(current_cycle_directories.map(|m| m.value), Some(20.0)); + + let current_cycle_bucket_drive_scans = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); + assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0)); + + let current_cycle_objects_rate = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(current_cycle_objects_rate.map(|m| m.value), Some(12.5)); + + let current_cycle_directories_rate = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(current_cycle_directories_rate.map(|m| m.value), Some(1.0)); + + let current_cycle_bucket_drive_scans_rate = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1)); + let current_scan_mode = metrics .iter() .find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name()); @@ -153,6 +254,36 @@ mod tests { .find(|m| m.name == SCANNER_LAST_CYCLE_DURATION_SECONDS_MD.get_full_metric_name()); assert_eq!(last_cycle_duration.map(|m| m.value), Some(42.5)); + let last_cycle_objects = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD.get_full_metric_name()); + assert_eq!(last_cycle_objects.map(|m| m.value), Some(900.0)); + + let last_cycle_directories = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD.get_full_metric_name()); + assert_eq!(last_cycle_directories.map(|m| m.value), Some(80.0)); + + let last_cycle_bucket_drive_scans = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); + assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0)); + + let last_cycle_objects_rate = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(last_cycle_objects_rate.map(|m| m.value), Some(18.0)); + + let last_cycle_directories_rate = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(last_cycle_directories_rate.map(|m| m.value), Some(1.6)); + + let last_cycle_bucket_drive_scans_rate = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); + assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12)); + let failed_cycles = metrics .iter() .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name()); @@ -164,7 +295,7 @@ mod tests { let stats = ScannerStats::default(); let metrics = collect_scanner_metrics(&stats); - assert_eq!(metrics.len(), 14); + assert_eq!(metrics.len(), 26); for metric in &metrics { assert_eq!(metric.value, 0.0); assert!(metric.labels.is_empty()); diff --git a/crates/obs/src/metrics/schema/entry/metric_name.rs b/crates/obs/src/metrics/schema/entry/metric_name.rs index 7c11e7495..92ade59a8 100644 --- a/crates/obs/src/metrics/schema/entry/metric_name.rs +++ b/crates/obs/src/metrics/schema/entry/metric_name.rs @@ -282,9 +282,21 @@ pub enum MetricName { ScannerCurrentCycle, ScannerCompletedCycles, ScannerCurrentCycleAgeSeconds, + ScannerCurrentCycleObjectsScanned, + ScannerCurrentCycleDirectoriesScanned, + ScannerCurrentCycleBucketDriveScans, + ScannerCurrentCycleObjectsPerSecond, + ScannerCurrentCycleDirectoriesPerSecond, + ScannerCurrentCycleBucketDriveScansPerSecond, ScannerCurrentScanMode, ScannerLastCycleResult, ScannerLastCycleDurationSeconds, + ScannerLastCycleObjectsScanned, + ScannerLastCycleDirectoriesScanned, + ScannerLastCycleBucketDriveScans, + ScannerLastCycleObjectsPerSecond, + ScannerLastCycleDirectoriesPerSecond, + ScannerLastCycleBucketDriveScansPerSecond, ScannerFailedCycles, // CPU system-related metrics @@ -630,9 +642,21 @@ impl MetricName { Self::ScannerCurrentCycle => "current_cycle".to_string(), Self::ScannerCompletedCycles => "completed_cycles".to_string(), Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(), + Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(), + Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(), + Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(), + Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(), + Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(), + Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(), Self::ScannerCurrentScanMode => "current_scan_mode".to_string(), Self::ScannerLastCycleResult => "last_cycle_result".to_string(), Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(), + Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(), + Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(), + Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(), + Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(), + Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(), + Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(), Self::ScannerFailedCycles => "failed_cycles".to_string(), // CPU system-related metrics diff --git a/crates/obs/src/metrics/schema/scanner.rs b/crates/obs/src/metrics/schema/scanner.rs index d25c5fcda..7b3462827 100644 --- a/crates/obs/src/metrics/schema/scanner.rs +++ b/crates/obs/src/metrics/schema/scanner.rs @@ -107,6 +107,60 @@ pub static SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD: LazyLock = La ) }); +pub static SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleObjectsScanned, + "Number of objects scanned by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleDirectoriesScanned, + "Number of directories scanned by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleBucketDriveScans, + "Number of bucket-drive scans finished by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleObjectsPerSecond, + "Object scan rate for the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleDirectoriesPerSecond, + "Directory scan rate for the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleBucketDriveScansPerSecond, + "Bucket-drive scan rate for the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerCurrentScanMode, @@ -134,6 +188,60 @@ pub static SCANNER_LAST_CYCLE_DURATION_SECONDS_MD: LazyLock = ) }); +pub static SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleObjectsScanned, + "Number of objects scanned by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleDirectoriesScanned, + "Number of directories scanned by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleBucketDriveScans, + "Number of bucket-drive scans finished by the last scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleObjectsPerSecond, + "Object scan rate for the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleDirectoriesPerSecond, + "Directory scan rate for the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleBucketDriveScansPerSecond, + "Bucket-drive scan rate for the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_FAILED_CYCLES_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerFailedCycles, diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index 439a432e8..c98ab8b96 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -65,6 +65,14 @@ fn scanner_scan_mode_code(scan_mode: &str) -> u64 { } } +fn scanner_work_rate_per_second(count: u64, seconds: f64) -> f64 { + if seconds > 0.0 && seconds.is_finite() { + count as f64 / seconds + } else { + 0.0 + } +} + const DRIVE_STATE_OK: &str = "ok"; const DRIVE_STATE_ONLINE: &str = "online"; const DRIVE_STATE_UNFORMATTED: &str = "unformatted"; @@ -901,6 +909,8 @@ pub async fn collect_scanner_metric_stats() -> Option { let active_paths = metrics.active_scan_paths as u64; let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now); let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode); + let current_cycle_age = current_cycle_age_seconds as f64; + let last_cycle_duration = metrics.last_cycle_duration_seconds; Some(ScannerStats { bucket_scans_finished, @@ -916,9 +926,33 @@ pub async fn collect_scanner_metric_stats() -> Option { current_cycle: metrics.current_cycle, completed_cycles, current_cycle_age_seconds, + current_cycle_objects_scanned: metrics.current_cycle_objects_scanned, + current_cycle_directories_scanned: metrics.current_cycle_directories_scanned, + current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans, + current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age), + current_cycle_directories_per_second: scanner_work_rate_per_second( + metrics.current_cycle_directories_scanned, + current_cycle_age, + ), + current_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( + metrics.current_cycle_bucket_drive_scans, + current_cycle_age, + ), current_scan_mode, last_cycle_result: metrics.last_cycle_result_code, last_cycle_duration_seconds: metrics.last_cycle_duration_seconds, + last_cycle_objects_scanned: metrics.last_cycle_objects_scanned, + last_cycle_directories_scanned: metrics.last_cycle_directories_scanned, + last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans, + last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration), + last_cycle_directories_per_second: scanner_work_rate_per_second( + metrics.last_cycle_directories_scanned, + last_cycle_duration, + ), + last_cycle_bucket_drive_scans_per_second: scanner_work_rate_per_second( + metrics.last_cycle_bucket_drive_scans, + last_cycle_duration, + ), failed_cycles: metrics.failed_cycles, }) } @@ -1021,4 +1055,16 @@ mod tests { fn scanner_scan_mode_code_maps_unknown_mode() { assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64); } + + #[test] + fn scanner_work_rate_per_second_reports_rate() { + assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0); + } + + #[test] + fn scanner_work_rate_per_second_returns_zero_for_invalid_seconds() { + assert_eq!(scanner_work_rate_per_second(90, 0.0), 0.0); + assert_eq!(scanner_work_rate_per_second(90, f64::INFINITY), 0.0); + assert_eq!(scanner_work_rate_per_second(90, f64::NAN), 0.0); + } } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 2e0396d57..0da43b0a3 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -455,12 +455,14 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc let done_cycle = Metrics::time(Metric::ScanCycle); let cycle_start = std::time::Instant::now(); + let cycle_work_start = global_metrics().start_scan_cycle_work(); if let Err(e) = storeapi .clone() .nsscanner(ctx.clone(), sender, cycle_info.current, scan_mode) .await { error!(duration = ?now.elapsed(), "Fail run data scanner cycle: {e}"); + global_metrics().finish_scan_cycle_work(cycle_work_start); emit_scan_cycle_complete(false, cycle_start.elapsed()); if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) { save_background_heal_info(storeapi.clone(), new_heal_info).await; @@ -468,6 +470,7 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc return; } done_cycle(); + global_metrics().finish_scan_cycle_work(cycle_work_start); emit_scan_cycle_complete(true, cycle_start.elapsed()); if let Some(new_heal_info) = background_heal_info_for_scan_complete(background_heal_info.clone(), scan_mode) { save_background_heal_info(storeapi.clone(), new_heal_info).await;