From 1d46047d6fca285f56aa51e441edf40e8700fe65 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Tue, 2 Jun 2026 09:29:53 +0800 Subject: [PATCH] feat(scanner): expand scanner observability metrics (#3159) * feat(scanner): expand scanner observability metrics * chore(scanner): align bucket-drive metric wording --------- Co-authored-by: Henry Guo --- crates/common/src/metrics.rs | 364 +++++++++++++++++- crates/obs/src/metrics/collectors/scanner.rs | 245 +++++++++++- .../src/metrics/schema/entry/metric_name.rs | 44 +++ crates/obs/src/metrics/schema/scanner.rs | 202 +++++++++- crates/obs/src/metrics/stats_collector.rs | 55 ++- crates/scanner/src/scanner.rs | 55 ++- crates/scanner/src/scanner_folder.rs | 21 +- crates/scanner/src/scanner_io.rs | 39 +- crates/scanner/src/sleeper.rs | 27 +- 9 files changed, 996 insertions(+), 56 deletions(-) diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index dd98a4e33..f6b7f0e64 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -146,6 +146,8 @@ pub enum Metric { ScanCycle, // Full cycle, cluster global. ScanBucketDrive, // Single bucket on one drive. CompactFolder, // Folder compacted. + ScanBucketDriveStart, + ScanBucketDriveFailure, // Must be last: Last, @@ -179,6 +181,8 @@ impl Metric { Self::ScanCycle => "scan_cycle", Self::ScanBucketDrive => "scan_bucket_drive", Self::CompactFolder => "compact_folder", + Self::ScanBucketDriveStart => "scan_bucket_drive_start", + Self::ScanBucketDriveFailure => "scan_bucket_drive_failure", Self::Last => "last", } } @@ -213,7 +217,9 @@ impl Metric { 21 => Some(Self::ScanCycle), 22 => Some(Self::ScanBucketDrive), 23 => Some(Self::CompactFolder), - 24 => Some(Self::Last), + 24 => Some(Self::ScanBucketDriveStart), + 25 => Some(Self::ScanBucketDriveFailure), + 26 => Some(Self::Last), _ => None, } } @@ -346,12 +352,35 @@ pub struct Metrics { current_scan_cycle_objects_start: AtomicU64, current_scan_cycle_directories_start: AtomicU64, current_scan_cycle_bucket_drive_scans_start: AtomicU64, + current_scan_cycle_bucket_drive_failures_start: AtomicU64, + current_scan_cycle_yield_events_start: AtomicU64, + current_scan_cycle_yield_duration_millis_start: AtomicU64, + current_scan_cycle_ilm_actions_start: AtomicU64, + current_scan_cycle_heal_objects_start: AtomicU64, + current_scan_cycle_replication_checks_start: AtomicU64, + current_scan_cycle_usage_saves_start: AtomicU64, last_scan_cycle_result: AtomicU8, last_scan_cycle_duration_millis: AtomicU64, last_scan_cycle_objects_scanned: AtomicU64, last_scan_cycle_directories_scanned: AtomicU64, last_scan_cycle_bucket_drive_scans: AtomicU64, + last_scan_cycle_bucket_drive_failures: AtomicU64, + last_scan_cycle_yield_events: AtomicU64, + last_scan_cycle_yield_duration_millis: AtomicU64, + last_scan_cycle_ilm_actions: AtomicU64, + last_scan_cycle_heal_objects: AtomicU64, + last_scan_cycle_replication_checks: AtomicU64, + last_scan_cycle_usage_saves: AtomicU64, failed_scan_cycles: AtomicU64, + scanner_yield_duration_millis: AtomicU64, + scanner_ilm_actions: AtomicU64, + scanner_throttle_idle_mode_enabled: AtomicBool, + scanner_throttle_sleep_factor_micros: AtomicU64, + scanner_throttle_max_sleep_millis: AtomicU64, + scanner_yield_every_n_objects: AtomicU64, + scanner_cycle_interval_millis: AtomicU64, + scanner_bitrot_cycle_enabled: AtomicBool, + scanner_bitrot_cycle_millis: AtomicU64, } const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0; @@ -374,6 +403,13 @@ pub struct ScanCycleWorkSnapshot { objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64, + bucket_drive_failures: u64, + yield_events: u64, + yield_duration_millis: u64, + ilm_actions: u64, + heal_objects: u64, + replication_checks: u64, + usage_saves: u64, } #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -409,6 +445,20 @@ pub struct ScannerMetricsReport { pub current_cycle_directories_scanned: u64, #[serde(default)] pub current_cycle_bucket_drive_scans: u64, + #[serde(default)] + pub current_cycle_bucket_drive_failures: u64, + #[serde(default)] + pub current_cycle_yield_events: u64, + #[serde(default)] + pub current_cycle_yield_duration_seconds: f64, + #[serde(default)] + pub current_cycle_ilm_actions: u64, + #[serde(default)] + pub current_cycle_heal_objects: u64, + #[serde(default)] + pub current_cycle_replication_checks: u64, + #[serde(default)] + pub current_cycle_usage_saves: u64, pub last_cycle_result: String, pub last_cycle_result_code: u64, pub last_cycle_duration_seconds: f64, @@ -418,7 +468,35 @@ pub struct ScannerMetricsReport { pub last_cycle_directories_scanned: u64, #[serde(default)] pub last_cycle_bucket_drive_scans: u64, + #[serde(default)] + pub last_cycle_bucket_drive_failures: u64, + #[serde(default)] + pub last_cycle_yield_events: u64, + #[serde(default)] + pub last_cycle_yield_duration_seconds: f64, + #[serde(default)] + pub last_cycle_ilm_actions: u64, + #[serde(default)] + pub last_cycle_heal_objects: u64, + #[serde(default)] + pub last_cycle_replication_checks: u64, + #[serde(default)] + pub last_cycle_usage_saves: u64, pub failed_cycles: u64, + #[serde(default)] + pub throttle_idle_mode_enabled: bool, + #[serde(default)] + pub throttle_sleep_factor: f64, + #[serde(default)] + pub throttle_max_sleep_seconds: f64, + #[serde(default)] + pub yield_every_n_objects: u64, + #[serde(default)] + pub cycle_interval_seconds: f64, + #[serde(default)] + pub bitrot_cycle_enabled: bool, + #[serde(default)] + pub bitrot_cycle_seconds: f64, } impl CurrentCycle { @@ -464,6 +542,14 @@ fn duration_millis_saturated(duration: Duration) -> u64 { duration.as_millis().min(u64::MAX as u128) as u64 } +fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 { + if !value.is_finite() || value <= 0.0 { + return 0; + } + + (value * scale).round().clamp(0.0, u64::MAX as f64) as u64 +} + pub fn emit_scan_cycle_complete(success: bool, duration: Duration) { let result = if success { SCAN_CYCLE_RESULT_SUCCESS_LABEL @@ -514,12 +600,35 @@ impl Metrics { current_scan_cycle_objects_start: AtomicU64::new(0), current_scan_cycle_directories_start: AtomicU64::new(0), current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0), + current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0), + current_scan_cycle_yield_events_start: AtomicU64::new(0), + current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0), + current_scan_cycle_ilm_actions_start: AtomicU64::new(0), + current_scan_cycle_heal_objects_start: AtomicU64::new(0), + current_scan_cycle_replication_checks_start: AtomicU64::new(0), + current_scan_cycle_usage_saves_start: AtomicU64::new(0), last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN), last_scan_cycle_duration_millis: AtomicU64::new(0), last_scan_cycle_objects_scanned: AtomicU64::new(0), last_scan_cycle_directories_scanned: AtomicU64::new(0), last_scan_cycle_bucket_drive_scans: AtomicU64::new(0), + last_scan_cycle_bucket_drive_failures: AtomicU64::new(0), + last_scan_cycle_yield_events: AtomicU64::new(0), + last_scan_cycle_yield_duration_millis: AtomicU64::new(0), + last_scan_cycle_ilm_actions: AtomicU64::new(0), + last_scan_cycle_heal_objects: AtomicU64::new(0), + last_scan_cycle_replication_checks: AtomicU64::new(0), + last_scan_cycle_usage_saves: AtomicU64::new(0), failed_scan_cycles: AtomicU64::new(0), + scanner_yield_duration_millis: AtomicU64::new(0), + scanner_ilm_actions: AtomicU64::new(0), + scanner_throttle_idle_mode_enabled: AtomicBool::new(false), + scanner_throttle_sleep_factor_micros: AtomicU64::new(0), + scanner_throttle_max_sleep_millis: AtomicU64::new(0), + scanner_yield_every_n_objects: AtomicU64::new(0), + scanner_cycle_interval_millis: AtomicU64::new(0), + scanner_bitrot_cycle_enabled: AtomicBool::new(false), + scanner_bitrot_cycle_millis: AtomicU64::new(0), } } @@ -621,6 +730,58 @@ impl Metrics { } } + pub fn record_scanner_yield(&self, duration: Duration) { + let metric_idx = Metric::Yield as usize; + self.operations[metric_idx].fetch_add(1, Ordering::Relaxed); + if metric_idx < Metric::LastRealtime as usize { + self.latency[metric_idx].add(duration); + } + let duration_millis = duration_millis_saturated(duration); + let _ = self + .scanner_yield_duration_millis + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_add(duration_millis)) + }); + } + + pub fn record_scanner_ilm_action(&self, count: u64) { + self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed); + } + + pub fn record_scan_bucket_drive_start(&self) { + self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed); + } + + pub fn record_scan_bucket_drive_failure(&self) { + self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed); + } + + pub fn record_scanner_throttle_config( + &self, + idle_mode_enabled: bool, + sleep_factor: f64, + max_sleep: Duration, + yield_every_n_objects: u64, + ) { + self.scanner_throttle_idle_mode_enabled + .store(idle_mode_enabled, Ordering::Relaxed); + self.scanner_throttle_sleep_factor_micros + .store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed); + self.scanner_throttle_max_sleep_millis + .store(duration_millis_saturated(max_sleep), Ordering::Relaxed); + self.scanner_yield_every_n_objects + .store(yield_every_n_objects, Ordering::Relaxed); + } + + pub fn record_scanner_cycle_config(&self, cycle_interval: Duration, bitrot_cycle: Option) { + self.scanner_cycle_interval_millis + .store(duration_millis_saturated(cycle_interval), Ordering::Relaxed); + self.scanner_bitrot_cycle_enabled + .store(bitrot_cycle.is_some(), Ordering::Relaxed); + self.scanner_bitrot_cycle_millis + .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed); + } + // ----------------------------------------------------------------------- // Read-side helpers // ----------------------------------------------------------------------- @@ -686,13 +847,27 @@ impl Metrics { .store(snapshot.directories_scanned, Ordering::Relaxed); self.current_scan_cycle_bucket_drive_scans_start .store(snapshot.bucket_drive_scans, Ordering::Relaxed); + self.current_scan_cycle_bucket_drive_failures_start + .store(snapshot.bucket_drive_failures, Ordering::Relaxed); + self.current_scan_cycle_yield_events_start + .store(snapshot.yield_events, Ordering::Relaxed); + self.current_scan_cycle_yield_duration_millis_start + .store(snapshot.yield_duration_millis, Ordering::Relaxed); + self.current_scan_cycle_ilm_actions_start + .store(snapshot.ilm_actions, Ordering::Relaxed); + self.current_scan_cycle_heal_objects_start + .store(snapshot.heal_objects, Ordering::Relaxed); + self.current_scan_cycle_replication_checks_start + .store(snapshot.replication_checks, Ordering::Relaxed); + self.current_scan_cycle_usage_saves_start + .store(snapshot.usage_saves, Ordering::Relaxed); self.current_scan_cycle_work_active.store(true, Ordering::Relaxed); snapshot } pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) { let work = self.scan_cycle_work_since(start); - self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans); + self.record_scan_cycle_work(work); self.current_scan_cycle_work_active.store(false, Ordering::Relaxed); } @@ -701,6 +876,13 @@ impl Metrics { objects_scanned: self.lifetime(Metric::ScanObject), directories_scanned: self.lifetime(Metric::ScanFolder), bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive), + bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure), + yield_events: self.lifetime(Metric::Yield), + yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed), + ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed), + heal_objects: self.lifetime(Metric::HealAbandonedObject), + replication_checks: self.lifetime(Metric::CheckReplication), + usage_saves: self.lifetime(Metric::SaveUsage), } } @@ -709,6 +891,13 @@ impl Metrics { objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed), directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed), bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed), + bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed), + yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed), + yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed), + ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed), + heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed), + replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed), + usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed), } } @@ -718,17 +907,35 @@ impl Metrics { objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned), directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned), bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans), + bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures), + yield_events: current.yield_events.saturating_sub(start.yield_events), + yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis), + ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions), + heal_objects: current.heal_objects.saturating_sub(start.heal_objects), + replication_checks: current.replication_checks.saturating_sub(start.replication_checks), + usage_saves: current.usage_saves.saturating_sub(start.usage_saves), } } - pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) { + pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) { // Telemetry-only gauges: readers may observe a transient mixed snapshot // while these independent atomic fields are updated. - self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed); + self.last_scan_cycle_objects_scanned + .store(work.objects_scanned, Ordering::Relaxed); self.last_scan_cycle_directories_scanned - .store(directories_scanned, Ordering::Relaxed); + .store(work.directories_scanned, Ordering::Relaxed); self.last_scan_cycle_bucket_drive_scans - .store(bucket_drive_scans, Ordering::Relaxed); + .store(work.bucket_drive_scans, Ordering::Relaxed); + self.last_scan_cycle_bucket_drive_failures + .store(work.bucket_drive_failures, Ordering::Relaxed); + self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed); + self.last_scan_cycle_yield_duration_millis + .store(work.yield_duration_millis, Ordering::Relaxed); + self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed); + self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed); + self.last_scan_cycle_replication_checks + .store(work.replication_checks, Ordering::Relaxed); + self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed); } /// Snapshot of every path currently being scanned. @@ -772,6 +979,13 @@ impl Metrics { m.current_cycle_objects_scanned = current_work.objects_scanned; m.current_cycle_directories_scanned = current_work.directories_scanned; m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans; + m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures; + m.current_cycle_yield_events = current_work.yield_events; + m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0; + m.current_cycle_ilm_actions = current_work.ilm_actions; + m.current_cycle_heal_objects = current_work.heal_objects; + m.current_cycle_replication_checks = current_work.replication_checks; + m.current_cycle_usage_saves = current_work.usage_saves; } let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed); m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string(); @@ -780,7 +994,21 @@ impl Metrics { m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed); m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed); m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed); + m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed); + m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed); + m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0; + m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed); + m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed); + m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed); + m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed); m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed); + m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed); + m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0; + m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0; + m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed); + m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0; + m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed); + m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0; // Lifetime operation counts for i in 0..Metric::Last as usize { @@ -998,13 +1226,43 @@ mod tests { #[tokio::test] async fn report_includes_last_scan_cycle_work() { let metrics = Metrics::new(); - metrics.record_scan_cycle_work(11, 7, 3); + metrics.record_scan_cycle_work(ScanCycleWorkSnapshot { + objects_scanned: 11, + directories_scanned: 7, + bucket_drive_scans: 3, + bucket_drive_failures: 2, + yield_events: 5, + yield_duration_millis: 250, + ilm_actions: 13, + heal_objects: 2, + replication_checks: 4, + usage_saves: 6, + }); let report = metrics.report().await; assert_eq!(report.last_cycle_objects_scanned, 11); assert_eq!(report.last_cycle_directories_scanned, 7); assert_eq!(report.last_cycle_bucket_drive_scans, 3); + assert_eq!(report.last_cycle_bucket_drive_failures, 2); + assert_eq!(report.last_cycle_yield_events, 5); + assert_eq!(report.last_cycle_yield_duration_seconds, 0.25); + assert_eq!(report.last_cycle_ilm_actions, 13); + assert_eq!(report.last_cycle_heal_objects, 2); + assert_eq!(report.last_cycle_replication_checks, 4); + assert_eq!(report.last_cycle_usage_saves, 6); + } + + #[tokio::test] + async fn report_includes_bucket_drive_scan_starts() { + let metrics = Metrics::new(); + metrics.record_scan_bucket_drive_start(); + metrics.record_scan_bucket_drive_failure(); + + let report = metrics.report().await; + + assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1)); + assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1)); } #[tokio::test] @@ -1013,17 +1271,39 @@ mod tests { metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed); metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed); metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed); + metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed); + metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed); + metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed); + metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed); + metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed); + metrics.actions[IlmAction::DeleteAction as usize].store(100, Ordering::Relaxed); + metrics.record_scanner_ilm_action(6); + metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed); let start = metrics.start_scan_cycle_work(); metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed); metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed); metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed); + metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed); + metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed); + metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed); + metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed); + metrics.actions[IlmAction::DeleteAction as usize].store(150, Ordering::Relaxed); + metrics.record_scanner_ilm_action(9); + metrics.record_scanner_yield(Duration::from_millis(150)); let report = metrics.report().await; assert_eq!(report.current_cycle_objects_scanned, 7); assert_eq!(report.current_cycle_directories_scanned, 3); assert_eq!(report.current_cycle_bucket_drive_scans, 2); + assert_eq!(report.current_cycle_bucket_drive_failures, 3); + assert_eq!(report.current_cycle_yield_events, 1); + assert_eq!(report.current_cycle_yield_duration_seconds, 0.15); + assert_eq!(report.current_cycle_ilm_actions, 9); + assert_eq!(report.current_cycle_heal_objects, 3); + assert_eq!(report.current_cycle_replication_checks, 6); + assert_eq!(report.current_cycle_usage_saves, 2); metrics.finish_scan_cycle_work(start); let report = metrics.report().await; @@ -1031,8 +1311,78 @@ mod tests { assert_eq!(report.current_cycle_objects_scanned, 0); assert_eq!(report.current_cycle_directories_scanned, 0); assert_eq!(report.current_cycle_bucket_drive_scans, 0); + assert_eq!(report.current_cycle_bucket_drive_failures, 0); + assert_eq!(report.current_cycle_yield_events, 0); + assert_eq!(report.current_cycle_yield_duration_seconds, 0.0); + assert_eq!(report.current_cycle_ilm_actions, 0); + assert_eq!(report.current_cycle_heal_objects, 0); + assert_eq!(report.current_cycle_replication_checks, 0); + assert_eq!(report.current_cycle_usage_saves, 0); assert_eq!(report.last_cycle_objects_scanned, 7); assert_eq!(report.last_cycle_directories_scanned, 3); assert_eq!(report.last_cycle_bucket_drive_scans, 2); + assert_eq!(report.last_cycle_bucket_drive_failures, 3); + assert_eq!(report.last_cycle_yield_events, 1); + assert_eq!(report.last_cycle_yield_duration_seconds, 0.15); + assert_eq!(report.last_cycle_ilm_actions, 9); + assert_eq!(report.last_cycle_heal_objects, 3); + assert_eq!(report.last_cycle_replication_checks, 6); + assert_eq!(report.last_cycle_usage_saves, 2); + } + + #[test] + fn record_scanner_yield_tracks_count_and_duration() { + let metrics = Metrics::new(); + + metrics.record_scanner_yield(Duration::from_millis(42)); + + assert_eq!(metrics.lifetime(Metric::Yield), 1); + assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42); + assert_eq!(metrics.last_minute(Metric::Yield).n, 1); + } + + #[tokio::test] + async fn report_includes_scanner_throttle_config() { + let metrics = Metrics::new(); + + metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128); + let report = metrics.report().await; + + assert!(report.throttle_idle_mode_enabled); + assert_eq!(report.throttle_sleep_factor, 2.5); + assert_eq!(report.throttle_max_sleep_seconds, 1.5); + assert_eq!(report.yield_every_n_objects, 128); + } + + #[tokio::test] + async fn scanner_throttle_config_rejects_invalid_sleep_factor() { + let metrics = Metrics::new(); + + metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0); + let report = metrics.report().await; + + assert!(!report.throttle_idle_mode_enabled); + assert_eq!(report.throttle_sleep_factor, 0.0); + assert_eq!(report.throttle_max_sleep_seconds, 0.0); + assert_eq!(report.yield_every_n_objects, 0); + } + + #[tokio::test] + async fn report_includes_scanner_cycle_config() { + let metrics = Metrics::new(); + + metrics.record_scanner_cycle_config(Duration::from_secs(3600), Some(Duration::from_secs(86400))); + let report = metrics.report().await; + + assert_eq!(report.cycle_interval_seconds, 3600.0); + assert!(report.bitrot_cycle_enabled); + assert_eq!(report.bitrot_cycle_seconds, 86400.0); + + metrics.record_scanner_cycle_config(Duration::from_secs(60), None); + let report = metrics.report().await; + + assert_eq!(report.cycle_interval_seconds, 60.0); + assert!(!report.bitrot_cycle_enabled); + assert_eq!(report.bitrot_cycle_seconds, 0.0); } } diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index ffdeb164b..b11c4ff80 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -16,29 +16,40 @@ //! Scanner metrics collector. //! -//! Collects background scanner metrics including bucket scans, +//! Collects background scanner metrics including bucket-drive scans, //! directory scans, and object scans. use crate::metrics::report::PrometheusMetric; use crate::metrics::schema::scanner::{ - SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD, - SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, - SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, - SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, - SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD, - SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, + SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD, + SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD, + SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD, + SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, + SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, + SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD, + SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, + SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, + SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD, + SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD, + SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, - SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, - SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD, + SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, + SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, + SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD, + SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD, + SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, + SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD, }; /// Scanner statistics. #[derive(Debug, Clone, Default)] pub struct ScannerStats { - /// Number of bucket scans finished + /// Number of bucket-drive scans finished pub bucket_scans_finished: u64, - /// Number of bucket scans started + /// Number of bucket-drive scans started pub bucket_scans_started: u64, + /// Number of bucket-drive scans that failed + pub bucket_scans_failed: u64, /// Number of directories scanned pub directories_scanned: u64, /// Number of objects scanned @@ -49,6 +60,20 @@ pub struct ScannerStats { pub last_activity_seconds: u64, /// Number of scanner paths currently being processed pub active_paths: u64, + /// Whether scanner idle-mode self-throttling is enabled + pub throttle_idle_mode_enabled: u64, + /// Effective scanner sleep factor + pub throttle_sleep_factor: f64, + /// Effective scanner maximum self-throttle sleep duration in seconds + pub throttle_max_sleep_seconds: f64, + /// Object interval for cooperative scanner runtime yields + pub yield_every_n_objects: u64, + /// Effective scanner cycle interval in seconds + pub cycle_interval_seconds: f64, + /// Whether periodic scanner bitrot deep scans are enabled + pub bitrot_cycle_enabled: u64, + /// Effective scanner bitrot deep-scan interval in seconds + pub bitrot_cycle_seconds: f64, /// Current scanner cycle number, or zero when idle pub current_cycle: u64, /// Number of scanner cycles completed since server start @@ -61,12 +86,26 @@ pub struct ScannerStats { pub current_cycle_directories_scanned: u64, /// Number of bucket-drive scans finished by the currently running scanner cycle pub current_cycle_bucket_drive_scans: u64, + /// Number of bucket-drive scans that failed in the currently running scanner cycle + pub current_cycle_bucket_drive_failures: u64, /// Object scan rate for the currently running scanner cycle pub current_cycle_objects_per_second: f64, /// Directory scan rate for the currently running scanner cycle pub current_cycle_directories_per_second: f64, /// Bucket-drive scan rate for the currently running scanner cycle pub current_cycle_bucket_drive_scans_per_second: f64, + /// Number of scanner self-throttle yield events in the current scanner cycle + pub current_cycle_yield_events: u64, + /// Total scanner self-throttle yield duration in seconds for the current scanner cycle + pub current_cycle_yield_duration_seconds: f64, + /// Number of lifecycle actions applied by the current scanner cycle + pub current_cycle_ilm_actions: u64, + /// Number of object heal candidates enqueued by the current scanner cycle + pub current_cycle_heal_objects: u64, + /// Number of replication heal checks run by the current scanner cycle + pub current_cycle_replication_checks: u64, + /// Number of data-usage save operations run by the current scanner cycle + pub current_cycle_usage_saves: u64, /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan pub current_scan_mode: u64, /// Last scanner cycle result: 0 unknown, 1 success, 2 error @@ -79,12 +118,26 @@ pub struct ScannerStats { pub last_cycle_directories_scanned: u64, /// Number of bucket-drive scans finished by the last scanner cycle pub last_cycle_bucket_drive_scans: u64, + /// Number of bucket-drive scans that failed in the last finished scanner cycle + pub last_cycle_bucket_drive_failures: u64, /// Object scan rate for the last finished scanner cycle pub last_cycle_objects_per_second: f64, /// Directory scan rate for the last finished scanner cycle pub last_cycle_directories_per_second: f64, /// Bucket-drive scan rate for the last finished scanner cycle pub last_cycle_bucket_drive_scans_per_second: f64, + /// Number of scanner self-throttle yield events in the last finished scanner cycle + pub last_cycle_yield_events: u64, + /// Total scanner self-throttle yield duration in seconds for the last finished scanner cycle + pub last_cycle_yield_duration_seconds: f64, + /// Number of lifecycle actions applied by the last finished scanner cycle + pub last_cycle_ilm_actions: u64, + /// Number of object heal candidates enqueued by the last finished scanner cycle + pub last_cycle_heal_objects: u64, + /// Number of replication heal checks run by the last finished scanner cycle + pub last_cycle_replication_checks: u64, + /// Number of data-usage save operations run by the last finished scanner cycle + pub last_cycle_usage_saves: u64, /// Number of scanner cycles that failed since server start pub failed_cycles: u64, } @@ -97,11 +150,19 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { vec![ PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64), PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64), + PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64), PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64), PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64), + PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, stats.throttle_idle_mode_enabled as f64), + PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_SLEEP_FACTOR_MD, stats.throttle_sleep_factor), + PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, stats.throttle_max_sleep_seconds), + PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64), + PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds), + PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, stats.bitrot_cycle_enabled as f64), + PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64), PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64), @@ -114,6 +175,10 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.current_cycle_bucket_drive_scans as f64, ), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD, + stats.current_cycle_bucket_drive_failures as f64, + ), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second), PrometheusMetric::from_descriptor( &SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, @@ -123,6 +188,18 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, stats.current_cycle_bucket_drive_scans_per_second, ), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, stats.current_cycle_yield_events as f64), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, + stats.current_cycle_yield_duration_seconds, + ), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, stats.current_cycle_ilm_actions as f64), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, stats.current_cycle_heal_objects as f64), + PrometheusMetric::from_descriptor( + &SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, + stats.current_cycle_replication_checks as f64, + ), + PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds), @@ -132,12 +209,22 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec { stats.last_cycle_directories_scanned as f64, ), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64), + PrometheusMetric::from_descriptor( + &SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, + stats.last_cycle_bucket_drive_failures as f64, + ), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second), PrometheusMetric::from_descriptor( &SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, stats.last_cycle_bucket_drive_scans_per_second, ), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, stats.last_cycle_yield_events as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, stats.last_cycle_yield_duration_seconds), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, stats.last_cycle_ilm_actions as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, stats.last_cycle_heal_objects as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64), + PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64), PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64), ] } @@ -152,36 +239,58 @@ mod tests { let stats = ScannerStats { bucket_scans_finished: 100, bucket_scans_started: 100, + bucket_scans_failed: 2, directories_scanned: 50000, objects_scanned: 1000000, versions_scanned: 1500000, last_activity_seconds: 30, active_paths: 4, + throttle_idle_mode_enabled: 1, + throttle_sleep_factor: 10.0, + throttle_max_sleep_seconds: 15.0, + yield_every_n_objects: 128, + cycle_interval_seconds: 3600.0, + bitrot_cycle_enabled: 1, + bitrot_cycle_seconds: 86400.0, current_cycle: 12, completed_cycles: 11, current_cycle_age_seconds: 90, current_cycle_objects_scanned: 250, current_cycle_directories_scanned: 20, current_cycle_bucket_drive_scans: 2, + current_cycle_bucket_drive_failures: 1, current_cycle_objects_per_second: 12.5, current_cycle_directories_per_second: 1.0, current_cycle_bucket_drive_scans_per_second: 0.1, + current_cycle_yield_events: 8, + current_cycle_yield_duration_seconds: 1.25, + current_cycle_ilm_actions: 6, + current_cycle_heal_objects: 2, + current_cycle_replication_checks: 5, + current_cycle_usage_saves: 3, current_scan_mode: 2, last_cycle_result: 1, last_cycle_duration_seconds: 42.5, last_cycle_objects_scanned: 900, last_cycle_directories_scanned: 80, last_cycle_bucket_drive_scans: 6, + last_cycle_bucket_drive_failures: 2, last_cycle_objects_per_second: 18.0, last_cycle_directories_per_second: 1.6, last_cycle_bucket_drive_scans_per_second: 0.12, + last_cycle_yield_events: 30, + last_cycle_yield_duration_seconds: 9.5, + last_cycle_ilm_actions: 44, + last_cycle_heal_objects: 7, + last_cycle_replication_checks: 12, + last_cycle_usage_saves: 9, failed_cycles: 3, }; let metrics = collect_scanner_metrics(&stats); report_metrics(&metrics); - assert_eq!(metrics.len(), 26); + assert_eq!(metrics.len(), 48); let objects = metrics.iter().find(|m| m.value == 1000000.0); assert!(objects.is_some()); @@ -194,6 +303,46 @@ mod tests { .find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name()); assert_eq!(active_paths.map(|m| m.value), Some(4.0)); + let bucket_scans_failed = metrics + .iter() + .find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name()); + assert_eq!(bucket_scans_failed.map(|m| m.value), Some(2.0)); + + let throttle_idle_mode_enabled = metrics + .iter() + .find(|m| m.name == SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD.get_full_metric_name()); + assert_eq!(throttle_idle_mode_enabled.map(|m| m.value), Some(1.0)); + + let throttle_sleep_factor = metrics + .iter() + .find(|m| m.name == SCANNER_THROTTLE_SLEEP_FACTOR_MD.get_full_metric_name()); + assert_eq!(throttle_sleep_factor.map(|m| m.value), Some(10.0)); + + let throttle_max_sleep = metrics + .iter() + .find(|m| m.name == SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD.get_full_metric_name()); + assert_eq!(throttle_max_sleep.map(|m| m.value), Some(15.0)); + + let yield_every_n_objects = metrics + .iter() + .find(|m| m.name == SCANNER_YIELD_EVERY_N_OBJECTS_MD.get_full_metric_name()); + assert_eq!(yield_every_n_objects.map(|m| m.value), Some(128.0)); + + let cycle_interval_seconds = metrics + .iter() + .find(|m| m.name == SCANNER_CYCLE_INTERVAL_SECONDS_MD.get_full_metric_name()); + assert_eq!(cycle_interval_seconds.map(|m| m.value), Some(3600.0)); + + let bitrot_cycle_enabled = metrics + .iter() + .find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name()); + assert_eq!(bitrot_cycle_enabled.map(|m| m.value), Some(1.0)); + + let bitrot_cycle_seconds = metrics + .iter() + .find(|m| m.name == SCANNER_BITROT_CYCLE_SECONDS_MD.get_full_metric_name()); + assert_eq!(bitrot_cycle_seconds.map(|m| m.value), Some(86400.0)); + let current_cycle = metrics .iter() .find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name()); @@ -224,6 +373,11 @@ mod tests { .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0)); + let current_cycle_bucket_drive_failures = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name()); + assert_eq!(current_cycle_bucket_drive_failures.map(|m| m.value), Some(1.0)); + let current_cycle_objects_rate = metrics .iter() .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); @@ -239,6 +393,36 @@ mod tests { .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1)); + let current_cycle_yield_events = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD.get_full_metric_name()); + assert_eq!(current_cycle_yield_events.map(|m| m.value), Some(8.0)); + + let current_cycle_yield_duration = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name()); + assert_eq!(current_cycle_yield_duration.map(|m| m.value), Some(1.25)); + + let current_cycle_ilm_actions = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD.get_full_metric_name()); + assert_eq!(current_cycle_ilm_actions.map(|m| m.value), Some(6.0)); + + let current_cycle_heal_objects = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name()); + assert_eq!(current_cycle_heal_objects.map(|m| m.value), Some(2.0)); + + let current_cycle_replication_checks = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name()); + assert_eq!(current_cycle_replication_checks.map(|m| m.value), Some(5.0)); + + let current_cycle_usage_saves = metrics + .iter() + .find(|m| m.name == SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD.get_full_metric_name()); + assert_eq!(current_cycle_usage_saves.map(|m| m.value), Some(3.0)); + let current_scan_mode = metrics .iter() .find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name()); @@ -269,6 +453,11 @@ mod tests { .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0)); + let last_cycle_bucket_drive_failures = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name()); + assert_eq!(last_cycle_bucket_drive_failures.map(|m| m.value), Some(2.0)); + let last_cycle_objects_rate = metrics .iter() .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); @@ -284,6 +473,36 @@ mod tests { .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12)); + let last_cycle_yield_events = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_EVENTS_MD.get_full_metric_name()); + assert_eq!(last_cycle_yield_events.map(|m| m.value), Some(30.0)); + + let last_cycle_yield_duration = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name()); + assert_eq!(last_cycle_yield_duration.map(|m| m.value), Some(9.5)); + + let last_cycle_ilm_actions = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_ILM_ACTIONS_MD.get_full_metric_name()); + assert_eq!(last_cycle_ilm_actions.map(|m| m.value), Some(44.0)); + + let last_cycle_heal_objects = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name()); + assert_eq!(last_cycle_heal_objects.map(|m| m.value), Some(7.0)); + + let last_cycle_replication_checks = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name()); + assert_eq!(last_cycle_replication_checks.map(|m| m.value), Some(12.0)); + + let last_cycle_usage_saves = metrics + .iter() + .find(|m| m.name == SCANNER_LAST_CYCLE_USAGE_SAVES_MD.get_full_metric_name()); + assert_eq!(last_cycle_usage_saves.map(|m| m.value), Some(9.0)); + let failed_cycles = metrics .iter() .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name()); @@ -295,7 +514,7 @@ mod tests { let stats = ScannerStats::default(); let metrics = collect_scanner_metrics(&stats); - assert_eq!(metrics.len(), 26); + assert_eq!(metrics.len(), 48); for metric in &metrics { assert_eq!(metric.value, 0.0); assert!(metric.labels.is_empty()); diff --git a/crates/obs/src/metrics/schema/entry/metric_name.rs b/crates/obs/src/metrics/schema/entry/metric_name.rs index 92ade59a8..ec70d5ba1 100644 --- a/crates/obs/src/metrics/schema/entry/metric_name.rs +++ b/crates/obs/src/metrics/schema/entry/metric_name.rs @@ -279,24 +279,46 @@ pub enum MetricName { ScannerVersionsScanned, ScannerLastActivitySeconds, ScannerActivePaths, + ScannerBucketScansFailed, + ScannerThrottleIdleModeEnabled, + ScannerThrottleSleepFactor, + ScannerThrottleMaxSleepSeconds, + ScannerYieldEveryNObjects, + ScannerCycleIntervalSeconds, + ScannerBitrotCycleEnabled, + ScannerBitrotCycleSeconds, ScannerCurrentCycle, ScannerCompletedCycles, ScannerCurrentCycleAgeSeconds, ScannerCurrentCycleObjectsScanned, ScannerCurrentCycleDirectoriesScanned, ScannerCurrentCycleBucketDriveScans, + ScannerCurrentCycleBucketDriveFailures, ScannerCurrentCycleObjectsPerSecond, ScannerCurrentCycleDirectoriesPerSecond, ScannerCurrentCycleBucketDriveScansPerSecond, + ScannerCurrentCycleYieldEvents, + ScannerCurrentCycleYieldDurationSeconds, + ScannerCurrentCycleIlmActions, + ScannerCurrentCycleHealObjects, + ScannerCurrentCycleReplicationChecks, + ScannerCurrentCycleUsageSaves, ScannerCurrentScanMode, ScannerLastCycleResult, ScannerLastCycleDurationSeconds, ScannerLastCycleObjectsScanned, ScannerLastCycleDirectoriesScanned, ScannerLastCycleBucketDriveScans, + ScannerLastCycleBucketDriveFailures, ScannerLastCycleObjectsPerSecond, ScannerLastCycleDirectoriesPerSecond, ScannerLastCycleBucketDriveScansPerSecond, + ScannerLastCycleYieldEvents, + ScannerLastCycleYieldDurationSeconds, + ScannerLastCycleIlmActions, + ScannerLastCycleHealObjects, + ScannerLastCycleReplicationChecks, + ScannerLastCycleUsageSaves, ScannerFailedCycles, // CPU system-related metrics @@ -639,24 +661,46 @@ impl MetricName { Self::ScannerVersionsScanned => "versions_scanned".to_string(), Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(), Self::ScannerActivePaths => "active_paths".to_string(), + Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(), + Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(), + Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(), + Self::ScannerThrottleMaxSleepSeconds => "throttle_max_sleep_seconds".to_string(), + Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(), + Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(), + Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(), + Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(), Self::ScannerCurrentCycle => "current_cycle".to_string(), Self::ScannerCompletedCycles => "completed_cycles".to_string(), Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(), Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(), Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(), Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(), + Self::ScannerCurrentCycleBucketDriveFailures => "current_cycle_bucket_drive_failures".to_string(), Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(), Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(), Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(), + Self::ScannerCurrentCycleYieldEvents => "current_cycle_yield_events".to_string(), + Self::ScannerCurrentCycleYieldDurationSeconds => "current_cycle_yield_duration_seconds".to_string(), + Self::ScannerCurrentCycleIlmActions => "current_cycle_ilm_actions".to_string(), + Self::ScannerCurrentCycleHealObjects => "current_cycle_heal_objects".to_string(), + Self::ScannerCurrentCycleReplicationChecks => "current_cycle_replication_checks".to_string(), + Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(), Self::ScannerCurrentScanMode => "current_scan_mode".to_string(), Self::ScannerLastCycleResult => "last_cycle_result".to_string(), Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(), Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(), Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(), Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(), + Self::ScannerLastCycleBucketDriveFailures => "last_cycle_bucket_drive_failures".to_string(), Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(), Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(), Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(), + Self::ScannerLastCycleYieldEvents => "last_cycle_yield_events".to_string(), + Self::ScannerLastCycleYieldDurationSeconds => "last_cycle_yield_duration_seconds".to_string(), + Self::ScannerLastCycleIlmActions => "last_cycle_ilm_actions".to_string(), + Self::ScannerLastCycleHealObjects => "last_cycle_heal_objects".to_string(), + Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(), + Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(), Self::ScannerFailedCycles => "failed_cycles".to_string(), // CPU system-related metrics diff --git a/crates/obs/src/metrics/schema/scanner.rs b/crates/obs/src/metrics/schema/scanner.rs index 7b3462827..ac1c16516 100644 --- a/crates/obs/src/metrics/schema/scanner.rs +++ b/crates/obs/src/metrics/schema/scanner.rs @@ -20,7 +20,7 @@ use std::sync::LazyLock; pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerBucketScansFinished, - "Total number of bucket scans finished since server start", + "Total number of bucket-drive scans finished since server start", &[], subsystems::SCANNER, ) @@ -29,7 +29,16 @@ pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock = LazyLo pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerBucketScansStarted, - "Total number of bucket scans started since server start", + "Total number of bucket-drive scans started since server start", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_BUCKET_SCANS_FAILED_MD: LazyLock = LazyLock::new(|| { + new_counter_md( + MetricName::ScannerBucketScansFailed, + "Total number of bucket-drive scans that failed since server start", &[], subsystems::SCANNER, ) @@ -80,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock = LazyLock::new(| ) }); +pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerThrottleIdleModeEnabled, + "Whether scanner idle-mode self-throttling is enabled: 1 enabled, 0 disabled.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_THROTTLE_SLEEP_FACTOR_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerThrottleSleepFactor, + "Effective scanner sleep factor used to compute proportional self-throttle sleeps.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerThrottleMaxSleepSeconds, + "Effective maximum scanner self-throttle sleep duration in seconds.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_YIELD_EVERY_N_OBJECTS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerYieldEveryNObjects, + "Current object interval for cooperative scanner runtime yields, or zero when disabled.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CYCLE_INTERVAL_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCycleIntervalSeconds, + "Effective scanner cycle interval in seconds.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerBitrotCycleEnabled, + "Whether periodic scanner bitrot deep scans are enabled: 1 enabled, 0 disabled.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_BITROT_CYCLE_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerBitrotCycleSeconds, + "Effective scanner bitrot deep-scan interval in seconds; zero means disabled or deep-scan every cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_CURRENT_CYCLE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerCurrentCycle, @@ -134,6 +206,15 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleBucketDriveFailures, + "Number of bucket-drive scans that failed in the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerCurrentCycleObjectsPerSecond, @@ -161,6 +242,60 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleYieldEvents, + "Number of scanner self-throttle yield events in the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleYieldDurationSeconds, + "Total scanner self-throttle yield duration in seconds for the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleIlmActions, + "Number of lifecycle actions applied by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleHealObjects, + "Number of object heal candidates enqueued by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleReplicationChecks, + "Number of replication heal checks run by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerCurrentCycleUsageSaves, + "Number of data-usage save operations run by the currently running scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerCurrentScanMode, @@ -215,6 +350,15 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock ) }); +pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleBucketDriveFailures, + "Number of bucket-drive scans that failed in the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { new_gauge_md( MetricName::ScannerLastCycleObjectsPerSecond, @@ -242,6 +386,60 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleYieldEvents, + "Number of scanner self-throttle yield events in the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleYieldDurationSeconds, + "Total scanner self-throttle yield duration in seconds for the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_ILM_ACTIONS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleIlmActions, + "Number of lifecycle actions applied by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleHealObjects, + "Number of object heal candidates enqueued by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleReplicationChecks, + "Number of replication heal checks run by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + +pub static SCANNER_LAST_CYCLE_USAGE_SAVES_MD: LazyLock = LazyLock::new(|| { + new_gauge_md( + MetricName::ScannerLastCycleUsageSaves, + "Number of data-usage save operations run by the last finished scanner cycle.", + &[], + subsystems::SCANNER, + ) +}); + pub static SCANNER_FAILED_CYCLES_MD: LazyLock = LazyLock::new(|| { new_counter_md( MetricName::ScannerFailedCycles, diff --git a/crates/obs/src/metrics/stats_collector.rs b/crates/obs/src/metrics/stats_collector.rs index c98ab8b96..797c24488 100644 --- a/crates/obs/src/metrics/stats_collector.rs +++ b/crates/obs/src/metrics/stats_collector.rs @@ -896,10 +896,23 @@ pub async fn collect_ilm_metric_stats() -> Option { /// /// Task 5 maps scanner runtime snapshots from `global_metrics()` into the /// rustfs-obs scanner collector shape. +fn scanner_bucket_scans_started(life_time_ops: &HashMap, bucket_scans_finished: u64) -> u64 { + life_time_ops + .get("scan_bucket_drive_start") + .copied() + .unwrap_or(bucket_scans_finished) +} + pub async fn collect_scanner_metric_stats() -> Option { let metrics = global_metrics().report().await; let now = Utc::now(); let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default(); + let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished); + let bucket_scans_failed = metrics + .life_time_ops + .get("scan_bucket_drive_failure") + .copied() + .unwrap_or_default(); let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default(); let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default(); let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default(); @@ -914,21 +927,27 @@ pub async fn collect_scanner_metric_stats() -> Option { Some(ScannerStats { bucket_scans_finished, - // `global_metrics()` currently tracks completed bucket-drive scans, not a - // separate started counter. Mirror the finished count until Task 5/Task 10 - // expands the scanner runtime source shape. - bucket_scans_started: bucket_scans_finished, + bucket_scans_started, + bucket_scans_failed, directories_scanned, objects_scanned, versions_scanned, last_activity_seconds, active_paths, + throttle_idle_mode_enabled: u64::from(metrics.throttle_idle_mode_enabled), + throttle_sleep_factor: metrics.throttle_sleep_factor, + throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds, + yield_every_n_objects: metrics.yield_every_n_objects, + cycle_interval_seconds: metrics.cycle_interval_seconds, + bitrot_cycle_enabled: u64::from(metrics.bitrot_cycle_enabled), + bitrot_cycle_seconds: metrics.bitrot_cycle_seconds, current_cycle: metrics.current_cycle, completed_cycles, current_cycle_age_seconds, current_cycle_objects_scanned: metrics.current_cycle_objects_scanned, current_cycle_directories_scanned: metrics.current_cycle_directories_scanned, current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans, + current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures, current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age), current_cycle_directories_per_second: scanner_work_rate_per_second( metrics.current_cycle_directories_scanned, @@ -938,12 +957,19 @@ pub async fn collect_scanner_metric_stats() -> Option { metrics.current_cycle_bucket_drive_scans, current_cycle_age, ), + current_cycle_yield_events: metrics.current_cycle_yield_events, + current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds, + current_cycle_ilm_actions: metrics.current_cycle_ilm_actions, + current_cycle_heal_objects: metrics.current_cycle_heal_objects, + current_cycle_replication_checks: metrics.current_cycle_replication_checks, + current_cycle_usage_saves: metrics.current_cycle_usage_saves, current_scan_mode, last_cycle_result: metrics.last_cycle_result_code, last_cycle_duration_seconds: metrics.last_cycle_duration_seconds, last_cycle_objects_scanned: metrics.last_cycle_objects_scanned, last_cycle_directories_scanned: metrics.last_cycle_directories_scanned, last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans, + last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures, last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration), last_cycle_directories_per_second: scanner_work_rate_per_second( metrics.last_cycle_directories_scanned, @@ -953,6 +979,12 @@ pub async fn collect_scanner_metric_stats() -> Option { metrics.last_cycle_bucket_drive_scans, last_cycle_duration, ), + last_cycle_yield_events: metrics.last_cycle_yield_events, + last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds, + last_cycle_ilm_actions: metrics.last_cycle_ilm_actions, + last_cycle_heal_objects: metrics.last_cycle_heal_objects, + last_cycle_replication_checks: metrics.last_cycle_replication_checks, + last_cycle_usage_saves: metrics.last_cycle_usage_saves, failed_cycles: metrics.failed_cycles, }) } @@ -1056,6 +1088,21 @@ mod tests { assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64); } + #[test] + fn scanner_bucket_scans_started_uses_explicit_started_count() { + let mut life_time_ops = HashMap::new(); + life_time_ops.insert("scan_bucket_drive_start".to_string(), 7); + + assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 7); + } + + #[test] + fn scanner_bucket_scans_started_falls_back_to_finished_count() { + let life_time_ops = HashMap::new(); + + assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5); + } + #[test] fn scanner_work_rate_per_second_reports_rate() { assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 0da43b0a3..229a9cee0 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -266,8 +266,13 @@ fn bitrot_scan_cycle() -> Option { } } -fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option>) -> HealScanMode { - let Some(bitrot_cycle) = bitrot_scan_cycle() else { +fn get_cycle_scan_mode( + current_cycle: u64, + bitrot_start_cycle: u64, + bitrot_start_time: Option>, + bitrot_cycle: Option, +) -> HealScanMode { + let Some(bitrot_cycle) = bitrot_cycle else { return HealScanMode::Normal; }; @@ -299,8 +304,10 @@ fn background_heal_info_for_scan_start( current_cycle: u64, scan_mode: HealScanMode, now: DateTime, + bitrot_cycle: Option, ) -> Option { - let reset_bitrot_start = scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now); + let reset_bitrot_start = + scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, bitrot_cycle); if info.current_scan_mode == scan_mode && !reset_bitrot_start { return None; } @@ -314,12 +321,17 @@ fn background_heal_info_for_scan_start( Some(info) } -fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now: DateTime) -> bool { +fn should_reset_bitrot_start( + info: &BackgroundHealInfo, + current_cycle: u64, + now: DateTime, + bitrot_cycle: Option, +) -> bool { let Some(bitrot_start_time) = info.bitrot_start_time else { return true; }; - let Some(bitrot_cycle) = bitrot_scan_cycle() else { + let Some(bitrot_cycle) = bitrot_cycle else { return false; }; @@ -424,6 +436,9 @@ fn get_lock_acquire_timeout() -> Duration { async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle) { let _activity_guard = ScannerActivityGuard::new(); SCANNER_SLEEPER.refresh_from_env(); + let configured_cycle_interval = cycle_interval(); + let configured_bitrot_cycle = bitrot_scan_cycle(); + global_metrics().record_scanner_cycle_config(configured_cycle_interval, configured_bitrot_cycle); info!("Start run data scanner cycle"); cycle_info.current = cycle_info.next; let now = Instant::now(); @@ -437,11 +452,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc cycle_info.current, background_heal_info.bitrot_start_cycle, background_heal_info.bitrot_start_time, + configured_bitrot_cycle, ); let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode); - if let Some(new_heal_info) = - background_heal_info_for_scan_start(background_heal_info.clone(), cycle_info.current, scan_mode, Utc::now()) - { + if let Some(new_heal_info) = background_heal_info_for_scan_start( + background_heal_info.clone(), + cycle_info.current, + scan_mode, + Utc::now(), + configured_bitrot_cycle, + ) { background_heal_info = new_heal_info.clone(); save_background_heal_info(storeapi.clone(), new_heal_info).await; } @@ -600,18 +620,22 @@ pub async fn store_data_usage_in_backend( // Save a backup every 10th update if attempts > 10 { let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await { warn!("Failed to save data usage backup to {}: {}", backup_path, e); } + done_save(); attempts = 1; } // Save main configuration + let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await { error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str()); } else { rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await; } + done_save(); attempts += 1; } @@ -866,7 +890,7 @@ mod tests { #[serial] fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { - let mode = get_cycle_scan_mode(10, 0, Some(Utc::now())); + let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle()); assert_eq!(mode, HealScanMode::Deep); }); } @@ -878,8 +902,8 @@ mod tests { let recent = Utc::now() - chrono::Duration::minutes(30); let old = Utc::now() - chrono::Duration::hours(2); - assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal); - assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep); + assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal); + assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep); }); } @@ -887,7 +911,7 @@ mod tests { #[serial] fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || { - assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal); + assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal); }); } @@ -895,8 +919,9 @@ mod tests { #[serial] fn test_background_heal_info_for_scan_start_marks_deep_active() { let now = Utc::now(); - let info = background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now) - .expect("deep scan should update background heal info"); + let info = + background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle()) + .expect("deep scan should update background heal info"); assert_eq!(info.current_scan_mode, HealScanMode::Deep); assert_eq!(info.bitrot_start_cycle, 7); @@ -914,7 +939,7 @@ mod tests { current_scan_mode: HealScanMode::Normal, }; - let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now()) + let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle()) .expect("deep scan should mark active status"); assert_eq!(info.current_scan_mode, HealScanMode::Deep); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index ee904b8dc..6b420f23e 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -16,19 +16,19 @@ use std::collections::HashSet; use std::fs::FileType; use std::io::ErrorKind; use std::sync::{Arc, Once}; -use std::time::{Duration, SystemTime}; +use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path}; use crate::error::ScannerError; use crate::scanner_io::ScannerIODisk as _; -use crate::sleeper::DynamicSleeper; +use crate::sleeper::{DynamicSleeper, scanner_yield_every_n_objects}; use metrics::{counter, describe_counter}; use rustfs_common::heal_channel::{ HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode, send_heal_request_with_admission, }; -use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater}; +use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater, global_metrics}; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule}; use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator; @@ -143,13 +143,6 @@ fn scanner_excess_folders_threshold() -> u64 { ) } -fn scanner_yield_every_n_objects() -> u64 { - rustfs_utils::get_env_u64( - rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS, - rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, - ) -} - fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool { yield_every > 0 && object_count.is_multiple_of(yield_every) } @@ -485,6 +478,7 @@ impl ScannerItem { debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; done_ilm(1)(); + global_metrics().record_scanner_ilm_action(1); break 'eventLoop; } @@ -497,6 +491,7 @@ impl ScannerItem { debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; done_ilm(1)(); + global_metrics().record_scanner_ilm_action(1); } IlmAction::DeleteVersionAction => { remaining_versions -= 1; @@ -510,11 +505,13 @@ impl ScannerItem { } noncurrent_events.push(event.clone()); done_ilm(1)(); + global_metrics().record_scanner_ilm_action(1); } IlmAction::TransitionAction | IlmAction::TransitionVersionAction => { debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action); apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; done_ilm(1)(); + global_metrics().record_scanner_ilm_action(1); } IlmAction::NoneAction | IlmAction::ActionCount => { @@ -560,7 +557,9 @@ impl ScannerItem { return; }; + let done_replication = Metrics::time(Metric::CheckReplication); let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await; + done_replication(); if !Self::should_account_replication_stats(oi) { return; } @@ -1110,7 +1109,9 @@ impl FolderScanner { timer.sleep().await; if should_yield_after_object(object_count, yield_every_objects) { + let yield_start = Instant::now(); tokio::task::yield_now().await; + global_metrics().record_scanner_yield(yield_start.elapsed()); } } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 8d33af5a6..2e6af1e06 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -22,7 +22,7 @@ use futures::future::join_all; use metrics::counter; use rand::seq::SliceRandom as _; use rustfs_common::heal_channel::HealScanMode; -use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete}; +use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, global_metrics}; use rustfs_config::{ DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, @@ -116,6 +116,28 @@ impl Drop for DiskBucketScanActiveGuard { } } +struct BucketDriveFailureGuard { + failed: bool, +} + +impl BucketDriveFailureGuard { + fn new() -> Self { + Self { failed: true } + } + + fn mark_success(&mut self) { + self.failed = false; + } +} + +impl Drop for BucketDriveFailureGuard { + fn drop(&mut self) { + if self.failed { + global_metrics().record_scan_bucket_drive_failure(); + } + } +} + fn decrement_atomic_usize(counter: &AtomicUsize) -> usize { counter .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1))) @@ -217,9 +239,11 @@ async fn persist_and_publish_cache_snapshot( ) -> Option { let last_update = cache_snapshot.info.last_update; + let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await { error!("Failed to save data usage cache: {}", e); } + done_save(); if let Err(e) = updates.send(cache_snapshot).await { error!("Failed to send data usage cache: {}", e); @@ -732,9 +756,12 @@ impl ScannerIOCache for SetDisks { if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before) && last_update > before_update - && let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await { - error!("Failed to save data usage cache: {}", e); + let done_save = Metrics::time(Metric::SaveUsage); + if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await { + error!("Failed to save data usage cache: {}", e); + } + done_save(); } if let Err(e) = update_fut.await { @@ -775,9 +802,11 @@ impl ScannerIOCache for SetDisks { error!("nsscanner_disk: Failed to send data usage entry info: {}", e); } + let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await { error!("nsscanner_disk: Failed to save data usage cache: {}", e); } + done_save(); } })); } @@ -900,6 +929,8 @@ impl ScannerIODisk for Disk { let drive_start = std::time::Instant::now(); let bucket = cache.info.name.clone(); let disk_path = self.path().to_string_lossy().to_string(); + global_metrics().record_scan_bucket_drive_start(); + let mut failure_guard = BucketDriveFailureGuard::new(); let _guard = self.start_scan(); let mut cache = cache; @@ -966,9 +997,11 @@ impl ScannerIODisk for Disk { done_drive(); emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed()); data_usage_info.info.last_update = Some(SystemTime::now()); + failure_guard.mark_success(); Ok(data_usage_info) } Err(e) => { + done_drive(); emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed()); Err(StorageError::other(format!("Failed to scan data folder: {e}"))) } diff --git a/crates/scanner/src/sleeper.rs b/crates/scanner/src/sleeper.rs index ab9b97ba4..fbdf9a944 100644 --- a/crates/scanner/src/sleeper.rs +++ b/crates/scanner/src/sleeper.rs @@ -16,7 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::{Arc, LazyLock, RwLock}; use std::time::Instant; -use rustfs_config::{DEFAULT_SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, ScannerSpeed}; +use rustfs_common::metrics::global_metrics; +use rustfs_config::{ + DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, + ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed, +}; use tokio::time::Duration; const MIN_SLEEP: Duration = Duration::from_millis(1); @@ -64,6 +68,10 @@ fn scanner_env_config() -> (ScannerSpeed, bool) { (speed, idle_mode) } +pub(crate) fn scanner_yield_every_n_objects() -> u64 { + rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS) +} + /// When `true` (default), the scanner throttles itself between operations. /// When `false`, all sleeps are skipped and the scanner runs at full speed. pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE); @@ -74,7 +82,9 @@ pub static SCANNER_SLEEPER: LazyLock = LazyLock::new(|| { let (speed, idle_mode) = scanner_env_config(); SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); - DynamicSleeper::new(speed) + let sleeper = DynamicSleeper::new(speed); + sleeper.record_throttle_config(); + sleeper }); /// Proportional-backoff sleeper for the data scanner. @@ -124,6 +134,7 @@ impl DynamicSleeper { let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep); if !sleep_dur.is_zero() { tokio::time::sleep(sleep_dur).await; + global_metrics().record_scanner_yield(sleep_dur); } } @@ -150,6 +161,17 @@ impl DynamicSleeper { let (speed, idle_mode) = scanner_env_config(); self.update(speed); SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); + self.record_throttle_config(); + } + + fn record_throttle_config(&self) { + let (factor, max_sleep) = self.read_params(); + global_metrics().record_scanner_throttle_config( + SCANNER_IDLE_MODE.load(Ordering::Relaxed), + factor, + max_sleep, + scanner_yield_every_n_objects(), + ); } } @@ -177,6 +199,7 @@ impl SleepTimer { .min(max_sleep); if !sleep_dur.is_zero() { tokio::time::sleep(sleep_dur).await; + global_metrics().record_scanner_yield(sleep_dur); } } }