feat(scanner): expand scanner observability metrics (#3159)

* feat(scanner): expand scanner observability metrics

* chore(scanner): align bucket-drive metric wording

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-02 09:29:53 +08:00
committed by GitHub
parent 39a283fbce
commit 1d46047d6f
9 changed files with 996 additions and 56 deletions
+357 -7
View File
@@ -146,6 +146,8 @@ pub enum Metric {
ScanCycle, // Full cycle, cluster global. ScanCycle, // Full cycle, cluster global.
ScanBucketDrive, // Single bucket on one drive. ScanBucketDrive, // Single bucket on one drive.
CompactFolder, // Folder compacted. CompactFolder, // Folder compacted.
ScanBucketDriveStart,
ScanBucketDriveFailure,
// Must be last: // Must be last:
Last, Last,
@@ -179,6 +181,8 @@ impl Metric {
Self::ScanCycle => "scan_cycle", Self::ScanCycle => "scan_cycle",
Self::ScanBucketDrive => "scan_bucket_drive", Self::ScanBucketDrive => "scan_bucket_drive",
Self::CompactFolder => "compact_folder", Self::CompactFolder => "compact_folder",
Self::ScanBucketDriveStart => "scan_bucket_drive_start",
Self::ScanBucketDriveFailure => "scan_bucket_drive_failure",
Self::Last => "last", Self::Last => "last",
} }
} }
@@ -213,7 +217,9 @@ impl Metric {
21 => Some(Self::ScanCycle), 21 => Some(Self::ScanCycle),
22 => Some(Self::ScanBucketDrive), 22 => Some(Self::ScanBucketDrive),
23 => Some(Self::CompactFolder), 23 => Some(Self::CompactFolder),
24 => Some(Self::Last), 24 => Some(Self::ScanBucketDriveStart),
25 => Some(Self::ScanBucketDriveFailure),
26 => Some(Self::Last),
_ => None, _ => None,
} }
} }
@@ -346,12 +352,35 @@ pub struct Metrics {
current_scan_cycle_objects_start: AtomicU64, current_scan_cycle_objects_start: AtomicU64,
current_scan_cycle_directories_start: AtomicU64, current_scan_cycle_directories_start: AtomicU64,
current_scan_cycle_bucket_drive_scans_start: AtomicU64, current_scan_cycle_bucket_drive_scans_start: AtomicU64,
current_scan_cycle_bucket_drive_failures_start: AtomicU64,
current_scan_cycle_yield_events_start: AtomicU64,
current_scan_cycle_yield_duration_millis_start: AtomicU64,
current_scan_cycle_ilm_actions_start: AtomicU64,
current_scan_cycle_heal_objects_start: AtomicU64,
current_scan_cycle_replication_checks_start: AtomicU64,
current_scan_cycle_usage_saves_start: AtomicU64,
last_scan_cycle_result: AtomicU8, last_scan_cycle_result: AtomicU8,
last_scan_cycle_duration_millis: AtomicU64, last_scan_cycle_duration_millis: AtomicU64,
last_scan_cycle_objects_scanned: AtomicU64, last_scan_cycle_objects_scanned: AtomicU64,
last_scan_cycle_directories_scanned: AtomicU64, last_scan_cycle_directories_scanned: AtomicU64,
last_scan_cycle_bucket_drive_scans: AtomicU64, last_scan_cycle_bucket_drive_scans: AtomicU64,
last_scan_cycle_bucket_drive_failures: AtomicU64,
last_scan_cycle_yield_events: AtomicU64,
last_scan_cycle_yield_duration_millis: AtomicU64,
last_scan_cycle_ilm_actions: AtomicU64,
last_scan_cycle_heal_objects: AtomicU64,
last_scan_cycle_replication_checks: AtomicU64,
last_scan_cycle_usage_saves: AtomicU64,
failed_scan_cycles: AtomicU64, failed_scan_cycles: AtomicU64,
scanner_yield_duration_millis: AtomicU64,
scanner_ilm_actions: AtomicU64,
scanner_throttle_idle_mode_enabled: AtomicBool,
scanner_throttle_sleep_factor_micros: AtomicU64,
scanner_throttle_max_sleep_millis: AtomicU64,
scanner_yield_every_n_objects: AtomicU64,
scanner_cycle_interval_millis: AtomicU64,
scanner_bitrot_cycle_enabled: AtomicBool,
scanner_bitrot_cycle_millis: AtomicU64,
} }
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0; const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
@@ -374,6 +403,13 @@ pub struct ScanCycleWorkSnapshot {
objects_scanned: u64, objects_scanned: u64,
directories_scanned: u64, directories_scanned: u64,
bucket_drive_scans: u64, bucket_drive_scans: u64,
bucket_drive_failures: u64,
yield_events: u64,
yield_duration_millis: u64,
ilm_actions: u64,
heal_objects: u64,
replication_checks: u64,
usage_saves: u64,
} }
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
@@ -409,6 +445,20 @@ pub struct ScannerMetricsReport {
pub current_cycle_directories_scanned: u64, pub current_cycle_directories_scanned: u64,
#[serde(default)] #[serde(default)]
pub current_cycle_bucket_drive_scans: u64, pub current_cycle_bucket_drive_scans: u64,
#[serde(default)]
pub current_cycle_bucket_drive_failures: u64,
#[serde(default)]
pub current_cycle_yield_events: u64,
#[serde(default)]
pub current_cycle_yield_duration_seconds: f64,
#[serde(default)]
pub current_cycle_ilm_actions: u64,
#[serde(default)]
pub current_cycle_heal_objects: u64,
#[serde(default)]
pub current_cycle_replication_checks: u64,
#[serde(default)]
pub current_cycle_usage_saves: u64,
pub last_cycle_result: String, pub last_cycle_result: String,
pub last_cycle_result_code: u64, pub last_cycle_result_code: u64,
pub last_cycle_duration_seconds: f64, pub last_cycle_duration_seconds: f64,
@@ -418,7 +468,35 @@ pub struct ScannerMetricsReport {
pub last_cycle_directories_scanned: u64, pub last_cycle_directories_scanned: u64,
#[serde(default)] #[serde(default)]
pub last_cycle_bucket_drive_scans: u64, pub last_cycle_bucket_drive_scans: u64,
#[serde(default)]
pub last_cycle_bucket_drive_failures: u64,
#[serde(default)]
pub last_cycle_yield_events: u64,
#[serde(default)]
pub last_cycle_yield_duration_seconds: f64,
#[serde(default)]
pub last_cycle_ilm_actions: u64,
#[serde(default)]
pub last_cycle_heal_objects: u64,
#[serde(default)]
pub last_cycle_replication_checks: u64,
#[serde(default)]
pub last_cycle_usage_saves: u64,
pub failed_cycles: u64, pub failed_cycles: u64,
#[serde(default)]
pub throttle_idle_mode_enabled: bool,
#[serde(default)]
pub throttle_sleep_factor: f64,
#[serde(default)]
pub throttle_max_sleep_seconds: f64,
#[serde(default)]
pub yield_every_n_objects: u64,
#[serde(default)]
pub cycle_interval_seconds: f64,
#[serde(default)]
pub bitrot_cycle_enabled: bool,
#[serde(default)]
pub bitrot_cycle_seconds: f64,
} }
impl CurrentCycle { impl CurrentCycle {
@@ -464,6 +542,14 @@ fn duration_millis_saturated(duration: Duration) -> u64 {
duration.as_millis().min(u64::MAX as u128) as u64 duration.as_millis().min(u64::MAX as u128) as u64
} }
fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 {
if !value.is_finite() || value <= 0.0 {
return 0;
}
(value * scale).round().clamp(0.0, u64::MAX as f64) as u64
}
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) { pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
let result = if success { let result = if success {
SCAN_CYCLE_RESULT_SUCCESS_LABEL SCAN_CYCLE_RESULT_SUCCESS_LABEL
@@ -514,12 +600,35 @@ impl Metrics {
current_scan_cycle_objects_start: AtomicU64::new(0), current_scan_cycle_objects_start: AtomicU64::new(0),
current_scan_cycle_directories_start: AtomicU64::new(0), current_scan_cycle_directories_start: AtomicU64::new(0),
current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0), current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0),
current_scan_cycle_yield_events_start: AtomicU64::new(0),
current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0),
current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
current_scan_cycle_heal_objects_start: AtomicU64::new(0),
current_scan_cycle_replication_checks_start: AtomicU64::new(0),
current_scan_cycle_usage_saves_start: AtomicU64::new(0),
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN), last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
last_scan_cycle_duration_millis: AtomicU64::new(0), last_scan_cycle_duration_millis: AtomicU64::new(0),
last_scan_cycle_objects_scanned: AtomicU64::new(0), last_scan_cycle_objects_scanned: AtomicU64::new(0),
last_scan_cycle_directories_scanned: AtomicU64::new(0), last_scan_cycle_directories_scanned: AtomicU64::new(0),
last_scan_cycle_bucket_drive_scans: AtomicU64::new(0), last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
last_scan_cycle_bucket_drive_failures: AtomicU64::new(0),
last_scan_cycle_yield_events: AtomicU64::new(0),
last_scan_cycle_yield_duration_millis: AtomicU64::new(0),
last_scan_cycle_ilm_actions: AtomicU64::new(0),
last_scan_cycle_heal_objects: AtomicU64::new(0),
last_scan_cycle_replication_checks: AtomicU64::new(0),
last_scan_cycle_usage_saves: AtomicU64::new(0),
failed_scan_cycles: AtomicU64::new(0), failed_scan_cycles: AtomicU64::new(0),
scanner_yield_duration_millis: AtomicU64::new(0),
scanner_ilm_actions: AtomicU64::new(0),
scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
scanner_throttle_max_sleep_millis: AtomicU64::new(0),
scanner_yield_every_n_objects: AtomicU64::new(0),
scanner_cycle_interval_millis: AtomicU64::new(0),
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
scanner_bitrot_cycle_millis: AtomicU64::new(0),
} }
} }
@@ -621,6 +730,58 @@ impl Metrics {
} }
} }
pub fn record_scanner_yield(&self, duration: Duration) {
let metric_idx = Metric::Yield as usize;
self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
if metric_idx < Metric::LastRealtime as usize {
self.latency[metric_idx].add(duration);
}
let duration_millis = duration_millis_saturated(duration);
let _ = self
.scanner_yield_duration_millis
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_add(duration_millis))
});
}
pub fn record_scanner_ilm_action(&self, count: u64) {
self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed);
}
pub fn record_scan_bucket_drive_start(&self) {
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
}
pub fn record_scan_bucket_drive_failure(&self) {
self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed);
}
pub fn record_scanner_throttle_config(
&self,
idle_mode_enabled: bool,
sleep_factor: f64,
max_sleep: Duration,
yield_every_n_objects: u64,
) {
self.scanner_throttle_idle_mode_enabled
.store(idle_mode_enabled, Ordering::Relaxed);
self.scanner_throttle_sleep_factor_micros
.store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed);
self.scanner_throttle_max_sleep_millis
.store(duration_millis_saturated(max_sleep), Ordering::Relaxed);
self.scanner_yield_every_n_objects
.store(yield_every_n_objects, Ordering::Relaxed);
}
pub fn record_scanner_cycle_config(&self, cycle_interval: Duration, bitrot_cycle: Option<Duration>) {
self.scanner_cycle_interval_millis
.store(duration_millis_saturated(cycle_interval), Ordering::Relaxed);
self.scanner_bitrot_cycle_enabled
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
self.scanner_bitrot_cycle_millis
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
}
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// Read-side helpers // Read-side helpers
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
@@ -686,13 +847,27 @@ impl Metrics {
.store(snapshot.directories_scanned, Ordering::Relaxed); .store(snapshot.directories_scanned, Ordering::Relaxed);
self.current_scan_cycle_bucket_drive_scans_start self.current_scan_cycle_bucket_drive_scans_start
.store(snapshot.bucket_drive_scans, Ordering::Relaxed); .store(snapshot.bucket_drive_scans, Ordering::Relaxed);
self.current_scan_cycle_bucket_drive_failures_start
.store(snapshot.bucket_drive_failures, Ordering::Relaxed);
self.current_scan_cycle_yield_events_start
.store(snapshot.yield_events, Ordering::Relaxed);
self.current_scan_cycle_yield_duration_millis_start
.store(snapshot.yield_duration_millis, Ordering::Relaxed);
self.current_scan_cycle_ilm_actions_start
.store(snapshot.ilm_actions, Ordering::Relaxed);
self.current_scan_cycle_heal_objects_start
.store(snapshot.heal_objects, Ordering::Relaxed);
self.current_scan_cycle_replication_checks_start
.store(snapshot.replication_checks, Ordering::Relaxed);
self.current_scan_cycle_usage_saves_start
.store(snapshot.usage_saves, Ordering::Relaxed);
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed); self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
snapshot snapshot
} }
pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) { pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
let work = self.scan_cycle_work_since(start); let work = self.scan_cycle_work_since(start);
self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans); self.record_scan_cycle_work(work);
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed); self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
} }
@@ -701,6 +876,13 @@ impl Metrics {
objects_scanned: self.lifetime(Metric::ScanObject), objects_scanned: self.lifetime(Metric::ScanObject),
directories_scanned: self.lifetime(Metric::ScanFolder), directories_scanned: self.lifetime(Metric::ScanFolder),
bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive), bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure),
yield_events: self.lifetime(Metric::Yield),
yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed),
ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
heal_objects: self.lifetime(Metric::HealAbandonedObject),
replication_checks: self.lifetime(Metric::CheckReplication),
usage_saves: self.lifetime(Metric::SaveUsage),
} }
} }
@@ -709,6 +891,13 @@ impl Metrics {
objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed), objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed), directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed), bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed),
yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed),
yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed),
ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
} }
} }
@@ -718,17 +907,35 @@ impl Metrics {
objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned), objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned), directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans), bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures),
yield_events: current.yield_events.saturating_sub(start.yield_events),
yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis),
ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
} }
} }
pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) { pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
// Telemetry-only gauges: readers may observe a transient mixed snapshot // Telemetry-only gauges: readers may observe a transient mixed snapshot
// while these independent atomic fields are updated. // while these independent atomic fields are updated.
self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed); self.last_scan_cycle_objects_scanned
.store(work.objects_scanned, Ordering::Relaxed);
self.last_scan_cycle_directories_scanned self.last_scan_cycle_directories_scanned
.store(directories_scanned, Ordering::Relaxed); .store(work.directories_scanned, Ordering::Relaxed);
self.last_scan_cycle_bucket_drive_scans self.last_scan_cycle_bucket_drive_scans
.store(bucket_drive_scans, Ordering::Relaxed); .store(work.bucket_drive_scans, Ordering::Relaxed);
self.last_scan_cycle_bucket_drive_failures
.store(work.bucket_drive_failures, Ordering::Relaxed);
self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed);
self.last_scan_cycle_yield_duration_millis
.store(work.yield_duration_millis, Ordering::Relaxed);
self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
self.last_scan_cycle_replication_checks
.store(work.replication_checks, Ordering::Relaxed);
self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed);
} }
/// Snapshot of every path currently being scanned. /// Snapshot of every path currently being scanned.
@@ -772,6 +979,13 @@ impl Metrics {
m.current_cycle_objects_scanned = current_work.objects_scanned; m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned; m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans; m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
m.current_cycle_yield_events = current_work.yield_events;
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
} }
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed); let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string(); m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
@@ -780,7 +994,21 @@ impl Metrics {
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed); m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed); m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed); m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed);
m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed);
m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed); m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed);
m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
// Lifetime operation counts // Lifetime operation counts
for i in 0..Metric::Last as usize { for i in 0..Metric::Last as usize {
@@ -998,13 +1226,43 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn report_includes_last_scan_cycle_work() { async fn report_includes_last_scan_cycle_work() {
let metrics = Metrics::new(); let metrics = Metrics::new();
metrics.record_scan_cycle_work(11, 7, 3); metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
objects_scanned: 11,
directories_scanned: 7,
bucket_drive_scans: 3,
bucket_drive_failures: 2,
yield_events: 5,
yield_duration_millis: 250,
ilm_actions: 13,
heal_objects: 2,
replication_checks: 4,
usage_saves: 6,
});
let report = metrics.report().await; let report = metrics.report().await;
assert_eq!(report.last_cycle_objects_scanned, 11); assert_eq!(report.last_cycle_objects_scanned, 11);
assert_eq!(report.last_cycle_directories_scanned, 7); assert_eq!(report.last_cycle_directories_scanned, 7);
assert_eq!(report.last_cycle_bucket_drive_scans, 3); assert_eq!(report.last_cycle_bucket_drive_scans, 3);
assert_eq!(report.last_cycle_bucket_drive_failures, 2);
assert_eq!(report.last_cycle_yield_events, 5);
assert_eq!(report.last_cycle_yield_duration_seconds, 0.25);
assert_eq!(report.last_cycle_ilm_actions, 13);
assert_eq!(report.last_cycle_heal_objects, 2);
assert_eq!(report.last_cycle_replication_checks, 4);
assert_eq!(report.last_cycle_usage_saves, 6);
}
#[tokio::test]
async fn report_includes_bucket_drive_scan_starts() {
let metrics = Metrics::new();
metrics.record_scan_bucket_drive_start();
metrics.record_scan_bucket_drive_failure();
let report = metrics.report().await;
assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1));
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
} }
#[tokio::test] #[tokio::test]
@@ -1013,17 +1271,39 @@ mod tests {
metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed); metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed); metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed); metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed);
metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed);
metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed);
metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed);
metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed);
metrics.actions[IlmAction::DeleteAction as usize].store(100, Ordering::Relaxed);
metrics.record_scanner_ilm_action(6);
metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed);
let start = metrics.start_scan_cycle_work(); let start = metrics.start_scan_cycle_work();
metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed); metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed); metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed); metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed);
metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed);
metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed);
metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed);
metrics.actions[IlmAction::DeleteAction as usize].store(150, Ordering::Relaxed);
metrics.record_scanner_ilm_action(9);
metrics.record_scanner_yield(Duration::from_millis(150));
let report = metrics.report().await; let report = metrics.report().await;
assert_eq!(report.current_cycle_objects_scanned, 7); assert_eq!(report.current_cycle_objects_scanned, 7);
assert_eq!(report.current_cycle_directories_scanned, 3); assert_eq!(report.current_cycle_directories_scanned, 3);
assert_eq!(report.current_cycle_bucket_drive_scans, 2); assert_eq!(report.current_cycle_bucket_drive_scans, 2);
assert_eq!(report.current_cycle_bucket_drive_failures, 3);
assert_eq!(report.current_cycle_yield_events, 1);
assert_eq!(report.current_cycle_yield_duration_seconds, 0.15);
assert_eq!(report.current_cycle_ilm_actions, 9);
assert_eq!(report.current_cycle_heal_objects, 3);
assert_eq!(report.current_cycle_replication_checks, 6);
assert_eq!(report.current_cycle_usage_saves, 2);
metrics.finish_scan_cycle_work(start); metrics.finish_scan_cycle_work(start);
let report = metrics.report().await; let report = metrics.report().await;
@@ -1031,8 +1311,78 @@ mod tests {
assert_eq!(report.current_cycle_objects_scanned, 0); assert_eq!(report.current_cycle_objects_scanned, 0);
assert_eq!(report.current_cycle_directories_scanned, 0); assert_eq!(report.current_cycle_directories_scanned, 0);
assert_eq!(report.current_cycle_bucket_drive_scans, 0); assert_eq!(report.current_cycle_bucket_drive_scans, 0);
assert_eq!(report.current_cycle_bucket_drive_failures, 0);
assert_eq!(report.current_cycle_yield_events, 0);
assert_eq!(report.current_cycle_yield_duration_seconds, 0.0);
assert_eq!(report.current_cycle_ilm_actions, 0);
assert_eq!(report.current_cycle_heal_objects, 0);
assert_eq!(report.current_cycle_replication_checks, 0);
assert_eq!(report.current_cycle_usage_saves, 0);
assert_eq!(report.last_cycle_objects_scanned, 7); assert_eq!(report.last_cycle_objects_scanned, 7);
assert_eq!(report.last_cycle_directories_scanned, 3); assert_eq!(report.last_cycle_directories_scanned, 3);
assert_eq!(report.last_cycle_bucket_drive_scans, 2); assert_eq!(report.last_cycle_bucket_drive_scans, 2);
assert_eq!(report.last_cycle_bucket_drive_failures, 3);
assert_eq!(report.last_cycle_yield_events, 1);
assert_eq!(report.last_cycle_yield_duration_seconds, 0.15);
assert_eq!(report.last_cycle_ilm_actions, 9);
assert_eq!(report.last_cycle_heal_objects, 3);
assert_eq!(report.last_cycle_replication_checks, 6);
assert_eq!(report.last_cycle_usage_saves, 2);
}
#[test]
fn record_scanner_yield_tracks_count_and_duration() {
let metrics = Metrics::new();
metrics.record_scanner_yield(Duration::from_millis(42));
assert_eq!(metrics.lifetime(Metric::Yield), 1);
assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42);
assert_eq!(metrics.last_minute(Metric::Yield).n, 1);
}
#[tokio::test]
async fn report_includes_scanner_throttle_config() {
let metrics = Metrics::new();
metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128);
let report = metrics.report().await;
assert!(report.throttle_idle_mode_enabled);
assert_eq!(report.throttle_sleep_factor, 2.5);
assert_eq!(report.throttle_max_sleep_seconds, 1.5);
assert_eq!(report.yield_every_n_objects, 128);
}
#[tokio::test]
async fn scanner_throttle_config_rejects_invalid_sleep_factor() {
let metrics = Metrics::new();
metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0);
let report = metrics.report().await;
assert!(!report.throttle_idle_mode_enabled);
assert_eq!(report.throttle_sleep_factor, 0.0);
assert_eq!(report.throttle_max_sleep_seconds, 0.0);
assert_eq!(report.yield_every_n_objects, 0);
}
#[tokio::test]
async fn report_includes_scanner_cycle_config() {
let metrics = Metrics::new();
metrics.record_scanner_cycle_config(Duration::from_secs(3600), Some(Duration::from_secs(86400)));
let report = metrics.report().await;
assert_eq!(report.cycle_interval_seconds, 3600.0);
assert!(report.bitrot_cycle_enabled);
assert_eq!(report.bitrot_cycle_seconds, 86400.0);
metrics.record_scanner_cycle_config(Duration::from_secs(60), None);
let report = metrics.report().await;
assert_eq!(report.cycle_interval_seconds, 60.0);
assert!(!report.bitrot_cycle_enabled);
assert_eq!(report.bitrot_cycle_seconds, 0.0);
} }
} }
+232 -13
View File
@@ -16,29 +16,40 @@
//! Scanner metrics collector. //! Scanner metrics collector.
//! //!
//! Collects background scanner metrics including bucket scans, //! Collects background scanner metrics including bucket-drive scans,
//! directory scans, and object scans. //! directory scans, and object scans.
use crate::metrics::report::PrometheusMetric; use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::scanner::{ use crate::metrics::schema::scanner::{
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD, SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD,
SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD,
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD,
SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD,
SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD,
SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
}; };
/// Scanner statistics. /// Scanner statistics.
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct ScannerStats { pub struct ScannerStats {
/// Number of bucket scans finished /// Number of bucket-drive scans finished
pub bucket_scans_finished: u64, pub bucket_scans_finished: u64,
/// Number of bucket scans started /// Number of bucket-drive scans started
pub bucket_scans_started: u64, pub bucket_scans_started: u64,
/// Number of bucket-drive scans that failed
pub bucket_scans_failed: u64,
/// Number of directories scanned /// Number of directories scanned
pub directories_scanned: u64, pub directories_scanned: u64,
/// Number of objects scanned /// Number of objects scanned
@@ -49,6 +60,20 @@ pub struct ScannerStats {
pub last_activity_seconds: u64, pub last_activity_seconds: u64,
/// Number of scanner paths currently being processed /// Number of scanner paths currently being processed
pub active_paths: u64, pub active_paths: u64,
/// Whether scanner idle-mode self-throttling is enabled
pub throttle_idle_mode_enabled: u64,
/// Effective scanner sleep factor
pub throttle_sleep_factor: f64,
/// Effective scanner maximum self-throttle sleep duration in seconds
pub throttle_max_sleep_seconds: f64,
/// Object interval for cooperative scanner runtime yields
pub yield_every_n_objects: u64,
/// Effective scanner cycle interval in seconds
pub cycle_interval_seconds: f64,
/// Whether periodic scanner bitrot deep scans are enabled
pub bitrot_cycle_enabled: u64,
/// Effective scanner bitrot deep-scan interval in seconds
pub bitrot_cycle_seconds: f64,
/// Current scanner cycle number, or zero when idle /// Current scanner cycle number, or zero when idle
pub current_cycle: u64, pub current_cycle: u64,
/// Number of scanner cycles completed since server start /// Number of scanner cycles completed since server start
@@ -61,12 +86,26 @@ pub struct ScannerStats {
pub current_cycle_directories_scanned: u64, pub current_cycle_directories_scanned: u64,
/// Number of bucket-drive scans finished by the currently running scanner cycle /// Number of bucket-drive scans finished by the currently running scanner cycle
pub current_cycle_bucket_drive_scans: u64, pub current_cycle_bucket_drive_scans: u64,
/// Number of bucket-drive scans that failed in the currently running scanner cycle
pub current_cycle_bucket_drive_failures: u64,
/// Object scan rate for the currently running scanner cycle /// Object scan rate for the currently running scanner cycle
pub current_cycle_objects_per_second: f64, pub current_cycle_objects_per_second: f64,
/// Directory scan rate for the currently running scanner cycle /// Directory scan rate for the currently running scanner cycle
pub current_cycle_directories_per_second: f64, pub current_cycle_directories_per_second: f64,
/// Bucket-drive scan rate for the currently running scanner cycle /// Bucket-drive scan rate for the currently running scanner cycle
pub current_cycle_bucket_drive_scans_per_second: f64, pub current_cycle_bucket_drive_scans_per_second: f64,
/// Number of scanner self-throttle yield events in the current scanner cycle
pub current_cycle_yield_events: u64,
/// Total scanner self-throttle yield duration in seconds for the current scanner cycle
pub current_cycle_yield_duration_seconds: f64,
/// Number of lifecycle actions applied by the current scanner cycle
pub current_cycle_ilm_actions: u64,
/// Number of object heal candidates enqueued by the current scanner cycle
pub current_cycle_heal_objects: u64,
/// Number of replication heal checks run by the current scanner cycle
pub current_cycle_replication_checks: u64,
/// Number of data-usage save operations run by the current scanner cycle
pub current_cycle_usage_saves: u64,
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan /// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
pub current_scan_mode: u64, pub current_scan_mode: u64,
/// Last scanner cycle result: 0 unknown, 1 success, 2 error /// Last scanner cycle result: 0 unknown, 1 success, 2 error
@@ -79,12 +118,26 @@ pub struct ScannerStats {
pub last_cycle_directories_scanned: u64, pub last_cycle_directories_scanned: u64,
/// Number of bucket-drive scans finished by the last scanner cycle /// Number of bucket-drive scans finished by the last scanner cycle
pub last_cycle_bucket_drive_scans: u64, pub last_cycle_bucket_drive_scans: u64,
/// Number of bucket-drive scans that failed in the last finished scanner cycle
pub last_cycle_bucket_drive_failures: u64,
/// Object scan rate for the last finished scanner cycle /// Object scan rate for the last finished scanner cycle
pub last_cycle_objects_per_second: f64, pub last_cycle_objects_per_second: f64,
/// Directory scan rate for the last finished scanner cycle /// Directory scan rate for the last finished scanner cycle
pub last_cycle_directories_per_second: f64, pub last_cycle_directories_per_second: f64,
/// Bucket-drive scan rate for the last finished scanner cycle /// Bucket-drive scan rate for the last finished scanner cycle
pub last_cycle_bucket_drive_scans_per_second: f64, pub last_cycle_bucket_drive_scans_per_second: f64,
/// Number of scanner self-throttle yield events in the last finished scanner cycle
pub last_cycle_yield_events: u64,
/// Total scanner self-throttle yield duration in seconds for the last finished scanner cycle
pub last_cycle_yield_duration_seconds: f64,
/// Number of lifecycle actions applied by the last finished scanner cycle
pub last_cycle_ilm_actions: u64,
/// Number of object heal candidates enqueued by the last finished scanner cycle
pub last_cycle_heal_objects: u64,
/// Number of replication heal checks run by the last finished scanner cycle
pub last_cycle_replication_checks: u64,
/// Number of data-usage save operations run by the last finished scanner cycle
pub last_cycle_usage_saves: u64,
/// Number of scanner cycles that failed since server start /// Number of scanner cycles that failed since server start
pub failed_cycles: u64, pub failed_cycles: u64,
} }
@@ -97,11 +150,19 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
vec![ vec![
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64), PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64), PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64), PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64), PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, stats.throttle_idle_mode_enabled as f64),
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_SLEEP_FACTOR_MD, stats.throttle_sleep_factor),
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, stats.throttle_max_sleep_seconds),
PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64),
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds),
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, stats.bitrot_cycle_enabled as f64),
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
@@ -114,6 +175,10 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
stats.current_cycle_bucket_drive_scans as f64, stats.current_cycle_bucket_drive_scans as f64,
), ),
PrometheusMetric::from_descriptor(
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
stats.current_cycle_bucket_drive_failures as f64,
),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
PrometheusMetric::from_descriptor( PrometheusMetric::from_descriptor(
&SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, &SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
@@ -123,6 +188,18 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, &SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
stats.current_cycle_bucket_drive_scans_per_second, stats.current_cycle_bucket_drive_scans_per_second,
), ),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, stats.current_cycle_yield_events as f64),
PrometheusMetric::from_descriptor(
&SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD,
stats.current_cycle_yield_duration_seconds,
),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, stats.current_cycle_ilm_actions as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, stats.current_cycle_heal_objects as f64),
PrometheusMetric::from_descriptor(
&SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD,
stats.current_cycle_replication_checks as f64,
),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64),
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64), PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
@@ -132,12 +209,22 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
stats.last_cycle_directories_scanned as f64, stats.last_cycle_directories_scanned as f64,
), ),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
PrometheusMetric::from_descriptor(
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD,
stats.last_cycle_bucket_drive_failures as f64,
),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second), PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
PrometheusMetric::from_descriptor( PrometheusMetric::from_descriptor(
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, &SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
stats.last_cycle_bucket_drive_scans_per_second, stats.last_cycle_bucket_drive_scans_per_second,
), ),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, stats.last_cycle_yield_events as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, stats.last_cycle_yield_duration_seconds),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, stats.last_cycle_ilm_actions as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, stats.last_cycle_heal_objects as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64), PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
] ]
} }
@@ -152,36 +239,58 @@ mod tests {
let stats = ScannerStats { let stats = ScannerStats {
bucket_scans_finished: 100, bucket_scans_finished: 100,
bucket_scans_started: 100, bucket_scans_started: 100,
bucket_scans_failed: 2,
directories_scanned: 50000, directories_scanned: 50000,
objects_scanned: 1000000, objects_scanned: 1000000,
versions_scanned: 1500000, versions_scanned: 1500000,
last_activity_seconds: 30, last_activity_seconds: 30,
active_paths: 4, active_paths: 4,
throttle_idle_mode_enabled: 1,
throttle_sleep_factor: 10.0,
throttle_max_sleep_seconds: 15.0,
yield_every_n_objects: 128,
cycle_interval_seconds: 3600.0,
bitrot_cycle_enabled: 1,
bitrot_cycle_seconds: 86400.0,
current_cycle: 12, current_cycle: 12,
completed_cycles: 11, completed_cycles: 11,
current_cycle_age_seconds: 90, current_cycle_age_seconds: 90,
current_cycle_objects_scanned: 250, current_cycle_objects_scanned: 250,
current_cycle_directories_scanned: 20, current_cycle_directories_scanned: 20,
current_cycle_bucket_drive_scans: 2, current_cycle_bucket_drive_scans: 2,
current_cycle_bucket_drive_failures: 1,
current_cycle_objects_per_second: 12.5, current_cycle_objects_per_second: 12.5,
current_cycle_directories_per_second: 1.0, current_cycle_directories_per_second: 1.0,
current_cycle_bucket_drive_scans_per_second: 0.1, current_cycle_bucket_drive_scans_per_second: 0.1,
current_cycle_yield_events: 8,
current_cycle_yield_duration_seconds: 1.25,
current_cycle_ilm_actions: 6,
current_cycle_heal_objects: 2,
current_cycle_replication_checks: 5,
current_cycle_usage_saves: 3,
current_scan_mode: 2, current_scan_mode: 2,
last_cycle_result: 1, last_cycle_result: 1,
last_cycle_duration_seconds: 42.5, last_cycle_duration_seconds: 42.5,
last_cycle_objects_scanned: 900, last_cycle_objects_scanned: 900,
last_cycle_directories_scanned: 80, last_cycle_directories_scanned: 80,
last_cycle_bucket_drive_scans: 6, last_cycle_bucket_drive_scans: 6,
last_cycle_bucket_drive_failures: 2,
last_cycle_objects_per_second: 18.0, last_cycle_objects_per_second: 18.0,
last_cycle_directories_per_second: 1.6, last_cycle_directories_per_second: 1.6,
last_cycle_bucket_drive_scans_per_second: 0.12, last_cycle_bucket_drive_scans_per_second: 0.12,
last_cycle_yield_events: 30,
last_cycle_yield_duration_seconds: 9.5,
last_cycle_ilm_actions: 44,
last_cycle_heal_objects: 7,
last_cycle_replication_checks: 12,
last_cycle_usage_saves: 9,
failed_cycles: 3, failed_cycles: 3,
}; };
let metrics = collect_scanner_metrics(&stats); let metrics = collect_scanner_metrics(&stats);
report_metrics(&metrics); report_metrics(&metrics);
assert_eq!(metrics.len(), 26); assert_eq!(metrics.len(), 48);
let objects = metrics.iter().find(|m| m.value == 1000000.0); let objects = metrics.iter().find(|m| m.value == 1000000.0);
assert!(objects.is_some()); assert!(objects.is_some());
@@ -194,6 +303,46 @@ mod tests {
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
assert_eq!(active_paths.map(|m| m.value), Some(4.0)); assert_eq!(active_paths.map(|m| m.value), Some(4.0));
let bucket_scans_failed = metrics
.iter()
.find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name());
assert_eq!(bucket_scans_failed.map(|m| m.value), Some(2.0));
let throttle_idle_mode_enabled = metrics
.iter()
.find(|m| m.name == SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD.get_full_metric_name());
assert_eq!(throttle_idle_mode_enabled.map(|m| m.value), Some(1.0));
let throttle_sleep_factor = metrics
.iter()
.find(|m| m.name == SCANNER_THROTTLE_SLEEP_FACTOR_MD.get_full_metric_name());
assert_eq!(throttle_sleep_factor.map(|m| m.value), Some(10.0));
let throttle_max_sleep = metrics
.iter()
.find(|m| m.name == SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD.get_full_metric_name());
assert_eq!(throttle_max_sleep.map(|m| m.value), Some(15.0));
let yield_every_n_objects = metrics
.iter()
.find(|m| m.name == SCANNER_YIELD_EVERY_N_OBJECTS_MD.get_full_metric_name());
assert_eq!(yield_every_n_objects.map(|m| m.value), Some(128.0));
let cycle_interval_seconds = metrics
.iter()
.find(|m| m.name == SCANNER_CYCLE_INTERVAL_SECONDS_MD.get_full_metric_name());
assert_eq!(cycle_interval_seconds.map(|m| m.value), Some(3600.0));
let bitrot_cycle_enabled = metrics
.iter()
.find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name());
assert_eq!(bitrot_cycle_enabled.map(|m| m.value), Some(1.0));
let bitrot_cycle_seconds = metrics
.iter()
.find(|m| m.name == SCANNER_BITROT_CYCLE_SECONDS_MD.get_full_metric_name());
assert_eq!(bitrot_cycle_seconds.map(|m| m.value), Some(86400.0));
let current_cycle = metrics let current_cycle = metrics
.iter() .iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
@@ -224,6 +373,11 @@ mod tests {
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0)); assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
let current_cycle_bucket_drive_failures = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
assert_eq!(current_cycle_bucket_drive_failures.map(|m| m.value), Some(1.0));
let current_cycle_objects_rate = metrics let current_cycle_objects_rate = metrics
.iter() .iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
@@ -239,6 +393,36 @@ mod tests {
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1)); assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
let current_cycle_yield_events = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
assert_eq!(current_cycle_yield_events.map(|m| m.value), Some(8.0));
let current_cycle_yield_duration = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
assert_eq!(current_cycle_yield_duration.map(|m| m.value), Some(1.25));
let current_cycle_ilm_actions = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
assert_eq!(current_cycle_ilm_actions.map(|m| m.value), Some(6.0));
let current_cycle_heal_objects = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
assert_eq!(current_cycle_heal_objects.map(|m| m.value), Some(2.0));
let current_cycle_replication_checks = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
assert_eq!(current_cycle_replication_checks.map(|m| m.value), Some(5.0));
let current_cycle_usage_saves = metrics
.iter()
.find(|m| m.name == SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
assert_eq!(current_cycle_usage_saves.map(|m| m.value), Some(3.0));
let current_scan_mode = metrics let current_scan_mode = metrics
.iter() .iter()
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
@@ -269,6 +453,11 @@ mod tests {
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0)); assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
let last_cycle_bucket_drive_failures = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
assert_eq!(last_cycle_bucket_drive_failures.map(|m| m.value), Some(2.0));
let last_cycle_objects_rate = metrics let last_cycle_objects_rate = metrics
.iter() .iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
@@ -284,6 +473,36 @@ mod tests {
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12)); assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
let last_cycle_yield_events = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
assert_eq!(last_cycle_yield_events.map(|m| m.value), Some(30.0));
let last_cycle_yield_duration = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
assert_eq!(last_cycle_yield_duration.map(|m| m.value), Some(9.5));
let last_cycle_ilm_actions = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
assert_eq!(last_cycle_ilm_actions.map(|m| m.value), Some(44.0));
let last_cycle_heal_objects = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
assert_eq!(last_cycle_heal_objects.map(|m| m.value), Some(7.0));
let last_cycle_replication_checks = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
assert_eq!(last_cycle_replication_checks.map(|m| m.value), Some(12.0));
let last_cycle_usage_saves = metrics
.iter()
.find(|m| m.name == SCANNER_LAST_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
assert_eq!(last_cycle_usage_saves.map(|m| m.value), Some(9.0));
let failed_cycles = metrics let failed_cycles = metrics
.iter() .iter()
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name()); .find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
@@ -295,7 +514,7 @@ mod tests {
let stats = ScannerStats::default(); let stats = ScannerStats::default();
let metrics = collect_scanner_metrics(&stats); let metrics = collect_scanner_metrics(&stats);
assert_eq!(metrics.len(), 26); assert_eq!(metrics.len(), 48);
for metric in &metrics { for metric in &metrics {
assert_eq!(metric.value, 0.0); assert_eq!(metric.value, 0.0);
assert!(metric.labels.is_empty()); assert!(metric.labels.is_empty());
@@ -279,24 +279,46 @@ pub enum MetricName {
ScannerVersionsScanned, ScannerVersionsScanned,
ScannerLastActivitySeconds, ScannerLastActivitySeconds,
ScannerActivePaths, ScannerActivePaths,
ScannerBucketScansFailed,
ScannerThrottleIdleModeEnabled,
ScannerThrottleSleepFactor,
ScannerThrottleMaxSleepSeconds,
ScannerYieldEveryNObjects,
ScannerCycleIntervalSeconds,
ScannerBitrotCycleEnabled,
ScannerBitrotCycleSeconds,
ScannerCurrentCycle, ScannerCurrentCycle,
ScannerCompletedCycles, ScannerCompletedCycles,
ScannerCurrentCycleAgeSeconds, ScannerCurrentCycleAgeSeconds,
ScannerCurrentCycleObjectsScanned, ScannerCurrentCycleObjectsScanned,
ScannerCurrentCycleDirectoriesScanned, ScannerCurrentCycleDirectoriesScanned,
ScannerCurrentCycleBucketDriveScans, ScannerCurrentCycleBucketDriveScans,
ScannerCurrentCycleBucketDriveFailures,
ScannerCurrentCycleObjectsPerSecond, ScannerCurrentCycleObjectsPerSecond,
ScannerCurrentCycleDirectoriesPerSecond, ScannerCurrentCycleDirectoriesPerSecond,
ScannerCurrentCycleBucketDriveScansPerSecond, ScannerCurrentCycleBucketDriveScansPerSecond,
ScannerCurrentCycleYieldEvents,
ScannerCurrentCycleYieldDurationSeconds,
ScannerCurrentCycleIlmActions,
ScannerCurrentCycleHealObjects,
ScannerCurrentCycleReplicationChecks,
ScannerCurrentCycleUsageSaves,
ScannerCurrentScanMode, ScannerCurrentScanMode,
ScannerLastCycleResult, ScannerLastCycleResult,
ScannerLastCycleDurationSeconds, ScannerLastCycleDurationSeconds,
ScannerLastCycleObjectsScanned, ScannerLastCycleObjectsScanned,
ScannerLastCycleDirectoriesScanned, ScannerLastCycleDirectoriesScanned,
ScannerLastCycleBucketDriveScans, ScannerLastCycleBucketDriveScans,
ScannerLastCycleBucketDriveFailures,
ScannerLastCycleObjectsPerSecond, ScannerLastCycleObjectsPerSecond,
ScannerLastCycleDirectoriesPerSecond, ScannerLastCycleDirectoriesPerSecond,
ScannerLastCycleBucketDriveScansPerSecond, ScannerLastCycleBucketDriveScansPerSecond,
ScannerLastCycleYieldEvents,
ScannerLastCycleYieldDurationSeconds,
ScannerLastCycleIlmActions,
ScannerLastCycleHealObjects,
ScannerLastCycleReplicationChecks,
ScannerLastCycleUsageSaves,
ScannerFailedCycles, ScannerFailedCycles,
// CPU system-related metrics // CPU system-related metrics
@@ -639,24 +661,46 @@ impl MetricName {
Self::ScannerVersionsScanned => "versions_scanned".to_string(), Self::ScannerVersionsScanned => "versions_scanned".to_string(),
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(), Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
Self::ScannerActivePaths => "active_paths".to_string(), Self::ScannerActivePaths => "active_paths".to_string(),
Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(),
Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(),
Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(),
Self::ScannerThrottleMaxSleepSeconds => "throttle_max_sleep_seconds".to_string(),
Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(),
Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(),
Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(),
Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(),
Self::ScannerCurrentCycle => "current_cycle".to_string(), Self::ScannerCurrentCycle => "current_cycle".to_string(),
Self::ScannerCompletedCycles => "completed_cycles".to_string(), Self::ScannerCompletedCycles => "completed_cycles".to_string(),
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(), Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(), Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(), Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(), Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
Self::ScannerCurrentCycleBucketDriveFailures => "current_cycle_bucket_drive_failures".to_string(),
Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(), Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(), Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(), Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
Self::ScannerCurrentCycleYieldEvents => "current_cycle_yield_events".to_string(),
Self::ScannerCurrentCycleYieldDurationSeconds => "current_cycle_yield_duration_seconds".to_string(),
Self::ScannerCurrentCycleIlmActions => "current_cycle_ilm_actions".to_string(),
Self::ScannerCurrentCycleHealObjects => "current_cycle_heal_objects".to_string(),
Self::ScannerCurrentCycleReplicationChecks => "current_cycle_replication_checks".to_string(),
Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(),
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(), Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
Self::ScannerLastCycleResult => "last_cycle_result".to_string(), Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(), Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(), Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(), Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(), Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
Self::ScannerLastCycleBucketDriveFailures => "last_cycle_bucket_drive_failures".to_string(),
Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(), Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(), Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(), Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
Self::ScannerLastCycleYieldEvents => "last_cycle_yield_events".to_string(),
Self::ScannerLastCycleYieldDurationSeconds => "last_cycle_yield_duration_seconds".to_string(),
Self::ScannerLastCycleIlmActions => "last_cycle_ilm_actions".to_string(),
Self::ScannerLastCycleHealObjects => "last_cycle_heal_objects".to_string(),
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
Self::ScannerFailedCycles => "failed_cycles".to_string(), Self::ScannerFailedCycles => "failed_cycles".to_string(),
// CPU system-related metrics // CPU system-related metrics
+200 -2
View File
@@ -20,7 +20,7 @@ use std::sync::LazyLock;
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md( new_counter_md(
MetricName::ScannerBucketScansFinished, MetricName::ScannerBucketScansFinished,
"Total number of bucket scans finished since server start", "Total number of bucket-drive scans finished since server start",
&[], &[],
subsystems::SCANNER, subsystems::SCANNER,
) )
@@ -29,7 +29,16 @@ pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLo
pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md( new_counter_md(
MetricName::ScannerBucketScansStarted, MetricName::ScannerBucketScansStarted,
"Total number of bucket scans started since server start", "Total number of bucket-drive scans started since server start",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_BUCKET_SCANS_FAILED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ScannerBucketScansFailed,
"Total number of bucket-drive scans that failed since server start",
&[], &[],
subsystems::SCANNER, subsystems::SCANNER,
) )
@@ -80,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
) )
}); });
pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerThrottleIdleModeEnabled,
"Whether scanner idle-mode self-throttling is enabled: 1 enabled, 0 disabled.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_THROTTLE_SLEEP_FACTOR_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerThrottleSleepFactor,
"Effective scanner sleep factor used to compute proportional self-throttle sleeps.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerThrottleMaxSleepSeconds,
"Effective maximum scanner self-throttle sleep duration in seconds.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_YIELD_EVERY_N_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerYieldEveryNObjects,
"Current object interval for cooperative scanner runtime yields, or zero when disabled.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CYCLE_INTERVAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCycleIntervalSeconds,
"Effective scanner cycle interval in seconds.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerBitrotCycleEnabled,
"Whether periodic scanner bitrot deep scans are enabled: 1 enabled, 0 disabled.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_BITROT_CYCLE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerBitrotCycleSeconds,
"Effective scanner bitrot deep-scan interval in seconds; zero means disabled or deep-scan every cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md( new_gauge_md(
MetricName::ScannerCurrentCycle, MetricName::ScannerCurrentCycle,
@@ -134,6 +206,15 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescripto
) )
}); });
pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleBucketDriveFailures,
"Number of bucket-drive scans that failed in the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md( new_gauge_md(
MetricName::ScannerCurrentCycleObjectsPerSecond, MetricName::ScannerCurrentCycleObjectsPerSecond,
@@ -161,6 +242,60 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<Metr
) )
}); });
pub static SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleYieldEvents,
"Number of scanner self-throttle yield events in the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleYieldDurationSeconds,
"Total scanner self-throttle yield duration in seconds for the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleIlmActions,
"Number of lifecycle actions applied by the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleHealObjects,
"Number of object heal candidates enqueued by the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleReplicationChecks,
"Number of replication heal checks run by the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerCurrentCycleUsageSaves,
"Number of data-usage save operations run by the currently running scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md( new_gauge_md(
MetricName::ScannerCurrentScanMode, MetricName::ScannerCurrentScanMode,
@@ -215,6 +350,15 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor>
) )
}); });
pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleBucketDriveFailures,
"Number of bucket-drive scans that failed in the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md( new_gauge_md(
MetricName::ScannerLastCycleObjectsPerSecond, MetricName::ScannerLastCycleObjectsPerSecond,
@@ -242,6 +386,60 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricD
) )
}); });
pub static SCANNER_LAST_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleYieldEvents,
"Number of scanner self-throttle yield events in the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleYieldDurationSeconds,
"Total scanner self-throttle yield duration in seconds for the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleIlmActions,
"Number of lifecycle actions applied by the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleHealObjects,
"Number of object heal candidates enqueued by the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleReplicationChecks,
"Number of replication heal checks run by the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_LAST_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleUsageSaves,
"Number of data-usage save operations run by the last finished scanner cycle.",
&[],
subsystems::SCANNER,
)
});
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| { pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md( new_counter_md(
MetricName::ScannerFailedCycles, MetricName::ScannerFailedCycles,
+51 -4
View File
@@ -896,10 +896,23 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
/// ///
/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the /// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
/// rustfs-obs scanner collector shape. /// rustfs-obs scanner collector shape.
fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_scans_finished: u64) -> u64 {
life_time_ops
.get("scan_bucket_drive_start")
.copied()
.unwrap_or(bucket_scans_finished)
}
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> { pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let metrics = global_metrics().report().await; let metrics = global_metrics().report().await;
let now = Utc::now(); let now = Utc::now();
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default(); let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
let bucket_scans_failed = metrics
.life_time_ops
.get("scan_bucket_drive_failure")
.copied()
.unwrap_or_default();
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default(); let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default(); let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default(); let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
@@ -914,21 +927,27 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
Some(ScannerStats { Some(ScannerStats {
bucket_scans_finished, bucket_scans_finished,
// `global_metrics()` currently tracks completed bucket-drive scans, not a bucket_scans_started,
// separate started counter. Mirror the finished count until Task 5/Task 10 bucket_scans_failed,
// expands the scanner runtime source shape.
bucket_scans_started: bucket_scans_finished,
directories_scanned, directories_scanned,
objects_scanned, objects_scanned,
versions_scanned, versions_scanned,
last_activity_seconds, last_activity_seconds,
active_paths, active_paths,
throttle_idle_mode_enabled: u64::from(metrics.throttle_idle_mode_enabled),
throttle_sleep_factor: metrics.throttle_sleep_factor,
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
yield_every_n_objects: metrics.yield_every_n_objects,
cycle_interval_seconds: metrics.cycle_interval_seconds,
bitrot_cycle_enabled: u64::from(metrics.bitrot_cycle_enabled),
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
current_cycle: metrics.current_cycle, current_cycle: metrics.current_cycle,
completed_cycles, completed_cycles,
current_cycle_age_seconds, current_cycle_age_seconds,
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned, current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned, current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans, current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age), current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
current_cycle_directories_per_second: scanner_work_rate_per_second( current_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.current_cycle_directories_scanned, metrics.current_cycle_directories_scanned,
@@ -938,12 +957,19 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
metrics.current_cycle_bucket_drive_scans, metrics.current_cycle_bucket_drive_scans,
current_cycle_age, current_cycle_age,
), ),
current_cycle_yield_events: metrics.current_cycle_yield_events,
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
current_scan_mode, current_scan_mode,
last_cycle_result: metrics.last_cycle_result_code, last_cycle_result: metrics.last_cycle_result_code,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds, last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned, last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned, last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans, last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration), last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
last_cycle_directories_per_second: scanner_work_rate_per_second( last_cycle_directories_per_second: scanner_work_rate_per_second(
metrics.last_cycle_directories_scanned, metrics.last_cycle_directories_scanned,
@@ -953,6 +979,12 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
metrics.last_cycle_bucket_drive_scans, metrics.last_cycle_bucket_drive_scans,
last_cycle_duration, last_cycle_duration,
), ),
last_cycle_yield_events: metrics.last_cycle_yield_events,
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
failed_cycles: metrics.failed_cycles, failed_cycles: metrics.failed_cycles,
}) })
} }
@@ -1056,6 +1088,21 @@ mod tests {
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64); assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
} }
#[test]
fn scanner_bucket_scans_started_uses_explicit_started_count() {
let mut life_time_ops = HashMap::new();
life_time_ops.insert("scan_bucket_drive_start".to_string(), 7);
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 7);
}
#[test]
fn scanner_bucket_scans_started_falls_back_to_finished_count() {
let life_time_ops = HashMap::new();
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
}
#[test] #[test]
fn scanner_work_rate_per_second_reports_rate() { fn scanner_work_rate_per_second_reports_rate() {
assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0); assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
+40 -15
View File
@@ -266,8 +266,13 @@ fn bitrot_scan_cycle() -> Option<Duration> {
} }
} }
fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode { fn get_cycle_scan_mode(
let Some(bitrot_cycle) = bitrot_scan_cycle() else { current_cycle: u64,
bitrot_start_cycle: u64,
bitrot_start_time: Option<DateTime<Utc>>,
bitrot_cycle: Option<Duration>,
) -> HealScanMode {
let Some(bitrot_cycle) = bitrot_cycle else {
return HealScanMode::Normal; return HealScanMode::Normal;
}; };
@@ -299,8 +304,10 @@ fn background_heal_info_for_scan_start(
current_cycle: u64, current_cycle: u64,
scan_mode: HealScanMode, scan_mode: HealScanMode,
now: DateTime<Utc>, now: DateTime<Utc>,
bitrot_cycle: Option<Duration>,
) -> Option<BackgroundHealInfo> { ) -> Option<BackgroundHealInfo> {
let reset_bitrot_start = scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now); let reset_bitrot_start =
scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, bitrot_cycle);
if info.current_scan_mode == scan_mode && !reset_bitrot_start { if info.current_scan_mode == scan_mode && !reset_bitrot_start {
return None; return None;
} }
@@ -314,12 +321,17 @@ fn background_heal_info_for_scan_start(
Some(info) Some(info)
} }
fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now: DateTime<Utc>) -> bool { fn should_reset_bitrot_start(
info: &BackgroundHealInfo,
current_cycle: u64,
now: DateTime<Utc>,
bitrot_cycle: Option<Duration>,
) -> bool {
let Some(bitrot_start_time) = info.bitrot_start_time else { let Some(bitrot_start_time) = info.bitrot_start_time else {
return true; return true;
}; };
let Some(bitrot_cycle) = bitrot_scan_cycle() else { let Some(bitrot_cycle) = bitrot_cycle else {
return false; return false;
}; };
@@ -424,6 +436,9 @@ fn get_lock_acquire_timeout() -> Duration {
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) { async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
let _activity_guard = ScannerActivityGuard::new(); let _activity_guard = ScannerActivityGuard::new();
SCANNER_SLEEPER.refresh_from_env(); SCANNER_SLEEPER.refresh_from_env();
let configured_cycle_interval = cycle_interval();
let configured_bitrot_cycle = bitrot_scan_cycle();
global_metrics().record_scanner_cycle_config(configured_cycle_interval, configured_bitrot_cycle);
info!("Start run data scanner cycle"); info!("Start run data scanner cycle");
cycle_info.current = cycle_info.next; cycle_info.current = cycle_info.next;
let now = Instant::now(); let now = Instant::now();
@@ -437,11 +452,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
cycle_info.current, cycle_info.current,
background_heal_info.bitrot_start_cycle, background_heal_info.bitrot_start_cycle,
background_heal_info.bitrot_start_time, background_heal_info.bitrot_start_time,
configured_bitrot_cycle,
); );
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode); let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
if let Some(new_heal_info) = if let Some(new_heal_info) = background_heal_info_for_scan_start(
background_heal_info_for_scan_start(background_heal_info.clone(), cycle_info.current, scan_mode, Utc::now()) background_heal_info.clone(),
{ cycle_info.current,
scan_mode,
Utc::now(),
configured_bitrot_cycle,
) {
background_heal_info = new_heal_info.clone(); background_heal_info = new_heal_info.clone();
save_background_heal_info(storeapi.clone(), new_heal_info).await; save_background_heal_info(storeapi.clone(), new_heal_info).await;
} }
@@ -600,18 +620,22 @@ pub async fn store_data_usage_in_backend(
// Save a backup every 10th update // Save a backup every 10th update
if attempts > 10 { if attempts > 10 {
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await { if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
warn!("Failed to save data usage backup to {}: {}", backup_path, e); warn!("Failed to save data usage backup to {}: {}", backup_path, e);
} }
done_save();
attempts = 1; attempts = 1;
} }
// Save main configuration // Save main configuration
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await { if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str()); error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
} else { } else {
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await; rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
} }
done_save();
attempts += 1; attempts += 1;
} }
@@ -866,7 +890,7 @@ mod tests {
#[serial] #[serial]
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() { fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now())); let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
assert_eq!(mode, HealScanMode::Deep); assert_eq!(mode, HealScanMode::Deep);
}); });
} }
@@ -878,8 +902,8 @@ mod tests {
let recent = Utc::now() - chrono::Duration::minutes(30); let recent = Utc::now() - chrono::Duration::minutes(30);
let old = Utc::now() - chrono::Duration::hours(2); let old = Utc::now() - chrono::Duration::hours(2);
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal); assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal);
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep); assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep);
}); });
} }
@@ -887,7 +911,7 @@ mod tests {
#[serial] #[serial]
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() { fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || { with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal); assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
}); });
} }
@@ -895,8 +919,9 @@ mod tests {
#[serial] #[serial]
fn test_background_heal_info_for_scan_start_marks_deep_active() { fn test_background_heal_info_for_scan_start_marks_deep_active() {
let now = Utc::now(); let now = Utc::now();
let info = background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now) let info =
.expect("deep scan should update background heal info"); background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle())
.expect("deep scan should update background heal info");
assert_eq!(info.current_scan_mode, HealScanMode::Deep); assert_eq!(info.current_scan_mode, HealScanMode::Deep);
assert_eq!(info.bitrot_start_cycle, 7); assert_eq!(info.bitrot_start_cycle, 7);
@@ -914,7 +939,7 @@ mod tests {
current_scan_mode: HealScanMode::Normal, current_scan_mode: HealScanMode::Normal,
}; };
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now()) let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle())
.expect("deep scan should mark active status"); .expect("deep scan should mark active status");
assert_eq!(info.current_scan_mode, HealScanMode::Deep); assert_eq!(info.current_scan_mode, HealScanMode::Deep);
+11 -10
View File
@@ -16,19 +16,19 @@ use std::collections::HashSet;
use std::fs::FileType; use std::fs::FileType;
use std::io::ErrorKind; use std::io::ErrorKind;
use std::sync::{Arc, Once}; use std::sync::{Arc, Once};
use std::time::{Duration, SystemTime}; use std::time::{Duration, Instant, SystemTime};
use crate::ReplTargetSizeSummary; use crate::ReplTargetSizeSummary;
use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path}; use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path};
use crate::error::ScannerError; use crate::error::ScannerError;
use crate::scanner_io::ScannerIODisk as _; use crate::scanner_io::ScannerIODisk as _;
use crate::sleeper::DynamicSleeper; use crate::sleeper::{DynamicSleeper, scanner_yield_every_n_objects};
use metrics::{counter, describe_counter}; use metrics::{counter, describe_counter};
use rustfs_common::heal_channel::{ use rustfs_common::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode, HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode,
send_heal_request_with_admission, send_heal_request_with_admission,
}; };
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater}; use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater, global_metrics};
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule}; use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator; use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
@@ -143,13 +143,6 @@ fn scanner_excess_folders_threshold() -> u64 {
) )
} }
fn scanner_yield_every_n_objects() -> u64 {
rustfs_utils::get_env_u64(
rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS,
rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS,
)
}
fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool { fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
yield_every > 0 && object_count.is_multiple_of(yield_every) yield_every > 0 && object_count.is_multiple_of(yield_every)
} }
@@ -485,6 +478,7 @@ impl ScannerItem {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)(); done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
break 'eventLoop; break 'eventLoop;
} }
@@ -497,6 +491,7 @@ impl ScannerItem {
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)(); done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
} }
IlmAction::DeleteVersionAction => { IlmAction::DeleteVersionAction => {
remaining_versions -= 1; remaining_versions -= 1;
@@ -510,11 +505,13 @@ impl ScannerItem {
} }
noncurrent_events.push(event.clone()); noncurrent_events.push(event.clone());
done_ilm(1)(); done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
} }
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => { IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action); debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)(); done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
} }
IlmAction::NoneAction | IlmAction::ActionCount => { IlmAction::NoneAction | IlmAction::ActionCount => {
@@ -560,7 +557,9 @@ impl ScannerItem {
return; return;
}; };
let done_replication = Metrics::time(Metric::CheckReplication);
let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await; let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
done_replication();
if !Self::should_account_replication_stats(oi) { if !Self::should_account_replication_stats(oi) {
return; return;
} }
@@ -1110,7 +1109,9 @@ impl FolderScanner {
timer.sleep().await; timer.sleep().await;
if should_yield_after_object(object_count, yield_every_objects) { if should_yield_after_object(object_count, yield_every_objects) {
let yield_start = Instant::now();
tokio::task::yield_now().await; tokio::task::yield_now().await;
global_metrics().record_scanner_yield(yield_start.elapsed());
} }
} }
+36 -3
View File
@@ -22,7 +22,7 @@ use futures::future::join_all;
use metrics::counter; use metrics::counter;
use rand::seq::SliceRandom as _; use rand::seq::SliceRandom as _;
use rustfs_common::heal_channel::HealScanMode; use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete}; use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, global_metrics};
use rustfs_config::{ use rustfs_config::{
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS,
ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS,
@@ -116,6 +116,28 @@ impl Drop for DiskBucketScanActiveGuard {
} }
} }
struct BucketDriveFailureGuard {
failed: bool,
}
impl BucketDriveFailureGuard {
fn new() -> Self {
Self { failed: true }
}
fn mark_success(&mut self) {
self.failed = false;
}
}
impl Drop for BucketDriveFailureGuard {
fn drop(&mut self) {
if self.failed {
global_metrics().record_scan_bucket_drive_failure();
}
}
}
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize { fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
counter counter
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1))) .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
@@ -217,9 +239,11 @@ async fn persist_and_publish_cache_snapshot<S: StorageAPI>(
) -> Option<SystemTime> { ) -> Option<SystemTime> {
let last_update = cache_snapshot.info.last_update; let last_update = cache_snapshot.info.last_update;
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await { if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
error!("Failed to save data usage cache: {}", e); error!("Failed to save data usage cache: {}", e);
} }
done_save();
if let Err(e) = updates.send(cache_snapshot).await { if let Err(e) = updates.send(cache_snapshot).await {
error!("Failed to send data usage cache: {}", e); error!("Failed to send data usage cache: {}", e);
@@ -732,9 +756,12 @@ impl ScannerIOCache for SetDisks {
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before) if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
&& last_update > before_update && last_update > before_update
&& let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await
{ {
error!("Failed to save data usage cache: {}", e); let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
error!("Failed to save data usage cache: {}", e);
}
done_save();
} }
if let Err(e) = update_fut.await { if let Err(e) = update_fut.await {
@@ -775,9 +802,11 @@ impl ScannerIOCache for SetDisks {
error!("nsscanner_disk: Failed to send data usage entry info: {}", e); error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
} }
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await { if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
error!("nsscanner_disk: Failed to save data usage cache: {}", e); error!("nsscanner_disk: Failed to save data usage cache: {}", e);
} }
done_save();
} }
})); }));
} }
@@ -900,6 +929,8 @@ impl ScannerIODisk for Disk {
let drive_start = std::time::Instant::now(); let drive_start = std::time::Instant::now();
let bucket = cache.info.name.clone(); let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string(); let disk_path = self.path().to_string_lossy().to_string();
global_metrics().record_scan_bucket_drive_start();
let mut failure_guard = BucketDriveFailureGuard::new();
let _guard = self.start_scan(); let _guard = self.start_scan();
let mut cache = cache; let mut cache = cache;
@@ -966,9 +997,11 @@ impl ScannerIODisk for Disk {
done_drive(); done_drive();
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed()); emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
data_usage_info.info.last_update = Some(SystemTime::now()); data_usage_info.info.last_update = Some(SystemTime::now());
failure_guard.mark_success();
Ok(data_usage_info) Ok(data_usage_info)
} }
Err(e) => { Err(e) => {
done_drive();
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed()); emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
Err(StorageError::other(format!("Failed to scan data folder: {e}"))) Err(StorageError::other(format!("Failed to scan data folder: {e}")))
} }
+25 -2
View File
@@ -16,7 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::{Arc, LazyLock, RwLock}; use std::sync::{Arc, LazyLock, RwLock};
use std::time::Instant; use std::time::Instant;
use rustfs_config::{DEFAULT_SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, ScannerSpeed}; use rustfs_common::metrics::global_metrics;
use rustfs_config::{
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
};
use tokio::time::Duration; use tokio::time::Duration;
const MIN_SLEEP: Duration = Duration::from_millis(1); const MIN_SLEEP: Duration = Duration::from_millis(1);
@@ -64,6 +68,10 @@ fn scanner_env_config() -> (ScannerSpeed, bool) {
(speed, idle_mode) (speed, idle_mode)
} }
pub(crate) fn scanner_yield_every_n_objects() -> u64 {
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
}
/// When `true` (default), the scanner throttles itself between operations. /// When `true` (default), the scanner throttles itself between operations.
/// When `false`, all sleeps are skipped and the scanner runs at full speed. /// When `false`, all sleeps are skipped and the scanner runs at full speed.
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE); pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
@@ -74,7 +82,9 @@ pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
let (speed, idle_mode) = scanner_env_config(); let (speed, idle_mode) = scanner_env_config();
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
DynamicSleeper::new(speed) let sleeper = DynamicSleeper::new(speed);
sleeper.record_throttle_config();
sleeper
}); });
/// Proportional-backoff sleeper for the data scanner. /// Proportional-backoff sleeper for the data scanner.
@@ -124,6 +134,7 @@ impl DynamicSleeper {
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep); let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep);
if !sleep_dur.is_zero() { if !sleep_dur.is_zero() {
tokio::time::sleep(sleep_dur).await; tokio::time::sleep(sleep_dur).await;
global_metrics().record_scanner_yield(sleep_dur);
} }
} }
@@ -150,6 +161,17 @@ impl DynamicSleeper {
let (speed, idle_mode) = scanner_env_config(); let (speed, idle_mode) = scanner_env_config();
self.update(speed); self.update(speed);
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed); SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
self.record_throttle_config();
}
fn record_throttle_config(&self) {
let (factor, max_sleep) = self.read_params();
global_metrics().record_scanner_throttle_config(
SCANNER_IDLE_MODE.load(Ordering::Relaxed),
factor,
max_sleep,
scanner_yield_every_n_objects(),
);
} }
} }
@@ -177,6 +199,7 @@ impl SleepTimer {
.min(max_sleep); .min(max_sleep);
if !sleep_dur.is_zero() { if !sleep_dur.is_zero() {
tokio::time::sleep(sleep_dur).await; tokio::time::sleep(sleep_dur).await;
global_metrics().record_scanner_yield(sleep_dur);
} }
} }
} }