mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat(scanner): expand scanner observability metrics (#3159)
* feat(scanner): expand scanner observability metrics * chore(scanner): align bucket-drive metric wording --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
@@ -146,6 +146,8 @@ pub enum Metric {
|
||||
ScanCycle, // Full cycle, cluster global.
|
||||
ScanBucketDrive, // Single bucket on one drive.
|
||||
CompactFolder, // Folder compacted.
|
||||
ScanBucketDriveStart,
|
||||
ScanBucketDriveFailure,
|
||||
|
||||
// Must be last:
|
||||
Last,
|
||||
@@ -179,6 +181,8 @@ impl Metric {
|
||||
Self::ScanCycle => "scan_cycle",
|
||||
Self::ScanBucketDrive => "scan_bucket_drive",
|
||||
Self::CompactFolder => "compact_folder",
|
||||
Self::ScanBucketDriveStart => "scan_bucket_drive_start",
|
||||
Self::ScanBucketDriveFailure => "scan_bucket_drive_failure",
|
||||
Self::Last => "last",
|
||||
}
|
||||
}
|
||||
@@ -213,7 +217,9 @@ impl Metric {
|
||||
21 => Some(Self::ScanCycle),
|
||||
22 => Some(Self::ScanBucketDrive),
|
||||
23 => Some(Self::CompactFolder),
|
||||
24 => Some(Self::Last),
|
||||
24 => Some(Self::ScanBucketDriveStart),
|
||||
25 => Some(Self::ScanBucketDriveFailure),
|
||||
26 => Some(Self::Last),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -346,12 +352,35 @@ pub struct Metrics {
|
||||
current_scan_cycle_objects_start: AtomicU64,
|
||||
current_scan_cycle_directories_start: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_scans_start: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_failures_start: AtomicU64,
|
||||
current_scan_cycle_yield_events_start: AtomicU64,
|
||||
current_scan_cycle_yield_duration_millis_start: AtomicU64,
|
||||
current_scan_cycle_ilm_actions_start: AtomicU64,
|
||||
current_scan_cycle_heal_objects_start: AtomicU64,
|
||||
current_scan_cycle_replication_checks_start: AtomicU64,
|
||||
current_scan_cycle_usage_saves_start: AtomicU64,
|
||||
last_scan_cycle_result: AtomicU8,
|
||||
last_scan_cycle_duration_millis: AtomicU64,
|
||||
last_scan_cycle_objects_scanned: AtomicU64,
|
||||
last_scan_cycle_directories_scanned: AtomicU64,
|
||||
last_scan_cycle_bucket_drive_scans: AtomicU64,
|
||||
last_scan_cycle_bucket_drive_failures: AtomicU64,
|
||||
last_scan_cycle_yield_events: AtomicU64,
|
||||
last_scan_cycle_yield_duration_millis: AtomicU64,
|
||||
last_scan_cycle_ilm_actions: AtomicU64,
|
||||
last_scan_cycle_heal_objects: AtomicU64,
|
||||
last_scan_cycle_replication_checks: AtomicU64,
|
||||
last_scan_cycle_usage_saves: AtomicU64,
|
||||
failed_scan_cycles: AtomicU64,
|
||||
scanner_yield_duration_millis: AtomicU64,
|
||||
scanner_ilm_actions: AtomicU64,
|
||||
scanner_throttle_idle_mode_enabled: AtomicBool,
|
||||
scanner_throttle_sleep_factor_micros: AtomicU64,
|
||||
scanner_throttle_max_sleep_millis: AtomicU64,
|
||||
scanner_yield_every_n_objects: AtomicU64,
|
||||
scanner_cycle_interval_millis: AtomicU64,
|
||||
scanner_bitrot_cycle_enabled: AtomicBool,
|
||||
scanner_bitrot_cycle_millis: AtomicU64,
|
||||
}
|
||||
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
||||
@@ -374,6 +403,13 @@ pub struct ScanCycleWorkSnapshot {
|
||||
objects_scanned: u64,
|
||||
directories_scanned: u64,
|
||||
bucket_drive_scans: u64,
|
||||
bucket_drive_failures: u64,
|
||||
yield_events: u64,
|
||||
yield_duration_millis: u64,
|
||||
ilm_actions: u64,
|
||||
heal_objects: u64,
|
||||
replication_checks: u64,
|
||||
usage_saves: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -409,6 +445,20 @@ pub struct ScannerMetricsReport {
|
||||
pub current_cycle_directories_scanned: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_bucket_drive_scans: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_bucket_drive_failures: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_yield_events: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_yield_duration_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_ilm_actions: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_heal_objects: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_replication_checks: u64,
|
||||
#[serde(default)]
|
||||
pub current_cycle_usage_saves: u64,
|
||||
pub last_cycle_result: String,
|
||||
pub last_cycle_result_code: u64,
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
@@ -418,7 +468,35 @@ pub struct ScannerMetricsReport {
|
||||
pub last_cycle_directories_scanned: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_scans: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_failures: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_yield_events: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_yield_duration_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_ilm_actions: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_heal_objects: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_replication_checks: u64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_usage_saves: u64,
|
||||
pub failed_cycles: u64,
|
||||
#[serde(default)]
|
||||
pub throttle_idle_mode_enabled: bool,
|
||||
#[serde(default)]
|
||||
pub throttle_sleep_factor: f64,
|
||||
#[serde(default)]
|
||||
pub throttle_max_sleep_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub yield_every_n_objects: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_interval_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_seconds: f64,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
@@ -464,6 +542,14 @@ fn duration_millis_saturated(duration: Duration) -> u64 {
|
||||
duration.as_millis().min(u64::MAX as u128) as u64
|
||||
}
|
||||
|
||||
fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 {
|
||||
if !value.is_finite() || value <= 0.0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
(value * scale).round().clamp(0.0, u64::MAX as f64) as u64
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
|
||||
let result = if success {
|
||||
SCAN_CYCLE_RESULT_SUCCESS_LABEL
|
||||
@@ -514,12 +600,35 @@ impl Metrics {
|
||||
current_scan_cycle_objects_start: AtomicU64::new(0),
|
||||
current_scan_cycle_directories_start: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0),
|
||||
current_scan_cycle_yield_events_start: AtomicU64::new(0),
|
||||
current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0),
|
||||
current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
|
||||
current_scan_cycle_heal_objects_start: AtomicU64::new(0),
|
||||
current_scan_cycle_replication_checks_start: AtomicU64::new(0),
|
||||
current_scan_cycle_usage_saves_start: AtomicU64::new(0),
|
||||
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
||||
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
||||
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
||||
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
||||
last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
|
||||
last_scan_cycle_bucket_drive_failures: AtomicU64::new(0),
|
||||
last_scan_cycle_yield_events: AtomicU64::new(0),
|
||||
last_scan_cycle_yield_duration_millis: AtomicU64::new(0),
|
||||
last_scan_cycle_ilm_actions: AtomicU64::new(0),
|
||||
last_scan_cycle_heal_objects: AtomicU64::new(0),
|
||||
last_scan_cycle_replication_checks: AtomicU64::new(0),
|
||||
last_scan_cycle_usage_saves: AtomicU64::new(0),
|
||||
failed_scan_cycles: AtomicU64::new(0),
|
||||
scanner_yield_duration_millis: AtomicU64::new(0),
|
||||
scanner_ilm_actions: AtomicU64::new(0),
|
||||
scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
|
||||
scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
|
||||
scanner_throttle_max_sleep_millis: AtomicU64::new(0),
|
||||
scanner_yield_every_n_objects: AtomicU64::new(0),
|
||||
scanner_cycle_interval_millis: AtomicU64::new(0),
|
||||
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
|
||||
scanner_bitrot_cycle_millis: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -621,6 +730,58 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scanner_yield(&self, duration: Duration) {
|
||||
let metric_idx = Metric::Yield as usize;
|
||||
self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
self.latency[metric_idx].add(duration);
|
||||
}
|
||||
let duration_millis = duration_millis_saturated(duration);
|
||||
let _ = self
|
||||
.scanner_yield_duration_millis
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_add(duration_millis))
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_scanner_ilm_action(&self, count: u64) {
|
||||
self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_start(&self) {
|
||||
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_failure(&self) {
|
||||
self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_throttle_config(
|
||||
&self,
|
||||
idle_mode_enabled: bool,
|
||||
sleep_factor: f64,
|
||||
max_sleep: Duration,
|
||||
yield_every_n_objects: u64,
|
||||
) {
|
||||
self.scanner_throttle_idle_mode_enabled
|
||||
.store(idle_mode_enabled, Ordering::Relaxed);
|
||||
self.scanner_throttle_sleep_factor_micros
|
||||
.store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed);
|
||||
self.scanner_throttle_max_sleep_millis
|
||||
.store(duration_millis_saturated(max_sleep), Ordering::Relaxed);
|
||||
self.scanner_yield_every_n_objects
|
||||
.store(yield_every_n_objects, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_cycle_config(&self, cycle_interval: Duration, bitrot_cycle: Option<Duration>) {
|
||||
self.scanner_cycle_interval_millis
|
||||
.store(duration_millis_saturated(cycle_interval), Ordering::Relaxed);
|
||||
self.scanner_bitrot_cycle_enabled
|
||||
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
|
||||
self.scanner_bitrot_cycle_millis
|
||||
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Read-side helpers
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -686,13 +847,27 @@ impl Metrics {
|
||||
.store(snapshot.directories_scanned, Ordering::Relaxed);
|
||||
self.current_scan_cycle_bucket_drive_scans_start
|
||||
.store(snapshot.bucket_drive_scans, Ordering::Relaxed);
|
||||
self.current_scan_cycle_bucket_drive_failures_start
|
||||
.store(snapshot.bucket_drive_failures, Ordering::Relaxed);
|
||||
self.current_scan_cycle_yield_events_start
|
||||
.store(snapshot.yield_events, Ordering::Relaxed);
|
||||
self.current_scan_cycle_yield_duration_millis_start
|
||||
.store(snapshot.yield_duration_millis, Ordering::Relaxed);
|
||||
self.current_scan_cycle_ilm_actions_start
|
||||
.store(snapshot.ilm_actions, Ordering::Relaxed);
|
||||
self.current_scan_cycle_heal_objects_start
|
||||
.store(snapshot.heal_objects, Ordering::Relaxed);
|
||||
self.current_scan_cycle_replication_checks_start
|
||||
.store(snapshot.replication_checks, Ordering::Relaxed);
|
||||
self.current_scan_cycle_usage_saves_start
|
||||
.store(snapshot.usage_saves, Ordering::Relaxed);
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
||||
snapshot
|
||||
}
|
||||
|
||||
pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
|
||||
let work = self.scan_cycle_work_since(start);
|
||||
self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans);
|
||||
self.record_scan_cycle_work(work);
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
@@ -701,6 +876,13 @@ impl Metrics {
|
||||
objects_scanned: self.lifetime(Metric::ScanObject),
|
||||
directories_scanned: self.lifetime(Metric::ScanFolder),
|
||||
bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
|
||||
bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure),
|
||||
yield_events: self.lifetime(Metric::Yield),
|
||||
yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed),
|
||||
ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
|
||||
heal_objects: self.lifetime(Metric::HealAbandonedObject),
|
||||
replication_checks: self.lifetime(Metric::CheckReplication),
|
||||
usage_saves: self.lifetime(Metric::SaveUsage),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -709,6 +891,13 @@ impl Metrics {
|
||||
objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
|
||||
directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
|
||||
bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
|
||||
bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed),
|
||||
yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed),
|
||||
yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed),
|
||||
ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
|
||||
heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
|
||||
replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
|
||||
usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -718,17 +907,35 @@ impl Metrics {
|
||||
objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
|
||||
directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
|
||||
bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
|
||||
bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures),
|
||||
yield_events: current.yield_events.saturating_sub(start.yield_events),
|
||||
yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis),
|
||||
ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
|
||||
heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
|
||||
replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
|
||||
usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) {
|
||||
pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
|
||||
// Telemetry-only gauges: readers may observe a transient mixed snapshot
|
||||
// while these independent atomic fields are updated.
|
||||
self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed);
|
||||
self.last_scan_cycle_objects_scanned
|
||||
.store(work.objects_scanned, Ordering::Relaxed);
|
||||
self.last_scan_cycle_directories_scanned
|
||||
.store(directories_scanned, Ordering::Relaxed);
|
||||
.store(work.directories_scanned, Ordering::Relaxed);
|
||||
self.last_scan_cycle_bucket_drive_scans
|
||||
.store(bucket_drive_scans, Ordering::Relaxed);
|
||||
.store(work.bucket_drive_scans, Ordering::Relaxed);
|
||||
self.last_scan_cycle_bucket_drive_failures
|
||||
.store(work.bucket_drive_failures, Ordering::Relaxed);
|
||||
self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed);
|
||||
self.last_scan_cycle_yield_duration_millis
|
||||
.store(work.yield_duration_millis, Ordering::Relaxed);
|
||||
self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
|
||||
self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
|
||||
self.last_scan_cycle_replication_checks
|
||||
.store(work.replication_checks, Ordering::Relaxed);
|
||||
self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Snapshot of every path currently being scanned.
|
||||
@@ -772,6 +979,13 @@ impl Metrics {
|
||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||
m.current_cycle_yield_events = current_work.yield_events;
|
||||
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||
}
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
@@ -780,7 +994,21 @@ impl Metrics {
|
||||
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
||||
m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed);
|
||||
m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed);
|
||||
m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
|
||||
m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
|
||||
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
|
||||
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
|
||||
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
|
||||
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
|
||||
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
|
||||
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed);
|
||||
m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
|
||||
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
|
||||
// Lifetime operation counts
|
||||
for i in 0..Metric::Last as usize {
|
||||
@@ -998,13 +1226,43 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_includes_last_scan_cycle_work() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_work(11, 7, 3);
|
||||
metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
|
||||
objects_scanned: 11,
|
||||
directories_scanned: 7,
|
||||
bucket_drive_scans: 3,
|
||||
bucket_drive_failures: 2,
|
||||
yield_events: 5,
|
||||
yield_duration_millis: 250,
|
||||
ilm_actions: 13,
|
||||
heal_objects: 2,
|
||||
replication_checks: 4,
|
||||
usage_saves: 6,
|
||||
});
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.last_cycle_objects_scanned, 11);
|
||||
assert_eq!(report.last_cycle_directories_scanned, 7);
|
||||
assert_eq!(report.last_cycle_bucket_drive_scans, 3);
|
||||
assert_eq!(report.last_cycle_bucket_drive_failures, 2);
|
||||
assert_eq!(report.last_cycle_yield_events, 5);
|
||||
assert_eq!(report.last_cycle_yield_duration_seconds, 0.25);
|
||||
assert_eq!(report.last_cycle_ilm_actions, 13);
|
||||
assert_eq!(report.last_cycle_heal_objects, 2);
|
||||
assert_eq!(report.last_cycle_replication_checks, 4);
|
||||
assert_eq!(report.last_cycle_usage_saves, 6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_bucket_drive_scan_starts() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_bucket_drive_start();
|
||||
metrics.record_scan_bucket_drive_failure();
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1));
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1013,17 +1271,39 @@ mod tests {
|
||||
metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed);
|
||||
metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed);
|
||||
metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed);
|
||||
metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed);
|
||||
metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed);
|
||||
metrics.actions[IlmAction::DeleteAction as usize].store(100, Ordering::Relaxed);
|
||||
metrics.record_scanner_ilm_action(6);
|
||||
metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed);
|
||||
|
||||
let start = metrics.start_scan_cycle_work();
|
||||
metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
|
||||
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed);
|
||||
metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed);
|
||||
metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed);
|
||||
metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed);
|
||||
metrics.actions[IlmAction::DeleteAction as usize].store(150, Ordering::Relaxed);
|
||||
metrics.record_scanner_ilm_action(9);
|
||||
metrics.record_scanner_yield(Duration::from_millis(150));
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.current_cycle_objects_scanned, 7);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 3);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
||||
assert_eq!(report.current_cycle_bucket_drive_failures, 3);
|
||||
assert_eq!(report.current_cycle_yield_events, 1);
|
||||
assert_eq!(report.current_cycle_yield_duration_seconds, 0.15);
|
||||
assert_eq!(report.current_cycle_ilm_actions, 9);
|
||||
assert_eq!(report.current_cycle_heal_objects, 3);
|
||||
assert_eq!(report.current_cycle_replication_checks, 6);
|
||||
assert_eq!(report.current_cycle_usage_saves, 2);
|
||||
|
||||
metrics.finish_scan_cycle_work(start);
|
||||
let report = metrics.report().await;
|
||||
@@ -1031,8 +1311,78 @@ mod tests {
|
||||
assert_eq!(report.current_cycle_objects_scanned, 0);
|
||||
assert_eq!(report.current_cycle_directories_scanned, 0);
|
||||
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
||||
assert_eq!(report.current_cycle_bucket_drive_failures, 0);
|
||||
assert_eq!(report.current_cycle_yield_events, 0);
|
||||
assert_eq!(report.current_cycle_yield_duration_seconds, 0.0);
|
||||
assert_eq!(report.current_cycle_ilm_actions, 0);
|
||||
assert_eq!(report.current_cycle_heal_objects, 0);
|
||||
assert_eq!(report.current_cycle_replication_checks, 0);
|
||||
assert_eq!(report.current_cycle_usage_saves, 0);
|
||||
assert_eq!(report.last_cycle_objects_scanned, 7);
|
||||
assert_eq!(report.last_cycle_directories_scanned, 3);
|
||||
assert_eq!(report.last_cycle_bucket_drive_scans, 2);
|
||||
assert_eq!(report.last_cycle_bucket_drive_failures, 3);
|
||||
assert_eq!(report.last_cycle_yield_events, 1);
|
||||
assert_eq!(report.last_cycle_yield_duration_seconds, 0.15);
|
||||
assert_eq!(report.last_cycle_ilm_actions, 9);
|
||||
assert_eq!(report.last_cycle_heal_objects, 3);
|
||||
assert_eq!(report.last_cycle_replication_checks, 6);
|
||||
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_scanner_yield_tracks_count_and_duration() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
metrics.record_scanner_yield(Duration::from_millis(42));
|
||||
|
||||
assert_eq!(metrics.lifetime(Metric::Yield), 1);
|
||||
assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42);
|
||||
assert_eq!(metrics.last_minute(Metric::Yield).n, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_scanner_throttle_config() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(report.throttle_idle_mode_enabled);
|
||||
assert_eq!(report.throttle_sleep_factor, 2.5);
|
||||
assert_eq!(report.throttle_max_sleep_seconds, 1.5);
|
||||
assert_eq!(report.yield_every_n_objects, 128);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_throttle_config_rejects_invalid_sleep_factor() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert!(!report.throttle_idle_mode_enabled);
|
||||
assert_eq!(report.throttle_sleep_factor, 0.0);
|
||||
assert_eq!(report.throttle_max_sleep_seconds, 0.0);
|
||||
assert_eq!(report.yield_every_n_objects, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_scanner_cycle_config() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
metrics.record_scanner_cycle_config(Duration::from_secs(3600), Some(Duration::from_secs(86400)));
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.cycle_interval_seconds, 3600.0);
|
||||
assert!(report.bitrot_cycle_enabled);
|
||||
assert_eq!(report.bitrot_cycle_seconds, 86400.0);
|
||||
|
||||
metrics.record_scanner_cycle_config(Duration::from_secs(60), None);
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.cycle_interval_seconds, 60.0);
|
||||
assert!(!report.bitrot_cycle_enabled);
|
||||
assert_eq!(report.bitrot_cycle_seconds, 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,29 +16,40 @@
|
||||
|
||||
//! Scanner metrics collector.
|
||||
//!
|
||||
//! Collects background scanner metrics including bucket scans,
|
||||
//! Collects background scanner metrics including bucket-drive scans,
|
||||
//! directory scans, and object scans.
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::scanner::{
|
||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
|
||||
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD,
|
||||
SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD,
|
||||
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
|
||||
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD,
|
||||
SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD,
|
||||
SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||
};
|
||||
|
||||
/// Scanner statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ScannerStats {
|
||||
/// Number of bucket scans finished
|
||||
/// Number of bucket-drive scans finished
|
||||
pub bucket_scans_finished: u64,
|
||||
/// Number of bucket scans started
|
||||
/// Number of bucket-drive scans started
|
||||
pub bucket_scans_started: u64,
|
||||
/// Number of bucket-drive scans that failed
|
||||
pub bucket_scans_failed: u64,
|
||||
/// Number of directories scanned
|
||||
pub directories_scanned: u64,
|
||||
/// Number of objects scanned
|
||||
@@ -49,6 +60,20 @@ pub struct ScannerStats {
|
||||
pub last_activity_seconds: u64,
|
||||
/// Number of scanner paths currently being processed
|
||||
pub active_paths: u64,
|
||||
/// Whether scanner idle-mode self-throttling is enabled
|
||||
pub throttle_idle_mode_enabled: u64,
|
||||
/// Effective scanner sleep factor
|
||||
pub throttle_sleep_factor: f64,
|
||||
/// Effective scanner maximum self-throttle sleep duration in seconds
|
||||
pub throttle_max_sleep_seconds: f64,
|
||||
/// Object interval for cooperative scanner runtime yields
|
||||
pub yield_every_n_objects: u64,
|
||||
/// Effective scanner cycle interval in seconds
|
||||
pub cycle_interval_seconds: f64,
|
||||
/// Whether periodic scanner bitrot deep scans are enabled
|
||||
pub bitrot_cycle_enabled: u64,
|
||||
/// Effective scanner bitrot deep-scan interval in seconds
|
||||
pub bitrot_cycle_seconds: f64,
|
||||
/// Current scanner cycle number, or zero when idle
|
||||
pub current_cycle: u64,
|
||||
/// Number of scanner cycles completed since server start
|
||||
@@ -61,12 +86,26 @@ pub struct ScannerStats {
|
||||
pub current_cycle_directories_scanned: u64,
|
||||
/// Number of bucket-drive scans finished by the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_scans: u64,
|
||||
/// Number of bucket-drive scans that failed in the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_failures: u64,
|
||||
/// Object scan rate for the currently running scanner cycle
|
||||
pub current_cycle_objects_per_second: f64,
|
||||
/// Directory scan rate for the currently running scanner cycle
|
||||
pub current_cycle_directories_per_second: f64,
|
||||
/// Bucket-drive scan rate for the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_scans_per_second: f64,
|
||||
/// Number of scanner self-throttle yield events in the current scanner cycle
|
||||
pub current_cycle_yield_events: u64,
|
||||
/// Total scanner self-throttle yield duration in seconds for the current scanner cycle
|
||||
pub current_cycle_yield_duration_seconds: f64,
|
||||
/// Number of lifecycle actions applied by the current scanner cycle
|
||||
pub current_cycle_ilm_actions: u64,
|
||||
/// Number of object heal candidates enqueued by the current scanner cycle
|
||||
pub current_cycle_heal_objects: u64,
|
||||
/// Number of replication heal checks run by the current scanner cycle
|
||||
pub current_cycle_replication_checks: u64,
|
||||
/// Number of data-usage save operations run by the current scanner cycle
|
||||
pub current_cycle_usage_saves: u64,
|
||||
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
|
||||
pub current_scan_mode: u64,
|
||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error
|
||||
@@ -79,12 +118,26 @@ pub struct ScannerStats {
|
||||
pub last_cycle_directories_scanned: u64,
|
||||
/// Number of bucket-drive scans finished by the last scanner cycle
|
||||
pub last_cycle_bucket_drive_scans: u64,
|
||||
/// Number of bucket-drive scans that failed in the last finished scanner cycle
|
||||
pub last_cycle_bucket_drive_failures: u64,
|
||||
/// Object scan rate for the last finished scanner cycle
|
||||
pub last_cycle_objects_per_second: f64,
|
||||
/// Directory scan rate for the last finished scanner cycle
|
||||
pub last_cycle_directories_per_second: f64,
|
||||
/// Bucket-drive scan rate for the last finished scanner cycle
|
||||
pub last_cycle_bucket_drive_scans_per_second: f64,
|
||||
/// Number of scanner self-throttle yield events in the last finished scanner cycle
|
||||
pub last_cycle_yield_events: u64,
|
||||
/// Total scanner self-throttle yield duration in seconds for the last finished scanner cycle
|
||||
pub last_cycle_yield_duration_seconds: f64,
|
||||
/// Number of lifecycle actions applied by the last finished scanner cycle
|
||||
pub last_cycle_ilm_actions: u64,
|
||||
/// Number of object heal candidates enqueued by the last finished scanner cycle
|
||||
pub last_cycle_heal_objects: u64,
|
||||
/// Number of replication heal checks run by the last finished scanner cycle
|
||||
pub last_cycle_replication_checks: u64,
|
||||
/// Number of data-usage save operations run by the last finished scanner cycle
|
||||
pub last_cycle_usage_saves: u64,
|
||||
/// Number of scanner cycles that failed since server start
|
||||
pub failed_cycles: u64,
|
||||
}
|
||||
@@ -97,11 +150,19 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
vec![
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, stats.throttle_idle_mode_enabled as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_SLEEP_FACTOR_MD, stats.throttle_sleep_factor),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, stats.throttle_max_sleep_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, stats.bitrot_cycle_enabled as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
|
||||
@@ -114,6 +175,10 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
stats.current_cycle_bucket_drive_scans as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
stats.current_cycle_bucket_drive_failures as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
@@ -123,6 +188,18 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
stats.current_cycle_bucket_drive_scans_per_second,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, stats.current_cycle_yield_events as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD,
|
||||
stats.current_cycle_yield_duration_seconds,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, stats.current_cycle_ilm_actions as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, stats.current_cycle_heal_objects as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD,
|
||||
stats.current_cycle_replication_checks as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
|
||||
@@ -132,12 +209,22 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
stats.last_cycle_directories_scanned as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
stats.last_cycle_bucket_drive_failures as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
stats.last_cycle_bucket_drive_scans_per_second,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, stats.last_cycle_yield_events as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, stats.last_cycle_yield_duration_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, stats.last_cycle_ilm_actions as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, stats.last_cycle_heal_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
|
||||
]
|
||||
}
|
||||
@@ -152,36 +239,58 @@ mod tests {
|
||||
let stats = ScannerStats {
|
||||
bucket_scans_finished: 100,
|
||||
bucket_scans_started: 100,
|
||||
bucket_scans_failed: 2,
|
||||
directories_scanned: 50000,
|
||||
objects_scanned: 1000000,
|
||||
versions_scanned: 1500000,
|
||||
last_activity_seconds: 30,
|
||||
active_paths: 4,
|
||||
throttle_idle_mode_enabled: 1,
|
||||
throttle_sleep_factor: 10.0,
|
||||
throttle_max_sleep_seconds: 15.0,
|
||||
yield_every_n_objects: 128,
|
||||
cycle_interval_seconds: 3600.0,
|
||||
bitrot_cycle_enabled: 1,
|
||||
bitrot_cycle_seconds: 86400.0,
|
||||
current_cycle: 12,
|
||||
completed_cycles: 11,
|
||||
current_cycle_age_seconds: 90,
|
||||
current_cycle_objects_scanned: 250,
|
||||
current_cycle_directories_scanned: 20,
|
||||
current_cycle_bucket_drive_scans: 2,
|
||||
current_cycle_bucket_drive_failures: 1,
|
||||
current_cycle_objects_per_second: 12.5,
|
||||
current_cycle_directories_per_second: 1.0,
|
||||
current_cycle_bucket_drive_scans_per_second: 0.1,
|
||||
current_cycle_yield_events: 8,
|
||||
current_cycle_yield_duration_seconds: 1.25,
|
||||
current_cycle_ilm_actions: 6,
|
||||
current_cycle_heal_objects: 2,
|
||||
current_cycle_replication_checks: 5,
|
||||
current_cycle_usage_saves: 3,
|
||||
current_scan_mode: 2,
|
||||
last_cycle_result: 1,
|
||||
last_cycle_duration_seconds: 42.5,
|
||||
last_cycle_objects_scanned: 900,
|
||||
last_cycle_directories_scanned: 80,
|
||||
last_cycle_bucket_drive_scans: 6,
|
||||
last_cycle_bucket_drive_failures: 2,
|
||||
last_cycle_objects_per_second: 18.0,
|
||||
last_cycle_directories_per_second: 1.6,
|
||||
last_cycle_bucket_drive_scans_per_second: 0.12,
|
||||
last_cycle_yield_events: 30,
|
||||
last_cycle_yield_duration_seconds: 9.5,
|
||||
last_cycle_ilm_actions: 44,
|
||||
last_cycle_heal_objects: 7,
|
||||
last_cycle_replication_checks: 12,
|
||||
last_cycle_usage_saves: 9,
|
||||
failed_cycles: 3,
|
||||
};
|
||||
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 26);
|
||||
assert_eq!(metrics.len(), 48);
|
||||
|
||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(objects.is_some());
|
||||
@@ -194,6 +303,46 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
||||
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
||||
|
||||
let bucket_scans_failed = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name());
|
||||
assert_eq!(bucket_scans_failed.map(|m| m.value), Some(2.0));
|
||||
|
||||
let throttle_idle_mode_enabled = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_idle_mode_enabled.map(|m| m.value), Some(1.0));
|
||||
|
||||
let throttle_sleep_factor = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_SLEEP_FACTOR_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_sleep_factor.map(|m| m.value), Some(10.0));
|
||||
|
||||
let throttle_max_sleep = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_max_sleep.map(|m| m.value), Some(15.0));
|
||||
|
||||
let yield_every_n_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_YIELD_EVERY_N_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(yield_every_n_objects.map(|m| m.value), Some(128.0));
|
||||
|
||||
let cycle_interval_seconds = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CYCLE_INTERVAL_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(cycle_interval_seconds.map(|m| m.value), Some(3600.0));
|
||||
|
||||
let bitrot_cycle_enabled = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name());
|
||||
assert_eq!(bitrot_cycle_enabled.map(|m| m.value), Some(1.0));
|
||||
|
||||
let bitrot_cycle_seconds = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BITROT_CYCLE_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(bitrot_cycle_seconds.map(|m| m.value), Some(86400.0));
|
||||
|
||||
let current_cycle = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
|
||||
@@ -224,6 +373,11 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
|
||||
|
||||
let current_cycle_bucket_drive_failures = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_failures.map(|m| m.value), Some(1.0));
|
||||
|
||||
let current_cycle_objects_rate = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||
@@ -239,6 +393,36 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
|
||||
|
||||
let current_cycle_yield_events = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_yield_events.map(|m| m.value), Some(8.0));
|
||||
|
||||
let current_cycle_yield_duration = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_yield_duration.map(|m| m.value), Some(1.25));
|
||||
|
||||
let current_cycle_ilm_actions = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_ilm_actions.map(|m| m.value), Some(6.0));
|
||||
|
||||
let current_cycle_heal_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_heal_objects.map(|m| m.value), Some(2.0));
|
||||
|
||||
let current_cycle_replication_checks = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_replication_checks.map(|m| m.value), Some(5.0));
|
||||
|
||||
let current_cycle_usage_saves = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_usage_saves.map(|m| m.value), Some(3.0));
|
||||
|
||||
let current_scan_mode = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
|
||||
@@ -269,6 +453,11 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
|
||||
|
||||
let last_cycle_bucket_drive_failures = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_failures.map(|m| m.value), Some(2.0));
|
||||
|
||||
let last_cycle_objects_rate = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||
@@ -284,6 +473,36 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
|
||||
|
||||
let last_cycle_yield_events = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_yield_events.map(|m| m.value), Some(30.0));
|
||||
|
||||
let last_cycle_yield_duration = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_yield_duration.map(|m| m.value), Some(9.5));
|
||||
|
||||
let last_cycle_ilm_actions = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_ilm_actions.map(|m| m.value), Some(44.0));
|
||||
|
||||
let last_cycle_heal_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_heal_objects.map(|m| m.value), Some(7.0));
|
||||
|
||||
let last_cycle_replication_checks = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_replication_checks.map(|m| m.value), Some(12.0));
|
||||
|
||||
let last_cycle_usage_saves = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_usage_saves.map(|m| m.value), Some(9.0));
|
||||
|
||||
let failed_cycles = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
|
||||
@@ -295,7 +514,7 @@ mod tests {
|
||||
let stats = ScannerStats::default();
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 26);
|
||||
assert_eq!(metrics.len(), 48);
|
||||
for metric in &metrics {
|
||||
assert_eq!(metric.value, 0.0);
|
||||
assert!(metric.labels.is_empty());
|
||||
|
||||
@@ -279,24 +279,46 @@ pub enum MetricName {
|
||||
ScannerVersionsScanned,
|
||||
ScannerLastActivitySeconds,
|
||||
ScannerActivePaths,
|
||||
ScannerBucketScansFailed,
|
||||
ScannerThrottleIdleModeEnabled,
|
||||
ScannerThrottleSleepFactor,
|
||||
ScannerThrottleMaxSleepSeconds,
|
||||
ScannerYieldEveryNObjects,
|
||||
ScannerCycleIntervalSeconds,
|
||||
ScannerBitrotCycleEnabled,
|
||||
ScannerBitrotCycleSeconds,
|
||||
ScannerCurrentCycle,
|
||||
ScannerCompletedCycles,
|
||||
ScannerCurrentCycleAgeSeconds,
|
||||
ScannerCurrentCycleObjectsScanned,
|
||||
ScannerCurrentCycleDirectoriesScanned,
|
||||
ScannerCurrentCycleBucketDriveScans,
|
||||
ScannerCurrentCycleBucketDriveFailures,
|
||||
ScannerCurrentCycleObjectsPerSecond,
|
||||
ScannerCurrentCycleDirectoriesPerSecond,
|
||||
ScannerCurrentCycleBucketDriveScansPerSecond,
|
||||
ScannerCurrentCycleYieldEvents,
|
||||
ScannerCurrentCycleYieldDurationSeconds,
|
||||
ScannerCurrentCycleIlmActions,
|
||||
ScannerCurrentCycleHealObjects,
|
||||
ScannerCurrentCycleReplicationChecks,
|
||||
ScannerCurrentCycleUsageSaves,
|
||||
ScannerCurrentScanMode,
|
||||
ScannerLastCycleResult,
|
||||
ScannerLastCycleDurationSeconds,
|
||||
ScannerLastCycleObjectsScanned,
|
||||
ScannerLastCycleDirectoriesScanned,
|
||||
ScannerLastCycleBucketDriveScans,
|
||||
ScannerLastCycleBucketDriveFailures,
|
||||
ScannerLastCycleObjectsPerSecond,
|
||||
ScannerLastCycleDirectoriesPerSecond,
|
||||
ScannerLastCycleBucketDriveScansPerSecond,
|
||||
ScannerLastCycleYieldEvents,
|
||||
ScannerLastCycleYieldDurationSeconds,
|
||||
ScannerLastCycleIlmActions,
|
||||
ScannerLastCycleHealObjects,
|
||||
ScannerLastCycleReplicationChecks,
|
||||
ScannerLastCycleUsageSaves,
|
||||
ScannerFailedCycles,
|
||||
|
||||
// CPU system-related metrics
|
||||
@@ -639,24 +661,46 @@ impl MetricName {
|
||||
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
|
||||
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
|
||||
Self::ScannerActivePaths => "active_paths".to_string(),
|
||||
Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(),
|
||||
Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(),
|
||||
Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(),
|
||||
Self::ScannerThrottleMaxSleepSeconds => "throttle_max_sleep_seconds".to_string(),
|
||||
Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(),
|
||||
Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(),
|
||||
Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(),
|
||||
Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(),
|
||||
Self::ScannerCurrentCycle => "current_cycle".to_string(),
|
||||
Self::ScannerCompletedCycles => "completed_cycles".to_string(),
|
||||
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
|
||||
Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
|
||||
Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveFailures => "current_cycle_bucket_drive_failures".to_string(),
|
||||
Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleYieldEvents => "current_cycle_yield_events".to_string(),
|
||||
Self::ScannerCurrentCycleYieldDurationSeconds => "current_cycle_yield_duration_seconds".to_string(),
|
||||
Self::ScannerCurrentCycleIlmActions => "current_cycle_ilm_actions".to_string(),
|
||||
Self::ScannerCurrentCycleHealObjects => "current_cycle_heal_objects".to_string(),
|
||||
Self::ScannerCurrentCycleReplicationChecks => "current_cycle_replication_checks".to_string(),
|
||||
Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(),
|
||||
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
|
||||
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
|
||||
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
|
||||
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
|
||||
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveFailures => "last_cycle_bucket_drive_failures".to_string(),
|
||||
Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
|
||||
Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
|
||||
Self::ScannerLastCycleYieldEvents => "last_cycle_yield_events".to_string(),
|
||||
Self::ScannerLastCycleYieldDurationSeconds => "last_cycle_yield_duration_seconds".to_string(),
|
||||
Self::ScannerLastCycleIlmActions => "last_cycle_ilm_actions".to_string(),
|
||||
Self::ScannerLastCycleHealObjects => "last_cycle_heal_objects".to_string(),
|
||||
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
|
||||
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
|
||||
Self::ScannerFailedCycles => "failed_cycles".to_string(),
|
||||
|
||||
// CPU system-related metrics
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::sync::LazyLock;
|
||||
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansFinished,
|
||||
"Total number of bucket scans finished since server start",
|
||||
"Total number of bucket-drive scans finished since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
@@ -29,7 +29,16 @@ pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLo
|
||||
pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansStarted,
|
||||
"Total number of bucket scans started since server start",
|
||||
"Total number of bucket-drive scans started since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BUCKET_SCANS_FAILED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansFailed,
|
||||
"Total number of bucket-drive scans that failed since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
@@ -80,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleIdleModeEnabled,
|
||||
"Whether scanner idle-mode self-throttling is enabled: 1 enabled, 0 disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_SLEEP_FACTOR_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleSleepFactor,
|
||||
"Effective scanner sleep factor used to compute proportional self-throttle sleeps.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleMaxSleepSeconds,
|
||||
"Effective maximum scanner self-throttle sleep duration in seconds.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_YIELD_EVERY_N_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerYieldEveryNObjects,
|
||||
"Current object interval for cooperative scanner runtime yields, or zero when disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_INTERVAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCycleIntervalSeconds,
|
||||
"Effective scanner cycle interval in seconds.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerBitrotCycleEnabled,
|
||||
"Whether periodic scanner bitrot deep scans are enabled: 1 enabled, 0 disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BITROT_CYCLE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerBitrotCycleSeconds,
|
||||
"Effective scanner bitrot deep-scan interval in seconds; zero means disabled or deep-scan every cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycle,
|
||||
@@ -134,6 +206,15 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescripto
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleBucketDriveFailures,
|
||||
"Number of bucket-drive scans that failed in the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleObjectsPerSecond,
|
||||
@@ -161,6 +242,60 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<Metr
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleYieldEvents,
|
||||
"Number of scanner self-throttle yield events in the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleYieldDurationSeconds,
|
||||
"Total scanner self-throttle yield duration in seconds for the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleIlmActions,
|
||||
"Number of lifecycle actions applied by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleHealObjects,
|
||||
"Number of object heal candidates enqueued by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleReplicationChecks,
|
||||
"Number of replication heal checks run by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleUsageSaves,
|
||||
"Number of data-usage save operations run by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentScanMode,
|
||||
@@ -215,6 +350,15 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor>
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleBucketDriveFailures,
|
||||
"Number of bucket-drive scans that failed in the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleObjectsPerSecond,
|
||||
@@ -242,6 +386,60 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricD
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleYieldEvents,
|
||||
"Number of scanner self-throttle yield events in the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleYieldDurationSeconds,
|
||||
"Total scanner self-throttle yield duration in seconds for the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleIlmActions,
|
||||
"Number of lifecycle actions applied by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleHealObjects,
|
||||
"Number of object heal candidates enqueued by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleReplicationChecks,
|
||||
"Number of replication heal checks run by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleUsageSaves,
|
||||
"Number of data-usage save operations run by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerFailedCycles,
|
||||
|
||||
@@ -896,10 +896,23 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
|
||||
///
|
||||
/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
|
||||
/// rustfs-obs scanner collector shape.
|
||||
fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_scans_finished: u64) -> u64 {
|
||||
life_time_ops
|
||||
.get("scan_bucket_drive_start")
|
||||
.copied()
|
||||
.unwrap_or(bucket_scans_finished)
|
||||
}
|
||||
|
||||
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
let metrics = global_metrics().report().await;
|
||||
let now = Utc::now();
|
||||
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
|
||||
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
|
||||
let bucket_scans_failed = metrics
|
||||
.life_time_ops
|
||||
.get("scan_bucket_drive_failure")
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
|
||||
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
|
||||
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
|
||||
@@ -914,21 +927,27 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
|
||||
Some(ScannerStats {
|
||||
bucket_scans_finished,
|
||||
// `global_metrics()` currently tracks completed bucket-drive scans, not a
|
||||
// separate started counter. Mirror the finished count until Task 5/Task 10
|
||||
// expands the scanner runtime source shape.
|
||||
bucket_scans_started: bucket_scans_finished,
|
||||
bucket_scans_started,
|
||||
bucket_scans_failed,
|
||||
directories_scanned,
|
||||
objects_scanned,
|
||||
versions_scanned,
|
||||
last_activity_seconds,
|
||||
active_paths,
|
||||
throttle_idle_mode_enabled: u64::from(metrics.throttle_idle_mode_enabled),
|
||||
throttle_sleep_factor: metrics.throttle_sleep_factor,
|
||||
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
|
||||
yield_every_n_objects: metrics.yield_every_n_objects,
|
||||
cycle_interval_seconds: metrics.cycle_interval_seconds,
|
||||
bitrot_cycle_enabled: u64::from(metrics.bitrot_cycle_enabled),
|
||||
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||
current_cycle: metrics.current_cycle,
|
||||
completed_cycles,
|
||||
current_cycle_age_seconds,
|
||||
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
|
||||
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
|
||||
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
|
||||
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
|
||||
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
|
||||
current_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||
metrics.current_cycle_directories_scanned,
|
||||
@@ -938,12 +957,19 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
metrics.current_cycle_bucket_drive_scans,
|
||||
current_cycle_age,
|
||||
),
|
||||
current_cycle_yield_events: metrics.current_cycle_yield_events,
|
||||
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
|
||||
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
|
||||
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
|
||||
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
|
||||
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
|
||||
current_scan_mode,
|
||||
last_cycle_result: metrics.last_cycle_result_code,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
|
||||
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
|
||||
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
|
||||
last_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||
metrics.last_cycle_directories_scanned,
|
||||
@@ -953,6 +979,12 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
metrics.last_cycle_bucket_drive_scans,
|
||||
last_cycle_duration,
|
||||
),
|
||||
last_cycle_yield_events: metrics.last_cycle_yield_events,
|
||||
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
|
||||
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
|
||||
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
|
||||
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
|
||||
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
|
||||
failed_cycles: metrics.failed_cycles,
|
||||
})
|
||||
}
|
||||
@@ -1056,6 +1088,21 @@ mod tests {
|
||||
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_bucket_scans_started_uses_explicit_started_count() {
|
||||
let mut life_time_ops = HashMap::new();
|
||||
life_time_ops.insert("scan_bucket_drive_start".to_string(), 7);
|
||||
|
||||
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_bucket_scans_started_falls_back_to_finished_count() {
|
||||
let life_time_ops = HashMap::new();
|
||||
|
||||
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_work_rate_per_second_reports_rate() {
|
||||
assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
|
||||
|
||||
@@ -266,8 +266,13 @@ fn bitrot_scan_cycle() -> Option<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
|
||||
let Some(bitrot_cycle) = bitrot_scan_cycle() else {
|
||||
fn get_cycle_scan_mode(
|
||||
current_cycle: u64,
|
||||
bitrot_start_cycle: u64,
|
||||
bitrot_start_time: Option<DateTime<Utc>>,
|
||||
bitrot_cycle: Option<Duration>,
|
||||
) -> HealScanMode {
|
||||
let Some(bitrot_cycle) = bitrot_cycle else {
|
||||
return HealScanMode::Normal;
|
||||
};
|
||||
|
||||
@@ -299,8 +304,10 @@ fn background_heal_info_for_scan_start(
|
||||
current_cycle: u64,
|
||||
scan_mode: HealScanMode,
|
||||
now: DateTime<Utc>,
|
||||
bitrot_cycle: Option<Duration>,
|
||||
) -> Option<BackgroundHealInfo> {
|
||||
let reset_bitrot_start = scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now);
|
||||
let reset_bitrot_start =
|
||||
scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, bitrot_cycle);
|
||||
if info.current_scan_mode == scan_mode && !reset_bitrot_start {
|
||||
return None;
|
||||
}
|
||||
@@ -314,12 +321,17 @@ fn background_heal_info_for_scan_start(
|
||||
Some(info)
|
||||
}
|
||||
|
||||
fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now: DateTime<Utc>) -> bool {
|
||||
fn should_reset_bitrot_start(
|
||||
info: &BackgroundHealInfo,
|
||||
current_cycle: u64,
|
||||
now: DateTime<Utc>,
|
||||
bitrot_cycle: Option<Duration>,
|
||||
) -> bool {
|
||||
let Some(bitrot_start_time) = info.bitrot_start_time else {
|
||||
return true;
|
||||
};
|
||||
|
||||
let Some(bitrot_cycle) = bitrot_scan_cycle() else {
|
||||
let Some(bitrot_cycle) = bitrot_cycle else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -424,6 +436,9 @@ fn get_lock_acquire_timeout() -> Duration {
|
||||
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
SCANNER_SLEEPER.refresh_from_env();
|
||||
let configured_cycle_interval = cycle_interval();
|
||||
let configured_bitrot_cycle = bitrot_scan_cycle();
|
||||
global_metrics().record_scanner_cycle_config(configured_cycle_interval, configured_bitrot_cycle);
|
||||
info!("Start run data scanner cycle");
|
||||
cycle_info.current = cycle_info.next;
|
||||
let now = Instant::now();
|
||||
@@ -437,11 +452,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
||||
cycle_info.current,
|
||||
background_heal_info.bitrot_start_cycle,
|
||||
background_heal_info.bitrot_start_time,
|
||||
configured_bitrot_cycle,
|
||||
);
|
||||
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
||||
if let Some(new_heal_info) =
|
||||
background_heal_info_for_scan_start(background_heal_info.clone(), cycle_info.current, scan_mode, Utc::now())
|
||||
{
|
||||
if let Some(new_heal_info) = background_heal_info_for_scan_start(
|
||||
background_heal_info.clone(),
|
||||
cycle_info.current,
|
||||
scan_mode,
|
||||
Utc::now(),
|
||||
configured_bitrot_cycle,
|
||||
) {
|
||||
background_heal_info = new_heal_info.clone();
|
||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||
}
|
||||
@@ -600,18 +620,22 @@ pub async fn store_data_usage_in_backend(
|
||||
// Save a backup every 10th update
|
||||
if attempts > 10 {
|
||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
|
||||
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
|
||||
}
|
||||
done_save();
|
||||
attempts = 1;
|
||||
}
|
||||
|
||||
// Save main configuration
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
|
||||
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
} else {
|
||||
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
}
|
||||
done_save();
|
||||
|
||||
attempts += 1;
|
||||
}
|
||||
@@ -866,7 +890,7 @@ mod tests {
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()));
|
||||
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
||||
assert_eq!(mode, HealScanMode::Deep);
|
||||
});
|
||||
}
|
||||
@@ -878,8 +902,8 @@ mod tests {
|
||||
let recent = Utc::now() - chrono::Duration::minutes(30);
|
||||
let old = Utc::now() - chrono::Duration::hours(2);
|
||||
|
||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal);
|
||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep);
|
||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal);
|
||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -887,7 +911,7 @@ mod tests {
|
||||
#[serial]
|
||||
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
||||
assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal);
|
||||
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -895,8 +919,9 @@ mod tests {
|
||||
#[serial]
|
||||
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||
let now = Utc::now();
|
||||
let info = background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now)
|
||||
.expect("deep scan should update background heal info");
|
||||
let info =
|
||||
background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle())
|
||||
.expect("deep scan should update background heal info");
|
||||
|
||||
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
||||
assert_eq!(info.bitrot_start_cycle, 7);
|
||||
@@ -914,7 +939,7 @@ mod tests {
|
||||
current_scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
|
||||
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now())
|
||||
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle())
|
||||
.expect("deep scan should mark active status");
|
||||
|
||||
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
||||
|
||||
@@ -16,19 +16,19 @@ use std::collections::HashSet;
|
||||
use std::fs::FileType;
|
||||
use std::io::ErrorKind;
|
||||
use std::sync::{Arc, Once};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use crate::ReplTargetSizeSummary;
|
||||
use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path};
|
||||
use crate::error::ScannerError;
|
||||
use crate::scanner_io::ScannerIODisk as _;
|
||||
use crate::sleeper::DynamicSleeper;
|
||||
use crate::sleeper::{DynamicSleeper, scanner_yield_every_n_objects};
|
||||
use metrics::{counter, describe_counter};
|
||||
use rustfs_common::heal_channel::{
|
||||
HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode,
|
||||
send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
|
||||
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater, global_metrics};
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
|
||||
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
|
||||
@@ -143,13 +143,6 @@ fn scanner_excess_folders_threshold() -> u64 {
|
||||
)
|
||||
}
|
||||
|
||||
fn scanner_yield_every_n_objects() -> u64 {
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
)
|
||||
}
|
||||
|
||||
fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
|
||||
yield_every > 0 && object_count.is_multiple_of(yield_every)
|
||||
}
|
||||
@@ -485,6 +478,7 @@ impl ScannerItem {
|
||||
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
||||
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
done_ilm(1)();
|
||||
global_metrics().record_scanner_ilm_action(1);
|
||||
break 'eventLoop;
|
||||
}
|
||||
|
||||
@@ -497,6 +491,7 @@ impl ScannerItem {
|
||||
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
||||
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
done_ilm(1)();
|
||||
global_metrics().record_scanner_ilm_action(1);
|
||||
}
|
||||
IlmAction::DeleteVersionAction => {
|
||||
remaining_versions -= 1;
|
||||
@@ -510,11 +505,13 @@ impl ScannerItem {
|
||||
}
|
||||
noncurrent_events.push(event.clone());
|
||||
done_ilm(1)();
|
||||
global_metrics().record_scanner_ilm_action(1);
|
||||
}
|
||||
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
||||
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
|
||||
apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
done_ilm(1)();
|
||||
global_metrics().record_scanner_ilm_action(1);
|
||||
}
|
||||
|
||||
IlmAction::NoneAction | IlmAction::ActionCount => {
|
||||
@@ -560,7 +557,9 @@ impl ScannerItem {
|
||||
return;
|
||||
};
|
||||
|
||||
let done_replication = Metrics::time(Metric::CheckReplication);
|
||||
let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
|
||||
done_replication();
|
||||
if !Self::should_account_replication_stats(oi) {
|
||||
return;
|
||||
}
|
||||
@@ -1110,7 +1109,9 @@ impl FolderScanner {
|
||||
timer.sleep().await;
|
||||
|
||||
if should_yield_after_object(object_count, yield_every_objects) {
|
||||
let yield_start = Instant::now();
|
||||
tokio::task::yield_now().await;
|
||||
global_metrics().record_scanner_yield(yield_start.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rand::seq::SliceRandom as _;
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete};
|
||||
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, global_metrics};
|
||||
use rustfs_config::{
|
||||
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
||||
ENV_SCANNER_MAX_CONCURRENT_SET_SCANS,
|
||||
@@ -116,6 +116,28 @@ impl Drop for DiskBucketScanActiveGuard {
|
||||
}
|
||||
}
|
||||
|
||||
struct BucketDriveFailureGuard {
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl BucketDriveFailureGuard {
|
||||
fn new() -> Self {
|
||||
Self { failed: true }
|
||||
}
|
||||
|
||||
fn mark_success(&mut self) {
|
||||
self.failed = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BucketDriveFailureGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.failed {
|
||||
global_metrics().record_scan_bucket_drive_failure();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
|
||||
counter
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
|
||||
@@ -217,9 +239,11 @@ async fn persist_and_publish_cache_snapshot<S: StorageAPI>(
|
||||
) -> Option<SystemTime> {
|
||||
let last_update = cache_snapshot.info.last_update;
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
|
||||
error!("Failed to save data usage cache: {}", e);
|
||||
}
|
||||
done_save();
|
||||
|
||||
if let Err(e) = updates.send(cache_snapshot).await {
|
||||
error!("Failed to send data usage cache: {}", e);
|
||||
@@ -732,9 +756,12 @@ impl ScannerIOCache for SetDisks {
|
||||
|
||||
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
|
||||
&& last_update > before_update
|
||||
&& let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await
|
||||
{
|
||||
error!("Failed to save data usage cache: {}", e);
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
|
||||
error!("Failed to save data usage cache: {}", e);
|
||||
}
|
||||
done_save();
|
||||
}
|
||||
|
||||
if let Err(e) = update_fut.await {
|
||||
@@ -775,9 +802,11 @@ impl ScannerIOCache for SetDisks {
|
||||
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
|
||||
}
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
|
||||
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
|
||||
}
|
||||
done_save();
|
||||
}
|
||||
}));
|
||||
}
|
||||
@@ -900,6 +929,8 @@ impl ScannerIODisk for Disk {
|
||||
let drive_start = std::time::Instant::now();
|
||||
let bucket = cache.info.name.clone();
|
||||
let disk_path = self.path().to_string_lossy().to_string();
|
||||
global_metrics().record_scan_bucket_drive_start();
|
||||
let mut failure_guard = BucketDriveFailureGuard::new();
|
||||
let _guard = self.start_scan();
|
||||
|
||||
let mut cache = cache;
|
||||
@@ -966,9 +997,11 @@ impl ScannerIODisk for Disk {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
|
||||
data_usage_info.info.last_update = Some(SystemTime::now());
|
||||
failure_guard.mark_success();
|
||||
Ok(data_usage_info)
|
||||
}
|
||||
Err(e) => {
|
||||
done_drive();
|
||||
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
|
||||
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
|
||||
}
|
||||
|
||||
@@ -16,7 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
use std::time::Instant;
|
||||
|
||||
use rustfs_config::{DEFAULT_SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, ScannerSpeed};
|
||||
use rustfs_common::metrics::global_metrics;
|
||||
use rustfs_config::{
|
||||
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
|
||||
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||
};
|
||||
use tokio::time::Duration;
|
||||
|
||||
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
||||
@@ -64,6 +68,10 @@ fn scanner_env_config() -> (ScannerSpeed, bool) {
|
||||
(speed, idle_mode)
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_yield_every_n_objects() -> u64 {
|
||||
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
|
||||
}
|
||||
|
||||
/// When `true` (default), the scanner throttles itself between operations.
|
||||
/// When `false`, all sleeps are skipped and the scanner runs at full speed.
|
||||
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
|
||||
@@ -74,7 +82,9 @@ pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
|
||||
let (speed, idle_mode) = scanner_env_config();
|
||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||
|
||||
DynamicSleeper::new(speed)
|
||||
let sleeper = DynamicSleeper::new(speed);
|
||||
sleeper.record_throttle_config();
|
||||
sleeper
|
||||
});
|
||||
|
||||
/// Proportional-backoff sleeper for the data scanner.
|
||||
@@ -124,6 +134,7 @@ impl DynamicSleeper {
|
||||
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep);
|
||||
if !sleep_dur.is_zero() {
|
||||
tokio::time::sleep(sleep_dur).await;
|
||||
global_metrics().record_scanner_yield(sleep_dur);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +161,17 @@ impl DynamicSleeper {
|
||||
let (speed, idle_mode) = scanner_env_config();
|
||||
self.update(speed);
|
||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||
self.record_throttle_config();
|
||||
}
|
||||
|
||||
fn record_throttle_config(&self) {
|
||||
let (factor, max_sleep) = self.read_params();
|
||||
global_metrics().record_scanner_throttle_config(
|
||||
SCANNER_IDLE_MODE.load(Ordering::Relaxed),
|
||||
factor,
|
||||
max_sleep,
|
||||
scanner_yield_every_n_objects(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +199,7 @@ impl SleepTimer {
|
||||
.min(max_sleep);
|
||||
if !sleep_dur.is_zero() {
|
||||
tokio::time::sleep(sleep_dur).await;
|
||||
global_metrics().record_scanner_yield(sleep_dur);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user