mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
feat(scanner): expand scanner observability metrics (#3159)
* feat(scanner): expand scanner observability metrics * chore(scanner): align bucket-drive metric wording --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -146,6 +146,8 @@ pub enum Metric {
|
|||||||
ScanCycle, // Full cycle, cluster global.
|
ScanCycle, // Full cycle, cluster global.
|
||||||
ScanBucketDrive, // Single bucket on one drive.
|
ScanBucketDrive, // Single bucket on one drive.
|
||||||
CompactFolder, // Folder compacted.
|
CompactFolder, // Folder compacted.
|
||||||
|
ScanBucketDriveStart,
|
||||||
|
ScanBucketDriveFailure,
|
||||||
|
|
||||||
// Must be last:
|
// Must be last:
|
||||||
Last,
|
Last,
|
||||||
@@ -179,6 +181,8 @@ impl Metric {
|
|||||||
Self::ScanCycle => "scan_cycle",
|
Self::ScanCycle => "scan_cycle",
|
||||||
Self::ScanBucketDrive => "scan_bucket_drive",
|
Self::ScanBucketDrive => "scan_bucket_drive",
|
||||||
Self::CompactFolder => "compact_folder",
|
Self::CompactFolder => "compact_folder",
|
||||||
|
Self::ScanBucketDriveStart => "scan_bucket_drive_start",
|
||||||
|
Self::ScanBucketDriveFailure => "scan_bucket_drive_failure",
|
||||||
Self::Last => "last",
|
Self::Last => "last",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -213,7 +217,9 @@ impl Metric {
|
|||||||
21 => Some(Self::ScanCycle),
|
21 => Some(Self::ScanCycle),
|
||||||
22 => Some(Self::ScanBucketDrive),
|
22 => Some(Self::ScanBucketDrive),
|
||||||
23 => Some(Self::CompactFolder),
|
23 => Some(Self::CompactFolder),
|
||||||
24 => Some(Self::Last),
|
24 => Some(Self::ScanBucketDriveStart),
|
||||||
|
25 => Some(Self::ScanBucketDriveFailure),
|
||||||
|
26 => Some(Self::Last),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -346,12 +352,35 @@ pub struct Metrics {
|
|||||||
current_scan_cycle_objects_start: AtomicU64,
|
current_scan_cycle_objects_start: AtomicU64,
|
||||||
current_scan_cycle_directories_start: AtomicU64,
|
current_scan_cycle_directories_start: AtomicU64,
|
||||||
current_scan_cycle_bucket_drive_scans_start: AtomicU64,
|
current_scan_cycle_bucket_drive_scans_start: AtomicU64,
|
||||||
|
current_scan_cycle_bucket_drive_failures_start: AtomicU64,
|
||||||
|
current_scan_cycle_yield_events_start: AtomicU64,
|
||||||
|
current_scan_cycle_yield_duration_millis_start: AtomicU64,
|
||||||
|
current_scan_cycle_ilm_actions_start: AtomicU64,
|
||||||
|
current_scan_cycle_heal_objects_start: AtomicU64,
|
||||||
|
current_scan_cycle_replication_checks_start: AtomicU64,
|
||||||
|
current_scan_cycle_usage_saves_start: AtomicU64,
|
||||||
last_scan_cycle_result: AtomicU8,
|
last_scan_cycle_result: AtomicU8,
|
||||||
last_scan_cycle_duration_millis: AtomicU64,
|
last_scan_cycle_duration_millis: AtomicU64,
|
||||||
last_scan_cycle_objects_scanned: AtomicU64,
|
last_scan_cycle_objects_scanned: AtomicU64,
|
||||||
last_scan_cycle_directories_scanned: AtomicU64,
|
last_scan_cycle_directories_scanned: AtomicU64,
|
||||||
last_scan_cycle_bucket_drive_scans: AtomicU64,
|
last_scan_cycle_bucket_drive_scans: AtomicU64,
|
||||||
|
last_scan_cycle_bucket_drive_failures: AtomicU64,
|
||||||
|
last_scan_cycle_yield_events: AtomicU64,
|
||||||
|
last_scan_cycle_yield_duration_millis: AtomicU64,
|
||||||
|
last_scan_cycle_ilm_actions: AtomicU64,
|
||||||
|
last_scan_cycle_heal_objects: AtomicU64,
|
||||||
|
last_scan_cycle_replication_checks: AtomicU64,
|
||||||
|
last_scan_cycle_usage_saves: AtomicU64,
|
||||||
failed_scan_cycles: AtomicU64,
|
failed_scan_cycles: AtomicU64,
|
||||||
|
scanner_yield_duration_millis: AtomicU64,
|
||||||
|
scanner_ilm_actions: AtomicU64,
|
||||||
|
scanner_throttle_idle_mode_enabled: AtomicBool,
|
||||||
|
scanner_throttle_sleep_factor_micros: AtomicU64,
|
||||||
|
scanner_throttle_max_sleep_millis: AtomicU64,
|
||||||
|
scanner_yield_every_n_objects: AtomicU64,
|
||||||
|
scanner_cycle_interval_millis: AtomicU64,
|
||||||
|
scanner_bitrot_cycle_enabled: AtomicBool,
|
||||||
|
scanner_bitrot_cycle_millis: AtomicU64,
|
||||||
}
|
}
|
||||||
|
|
||||||
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
||||||
@@ -374,6 +403,13 @@ pub struct ScanCycleWorkSnapshot {
|
|||||||
objects_scanned: u64,
|
objects_scanned: u64,
|
||||||
directories_scanned: u64,
|
directories_scanned: u64,
|
||||||
bucket_drive_scans: u64,
|
bucket_drive_scans: u64,
|
||||||
|
bucket_drive_failures: u64,
|
||||||
|
yield_events: u64,
|
||||||
|
yield_duration_millis: u64,
|
||||||
|
ilm_actions: u64,
|
||||||
|
heal_objects: u64,
|
||||||
|
replication_checks: u64,
|
||||||
|
usage_saves: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
@@ -409,6 +445,20 @@ pub struct ScannerMetricsReport {
|
|||||||
pub current_cycle_directories_scanned: u64,
|
pub current_cycle_directories_scanned: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub current_cycle_bucket_drive_scans: u64,
|
pub current_cycle_bucket_drive_scans: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_bucket_drive_failures: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_yield_events: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_yield_duration_seconds: f64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_ilm_actions: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_heal_objects: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_replication_checks: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub current_cycle_usage_saves: u64,
|
||||||
pub last_cycle_result: String,
|
pub last_cycle_result: String,
|
||||||
pub last_cycle_result_code: u64,
|
pub last_cycle_result_code: u64,
|
||||||
pub last_cycle_duration_seconds: f64,
|
pub last_cycle_duration_seconds: f64,
|
||||||
@@ -418,7 +468,35 @@ pub struct ScannerMetricsReport {
|
|||||||
pub last_cycle_directories_scanned: u64,
|
pub last_cycle_directories_scanned: u64,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub last_cycle_bucket_drive_scans: u64,
|
pub last_cycle_bucket_drive_scans: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_bucket_drive_failures: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_yield_events: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_yield_duration_seconds: f64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_ilm_actions: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_heal_objects: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_replication_checks: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub last_cycle_usage_saves: u64,
|
||||||
pub failed_cycles: u64,
|
pub failed_cycles: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub throttle_idle_mode_enabled: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub throttle_sleep_factor: f64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub throttle_max_sleep_seconds: f64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub yield_every_n_objects: u64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub cycle_interval_seconds: f64,
|
||||||
|
#[serde(default)]
|
||||||
|
pub bitrot_cycle_enabled: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub bitrot_cycle_seconds: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CurrentCycle {
|
impl CurrentCycle {
|
||||||
@@ -464,6 +542,14 @@ fn duration_millis_saturated(duration: Duration) -> u64 {
|
|||||||
duration.as_millis().min(u64::MAX as u128) as u64
|
duration.as_millis().min(u64::MAX as u128) as u64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn scaled_f64_to_u64_saturated(value: f64, scale: f64) -> u64 {
|
||||||
|
if !value.is_finite() || value <= 0.0 {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
(value * scale).round().clamp(0.0, u64::MAX as f64) as u64
|
||||||
|
}
|
||||||
|
|
||||||
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
|
pub fn emit_scan_cycle_complete(success: bool, duration: Duration) {
|
||||||
let result = if success {
|
let result = if success {
|
||||||
SCAN_CYCLE_RESULT_SUCCESS_LABEL
|
SCAN_CYCLE_RESULT_SUCCESS_LABEL
|
||||||
@@ -514,12 +600,35 @@ impl Metrics {
|
|||||||
current_scan_cycle_objects_start: AtomicU64::new(0),
|
current_scan_cycle_objects_start: AtomicU64::new(0),
|
||||||
current_scan_cycle_directories_start: AtomicU64::new(0),
|
current_scan_cycle_directories_start: AtomicU64::new(0),
|
||||||
current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
|
current_scan_cycle_bucket_drive_scans_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_bucket_drive_failures_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_yield_events_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_yield_duration_millis_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_ilm_actions_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_heal_objects_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_replication_checks_start: AtomicU64::new(0),
|
||||||
|
current_scan_cycle_usage_saves_start: AtomicU64::new(0),
|
||||||
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
||||||
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
||||||
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
||||||
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
||||||
last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
|
last_scan_cycle_bucket_drive_scans: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_bucket_drive_failures: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_yield_events: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_yield_duration_millis: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_ilm_actions: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_heal_objects: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_replication_checks: AtomicU64::new(0),
|
||||||
|
last_scan_cycle_usage_saves: AtomicU64::new(0),
|
||||||
failed_scan_cycles: AtomicU64::new(0),
|
failed_scan_cycles: AtomicU64::new(0),
|
||||||
|
scanner_yield_duration_millis: AtomicU64::new(0),
|
||||||
|
scanner_ilm_actions: AtomicU64::new(0),
|
||||||
|
scanner_throttle_idle_mode_enabled: AtomicBool::new(false),
|
||||||
|
scanner_throttle_sleep_factor_micros: AtomicU64::new(0),
|
||||||
|
scanner_throttle_max_sleep_millis: AtomicU64::new(0),
|
||||||
|
scanner_yield_every_n_objects: AtomicU64::new(0),
|
||||||
|
scanner_cycle_interval_millis: AtomicU64::new(0),
|
||||||
|
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
|
||||||
|
scanner_bitrot_cycle_millis: AtomicU64::new(0),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -621,6 +730,58 @@ impl Metrics {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn record_scanner_yield(&self, duration: Duration) {
|
||||||
|
let metric_idx = Metric::Yield as usize;
|
||||||
|
self.operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||||
|
if metric_idx < Metric::LastRealtime as usize {
|
||||||
|
self.latency[metric_idx].add(duration);
|
||||||
|
}
|
||||||
|
let duration_millis = duration_millis_saturated(duration);
|
||||||
|
let _ = self
|
||||||
|
.scanner_yield_duration_millis
|
||||||
|
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||||
|
Some(current.saturating_add(duration_millis))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_scanner_ilm_action(&self, count: u64) {
|
||||||
|
self.scanner_ilm_actions.fetch_add(count, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_scan_bucket_drive_start(&self) {
|
||||||
|
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_scan_bucket_drive_failure(&self) {
|
||||||
|
self.operations[Metric::ScanBucketDriveFailure as usize].fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_scanner_throttle_config(
|
||||||
|
&self,
|
||||||
|
idle_mode_enabled: bool,
|
||||||
|
sleep_factor: f64,
|
||||||
|
max_sleep: Duration,
|
||||||
|
yield_every_n_objects: u64,
|
||||||
|
) {
|
||||||
|
self.scanner_throttle_idle_mode_enabled
|
||||||
|
.store(idle_mode_enabled, Ordering::Relaxed);
|
||||||
|
self.scanner_throttle_sleep_factor_micros
|
||||||
|
.store(scaled_f64_to_u64_saturated(sleep_factor, 1_000_000.0), Ordering::Relaxed);
|
||||||
|
self.scanner_throttle_max_sleep_millis
|
||||||
|
.store(duration_millis_saturated(max_sleep), Ordering::Relaxed);
|
||||||
|
self.scanner_yield_every_n_objects
|
||||||
|
.store(yield_every_n_objects, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_scanner_cycle_config(&self, cycle_interval: Duration, bitrot_cycle: Option<Duration>) {
|
||||||
|
self.scanner_cycle_interval_millis
|
||||||
|
.store(duration_millis_saturated(cycle_interval), Ordering::Relaxed);
|
||||||
|
self.scanner_bitrot_cycle_enabled
|
||||||
|
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
|
||||||
|
self.scanner_bitrot_cycle_millis
|
||||||
|
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// Read-side helpers
|
// Read-side helpers
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
@@ -686,13 +847,27 @@ impl Metrics {
|
|||||||
.store(snapshot.directories_scanned, Ordering::Relaxed);
|
.store(snapshot.directories_scanned, Ordering::Relaxed);
|
||||||
self.current_scan_cycle_bucket_drive_scans_start
|
self.current_scan_cycle_bucket_drive_scans_start
|
||||||
.store(snapshot.bucket_drive_scans, Ordering::Relaxed);
|
.store(snapshot.bucket_drive_scans, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_bucket_drive_failures_start
|
||||||
|
.store(snapshot.bucket_drive_failures, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_yield_events_start
|
||||||
|
.store(snapshot.yield_events, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_yield_duration_millis_start
|
||||||
|
.store(snapshot.yield_duration_millis, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_ilm_actions_start
|
||||||
|
.store(snapshot.ilm_actions, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_heal_objects_start
|
||||||
|
.store(snapshot.heal_objects, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_replication_checks_start
|
||||||
|
.store(snapshot.replication_checks, Ordering::Relaxed);
|
||||||
|
self.current_scan_cycle_usage_saves_start
|
||||||
|
.store(snapshot.usage_saves, Ordering::Relaxed);
|
||||||
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
|
||||||
snapshot
|
snapshot
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
|
pub fn finish_scan_cycle_work(&self, start: ScanCycleWorkSnapshot) {
|
||||||
let work = self.scan_cycle_work_since(start);
|
let work = self.scan_cycle_work_since(start);
|
||||||
self.record_scan_cycle_work(work.objects_scanned, work.directories_scanned, work.bucket_drive_scans);
|
self.record_scan_cycle_work(work);
|
||||||
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -701,6 +876,13 @@ impl Metrics {
|
|||||||
objects_scanned: self.lifetime(Metric::ScanObject),
|
objects_scanned: self.lifetime(Metric::ScanObject),
|
||||||
directories_scanned: self.lifetime(Metric::ScanFolder),
|
directories_scanned: self.lifetime(Metric::ScanFolder),
|
||||||
bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
|
bucket_drive_scans: self.lifetime(Metric::ScanBucketDrive),
|
||||||
|
bucket_drive_failures: self.lifetime(Metric::ScanBucketDriveFailure),
|
||||||
|
yield_events: self.lifetime(Metric::Yield),
|
||||||
|
yield_duration_millis: self.scanner_yield_duration_millis.load(Ordering::Relaxed),
|
||||||
|
ilm_actions: self.scanner_ilm_actions.load(Ordering::Relaxed),
|
||||||
|
heal_objects: self.lifetime(Metric::HealAbandonedObject),
|
||||||
|
replication_checks: self.lifetime(Metric::CheckReplication),
|
||||||
|
usage_saves: self.lifetime(Metric::SaveUsage),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -709,6 +891,13 @@ impl Metrics {
|
|||||||
objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
|
objects_scanned: self.current_scan_cycle_objects_start.load(Ordering::Relaxed),
|
||||||
directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
|
directories_scanned: self.current_scan_cycle_directories_start.load(Ordering::Relaxed),
|
||||||
bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
|
bucket_drive_scans: self.current_scan_cycle_bucket_drive_scans_start.load(Ordering::Relaxed),
|
||||||
|
bucket_drive_failures: self.current_scan_cycle_bucket_drive_failures_start.load(Ordering::Relaxed),
|
||||||
|
yield_events: self.current_scan_cycle_yield_events_start.load(Ordering::Relaxed),
|
||||||
|
yield_duration_millis: self.current_scan_cycle_yield_duration_millis_start.load(Ordering::Relaxed),
|
||||||
|
ilm_actions: self.current_scan_cycle_ilm_actions_start.load(Ordering::Relaxed),
|
||||||
|
heal_objects: self.current_scan_cycle_heal_objects_start.load(Ordering::Relaxed),
|
||||||
|
replication_checks: self.current_scan_cycle_replication_checks_start.load(Ordering::Relaxed),
|
||||||
|
usage_saves: self.current_scan_cycle_usage_saves_start.load(Ordering::Relaxed),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -718,17 +907,35 @@ impl Metrics {
|
|||||||
objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
|
objects_scanned: current.objects_scanned.saturating_sub(start.objects_scanned),
|
||||||
directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
|
directories_scanned: current.directories_scanned.saturating_sub(start.directories_scanned),
|
||||||
bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
|
bucket_drive_scans: current.bucket_drive_scans.saturating_sub(start.bucket_drive_scans),
|
||||||
|
bucket_drive_failures: current.bucket_drive_failures.saturating_sub(start.bucket_drive_failures),
|
||||||
|
yield_events: current.yield_events.saturating_sub(start.yield_events),
|
||||||
|
yield_duration_millis: current.yield_duration_millis.saturating_sub(start.yield_duration_millis),
|
||||||
|
ilm_actions: current.ilm_actions.saturating_sub(start.ilm_actions),
|
||||||
|
heal_objects: current.heal_objects.saturating_sub(start.heal_objects),
|
||||||
|
replication_checks: current.replication_checks.saturating_sub(start.replication_checks),
|
||||||
|
usage_saves: current.usage_saves.saturating_sub(start.usage_saves),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_scan_cycle_work(&self, objects_scanned: u64, directories_scanned: u64, bucket_drive_scans: u64) {
|
pub fn record_scan_cycle_work(&self, work: ScanCycleWorkSnapshot) {
|
||||||
// Telemetry-only gauges: readers may observe a transient mixed snapshot
|
// Telemetry-only gauges: readers may observe a transient mixed snapshot
|
||||||
// while these independent atomic fields are updated.
|
// while these independent atomic fields are updated.
|
||||||
self.last_scan_cycle_objects_scanned.store(objects_scanned, Ordering::Relaxed);
|
self.last_scan_cycle_objects_scanned
|
||||||
|
.store(work.objects_scanned, Ordering::Relaxed);
|
||||||
self.last_scan_cycle_directories_scanned
|
self.last_scan_cycle_directories_scanned
|
||||||
.store(directories_scanned, Ordering::Relaxed);
|
.store(work.directories_scanned, Ordering::Relaxed);
|
||||||
self.last_scan_cycle_bucket_drive_scans
|
self.last_scan_cycle_bucket_drive_scans
|
||||||
.store(bucket_drive_scans, Ordering::Relaxed);
|
.store(work.bucket_drive_scans, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_bucket_drive_failures
|
||||||
|
.store(work.bucket_drive_failures, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_yield_events.store(work.yield_events, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_yield_duration_millis
|
||||||
|
.store(work.yield_duration_millis, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_ilm_actions.store(work.ilm_actions, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_heal_objects.store(work.heal_objects, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_replication_checks
|
||||||
|
.store(work.replication_checks, Ordering::Relaxed);
|
||||||
|
self.last_scan_cycle_usage_saves.store(work.usage_saves, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Snapshot of every path currently being scanned.
|
/// Snapshot of every path currently being scanned.
|
||||||
@@ -772,6 +979,13 @@ impl Metrics {
|
|||||||
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
m.current_cycle_objects_scanned = current_work.objects_scanned;
|
||||||
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
m.current_cycle_directories_scanned = current_work.directories_scanned;
|
||||||
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
|
||||||
|
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
|
||||||
|
m.current_cycle_yield_events = current_work.yield_events;
|
||||||
|
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
|
||||||
|
m.current_cycle_ilm_actions = current_work.ilm_actions;
|
||||||
|
m.current_cycle_heal_objects = current_work.heal_objects;
|
||||||
|
m.current_cycle_replication_checks = current_work.replication_checks;
|
||||||
|
m.current_cycle_usage_saves = current_work.usage_saves;
|
||||||
}
|
}
|
||||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||||
@@ -780,7 +994,21 @@ impl Metrics {
|
|||||||
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
||||||
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
||||||
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_bucket_drive_failures = self.last_scan_cycle_bucket_drive_failures.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_yield_events = self.last_scan_cycle_yield_events.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_yield_duration_seconds = self.last_scan_cycle_yield_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||||
|
m.last_cycle_ilm_actions = self.last_scan_cycle_ilm_actions.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_heal_objects = self.last_scan_cycle_heal_objects.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_replication_checks = self.last_scan_cycle_replication_checks.load(Ordering::Relaxed);
|
||||||
|
m.last_cycle_usage_saves = self.last_scan_cycle_usage_saves.load(Ordering::Relaxed);
|
||||||
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
|
m.failed_cycles = self.failed_scan_cycles.load(Ordering::Relaxed);
|
||||||
|
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
|
||||||
|
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
|
||||||
|
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||||
|
m.yield_every_n_objects = self.scanner_yield_every_n_objects.load(Ordering::Relaxed);
|
||||||
|
m.cycle_interval_seconds = self.scanner_cycle_interval_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||||
|
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
|
||||||
|
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||||
|
|
||||||
// Lifetime operation counts
|
// Lifetime operation counts
|
||||||
for i in 0..Metric::Last as usize {
|
for i in 0..Metric::Last as usize {
|
||||||
@@ -998,13 +1226,43 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn report_includes_last_scan_cycle_work() {
|
async fn report_includes_last_scan_cycle_work() {
|
||||||
let metrics = Metrics::new();
|
let metrics = Metrics::new();
|
||||||
metrics.record_scan_cycle_work(11, 7, 3);
|
metrics.record_scan_cycle_work(ScanCycleWorkSnapshot {
|
||||||
|
objects_scanned: 11,
|
||||||
|
directories_scanned: 7,
|
||||||
|
bucket_drive_scans: 3,
|
||||||
|
bucket_drive_failures: 2,
|
||||||
|
yield_events: 5,
|
||||||
|
yield_duration_millis: 250,
|
||||||
|
ilm_actions: 13,
|
||||||
|
heal_objects: 2,
|
||||||
|
replication_checks: 4,
|
||||||
|
usage_saves: 6,
|
||||||
|
});
|
||||||
|
|
||||||
let report = metrics.report().await;
|
let report = metrics.report().await;
|
||||||
|
|
||||||
assert_eq!(report.last_cycle_objects_scanned, 11);
|
assert_eq!(report.last_cycle_objects_scanned, 11);
|
||||||
assert_eq!(report.last_cycle_directories_scanned, 7);
|
assert_eq!(report.last_cycle_directories_scanned, 7);
|
||||||
assert_eq!(report.last_cycle_bucket_drive_scans, 3);
|
assert_eq!(report.last_cycle_bucket_drive_scans, 3);
|
||||||
|
assert_eq!(report.last_cycle_bucket_drive_failures, 2);
|
||||||
|
assert_eq!(report.last_cycle_yield_events, 5);
|
||||||
|
assert_eq!(report.last_cycle_yield_duration_seconds, 0.25);
|
||||||
|
assert_eq!(report.last_cycle_ilm_actions, 13);
|
||||||
|
assert_eq!(report.last_cycle_heal_objects, 2);
|
||||||
|
assert_eq!(report.last_cycle_replication_checks, 4);
|
||||||
|
assert_eq!(report.last_cycle_usage_saves, 6);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn report_includes_bucket_drive_scan_starts() {
|
||||||
|
let metrics = Metrics::new();
|
||||||
|
metrics.record_scan_bucket_drive_start();
|
||||||
|
metrics.record_scan_bucket_drive_failure();
|
||||||
|
|
||||||
|
let report = metrics.report().await;
|
||||||
|
|
||||||
|
assert_eq!(report.life_time_ops.get("scan_bucket_drive_start"), Some(&1));
|
||||||
|
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1013,17 +1271,39 @@ mod tests {
|
|||||||
metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
|
metrics.operations[Metric::ScanObject as usize].store(10, Ordering::Relaxed);
|
||||||
metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
|
metrics.operations[Metric::ScanFolder as usize].store(5, Ordering::Relaxed);
|
||||||
metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
|
metrics.operations[Metric::ScanBucketDrive as usize].store(1, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(1, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::Yield as usize].store(2, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::HealAbandonedObject as usize].store(4, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::CheckReplication as usize].store(5, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::SaveUsage as usize].store(3, Ordering::Relaxed);
|
||||||
|
metrics.actions[IlmAction::DeleteAction as usize].store(100, Ordering::Relaxed);
|
||||||
|
metrics.record_scanner_ilm_action(6);
|
||||||
|
metrics.scanner_yield_duration_millis.store(100, Ordering::Relaxed);
|
||||||
|
|
||||||
let start = metrics.start_scan_cycle_work();
|
let start = metrics.start_scan_cycle_work();
|
||||||
metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
|
metrics.operations[Metric::ScanObject as usize].store(17, Ordering::Relaxed);
|
||||||
metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
|
metrics.operations[Metric::ScanFolder as usize].store(8, Ordering::Relaxed);
|
||||||
metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
|
metrics.operations[Metric::ScanBucketDrive as usize].store(3, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::ScanBucketDriveFailure as usize].store(4, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::HealAbandonedObject as usize].store(7, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::CheckReplication as usize].store(11, Ordering::Relaxed);
|
||||||
|
metrics.operations[Metric::SaveUsage as usize].store(5, Ordering::Relaxed);
|
||||||
|
metrics.actions[IlmAction::DeleteAction as usize].store(150, Ordering::Relaxed);
|
||||||
|
metrics.record_scanner_ilm_action(9);
|
||||||
|
metrics.record_scanner_yield(Duration::from_millis(150));
|
||||||
|
|
||||||
let report = metrics.report().await;
|
let report = metrics.report().await;
|
||||||
|
|
||||||
assert_eq!(report.current_cycle_objects_scanned, 7);
|
assert_eq!(report.current_cycle_objects_scanned, 7);
|
||||||
assert_eq!(report.current_cycle_directories_scanned, 3);
|
assert_eq!(report.current_cycle_directories_scanned, 3);
|
||||||
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
|
||||||
|
assert_eq!(report.current_cycle_bucket_drive_failures, 3);
|
||||||
|
assert_eq!(report.current_cycle_yield_events, 1);
|
||||||
|
assert_eq!(report.current_cycle_yield_duration_seconds, 0.15);
|
||||||
|
assert_eq!(report.current_cycle_ilm_actions, 9);
|
||||||
|
assert_eq!(report.current_cycle_heal_objects, 3);
|
||||||
|
assert_eq!(report.current_cycle_replication_checks, 6);
|
||||||
|
assert_eq!(report.current_cycle_usage_saves, 2);
|
||||||
|
|
||||||
metrics.finish_scan_cycle_work(start);
|
metrics.finish_scan_cycle_work(start);
|
||||||
let report = metrics.report().await;
|
let report = metrics.report().await;
|
||||||
@@ -1031,8 +1311,78 @@ mod tests {
|
|||||||
assert_eq!(report.current_cycle_objects_scanned, 0);
|
assert_eq!(report.current_cycle_objects_scanned, 0);
|
||||||
assert_eq!(report.current_cycle_directories_scanned, 0);
|
assert_eq!(report.current_cycle_directories_scanned, 0);
|
||||||
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
|
||||||
|
assert_eq!(report.current_cycle_bucket_drive_failures, 0);
|
||||||
|
assert_eq!(report.current_cycle_yield_events, 0);
|
||||||
|
assert_eq!(report.current_cycle_yield_duration_seconds, 0.0);
|
||||||
|
assert_eq!(report.current_cycle_ilm_actions, 0);
|
||||||
|
assert_eq!(report.current_cycle_heal_objects, 0);
|
||||||
|
assert_eq!(report.current_cycle_replication_checks, 0);
|
||||||
|
assert_eq!(report.current_cycle_usage_saves, 0);
|
||||||
assert_eq!(report.last_cycle_objects_scanned, 7);
|
assert_eq!(report.last_cycle_objects_scanned, 7);
|
||||||
assert_eq!(report.last_cycle_directories_scanned, 3);
|
assert_eq!(report.last_cycle_directories_scanned, 3);
|
||||||
assert_eq!(report.last_cycle_bucket_drive_scans, 2);
|
assert_eq!(report.last_cycle_bucket_drive_scans, 2);
|
||||||
|
assert_eq!(report.last_cycle_bucket_drive_failures, 3);
|
||||||
|
assert_eq!(report.last_cycle_yield_events, 1);
|
||||||
|
assert_eq!(report.last_cycle_yield_duration_seconds, 0.15);
|
||||||
|
assert_eq!(report.last_cycle_ilm_actions, 9);
|
||||||
|
assert_eq!(report.last_cycle_heal_objects, 3);
|
||||||
|
assert_eq!(report.last_cycle_replication_checks, 6);
|
||||||
|
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn record_scanner_yield_tracks_count_and_duration() {
|
||||||
|
let metrics = Metrics::new();
|
||||||
|
|
||||||
|
metrics.record_scanner_yield(Duration::from_millis(42));
|
||||||
|
|
||||||
|
assert_eq!(metrics.lifetime(Metric::Yield), 1);
|
||||||
|
assert_eq!(metrics.scanner_yield_duration_millis.load(Ordering::Relaxed), 42);
|
||||||
|
assert_eq!(metrics.last_minute(Metric::Yield).n, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn report_includes_scanner_throttle_config() {
|
||||||
|
let metrics = Metrics::new();
|
||||||
|
|
||||||
|
metrics.record_scanner_throttle_config(true, 2.5, Duration::from_millis(1500), 128);
|
||||||
|
let report = metrics.report().await;
|
||||||
|
|
||||||
|
assert!(report.throttle_idle_mode_enabled);
|
||||||
|
assert_eq!(report.throttle_sleep_factor, 2.5);
|
||||||
|
assert_eq!(report.throttle_max_sleep_seconds, 1.5);
|
||||||
|
assert_eq!(report.yield_every_n_objects, 128);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scanner_throttle_config_rejects_invalid_sleep_factor() {
|
||||||
|
let metrics = Metrics::new();
|
||||||
|
|
||||||
|
metrics.record_scanner_throttle_config(false, f64::NAN, Duration::ZERO, 0);
|
||||||
|
let report = metrics.report().await;
|
||||||
|
|
||||||
|
assert!(!report.throttle_idle_mode_enabled);
|
||||||
|
assert_eq!(report.throttle_sleep_factor, 0.0);
|
||||||
|
assert_eq!(report.throttle_max_sleep_seconds, 0.0);
|
||||||
|
assert_eq!(report.yield_every_n_objects, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn report_includes_scanner_cycle_config() {
|
||||||
|
let metrics = Metrics::new();
|
||||||
|
|
||||||
|
metrics.record_scanner_cycle_config(Duration::from_secs(3600), Some(Duration::from_secs(86400)));
|
||||||
|
let report = metrics.report().await;
|
||||||
|
|
||||||
|
assert_eq!(report.cycle_interval_seconds, 3600.0);
|
||||||
|
assert!(report.bitrot_cycle_enabled);
|
||||||
|
assert_eq!(report.bitrot_cycle_seconds, 86400.0);
|
||||||
|
|
||||||
|
metrics.record_scanner_cycle_config(Duration::from_secs(60), None);
|
||||||
|
let report = metrics.report().await;
|
||||||
|
|
||||||
|
assert_eq!(report.cycle_interval_seconds, 60.0);
|
||||||
|
assert!(!report.bitrot_cycle_enabled);
|
||||||
|
assert_eq!(report.bitrot_cycle_seconds, 0.0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,29 +16,40 @@
|
|||||||
|
|
||||||
//! Scanner metrics collector.
|
//! Scanner metrics collector.
|
||||||
//!
|
//!
|
||||||
//! Collects background scanner metrics including bucket scans,
|
//! Collects background scanner metrics including bucket-drive scans,
|
||||||
//! directory scans, and object scans.
|
//! directory scans, and object scans.
|
||||||
|
|
||||||
use crate::metrics::report::PrometheusMetric;
|
use crate::metrics::report::PrometheusMetric;
|
||||||
use crate::metrics::schema::scanner::{
|
use crate::metrics::schema::scanner::{
|
||||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
|
||||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD,
|
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||||
SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
|
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
|
||||||
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
|
||||||
|
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
|
||||||
|
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
|
||||||
|
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD,
|
||||||
|
SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD,
|
||||||
|
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD,
|
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
|
||||||
SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
|
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
|
||||||
|
SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD,
|
||||||
|
SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD,
|
||||||
|
SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD,
|
||||||
|
SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Scanner statistics.
|
/// Scanner statistics.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct ScannerStats {
|
pub struct ScannerStats {
|
||||||
/// Number of bucket scans finished
|
/// Number of bucket-drive scans finished
|
||||||
pub bucket_scans_finished: u64,
|
pub bucket_scans_finished: u64,
|
||||||
/// Number of bucket scans started
|
/// Number of bucket-drive scans started
|
||||||
pub bucket_scans_started: u64,
|
pub bucket_scans_started: u64,
|
||||||
|
/// Number of bucket-drive scans that failed
|
||||||
|
pub bucket_scans_failed: u64,
|
||||||
/// Number of directories scanned
|
/// Number of directories scanned
|
||||||
pub directories_scanned: u64,
|
pub directories_scanned: u64,
|
||||||
/// Number of objects scanned
|
/// Number of objects scanned
|
||||||
@@ -49,6 +60,20 @@ pub struct ScannerStats {
|
|||||||
pub last_activity_seconds: u64,
|
pub last_activity_seconds: u64,
|
||||||
/// Number of scanner paths currently being processed
|
/// Number of scanner paths currently being processed
|
||||||
pub active_paths: u64,
|
pub active_paths: u64,
|
||||||
|
/// Whether scanner idle-mode self-throttling is enabled
|
||||||
|
pub throttle_idle_mode_enabled: u64,
|
||||||
|
/// Effective scanner sleep factor
|
||||||
|
pub throttle_sleep_factor: f64,
|
||||||
|
/// Effective scanner maximum self-throttle sleep duration in seconds
|
||||||
|
pub throttle_max_sleep_seconds: f64,
|
||||||
|
/// Object interval for cooperative scanner runtime yields
|
||||||
|
pub yield_every_n_objects: u64,
|
||||||
|
/// Effective scanner cycle interval in seconds
|
||||||
|
pub cycle_interval_seconds: f64,
|
||||||
|
/// Whether periodic scanner bitrot deep scans are enabled
|
||||||
|
pub bitrot_cycle_enabled: u64,
|
||||||
|
/// Effective scanner bitrot deep-scan interval in seconds
|
||||||
|
pub bitrot_cycle_seconds: f64,
|
||||||
/// Current scanner cycle number, or zero when idle
|
/// Current scanner cycle number, or zero when idle
|
||||||
pub current_cycle: u64,
|
pub current_cycle: u64,
|
||||||
/// Number of scanner cycles completed since server start
|
/// Number of scanner cycles completed since server start
|
||||||
@@ -61,12 +86,26 @@ pub struct ScannerStats {
|
|||||||
pub current_cycle_directories_scanned: u64,
|
pub current_cycle_directories_scanned: u64,
|
||||||
/// Number of bucket-drive scans finished by the currently running scanner cycle
|
/// Number of bucket-drive scans finished by the currently running scanner cycle
|
||||||
pub current_cycle_bucket_drive_scans: u64,
|
pub current_cycle_bucket_drive_scans: u64,
|
||||||
|
/// Number of bucket-drive scans that failed in the currently running scanner cycle
|
||||||
|
pub current_cycle_bucket_drive_failures: u64,
|
||||||
/// Object scan rate for the currently running scanner cycle
|
/// Object scan rate for the currently running scanner cycle
|
||||||
pub current_cycle_objects_per_second: f64,
|
pub current_cycle_objects_per_second: f64,
|
||||||
/// Directory scan rate for the currently running scanner cycle
|
/// Directory scan rate for the currently running scanner cycle
|
||||||
pub current_cycle_directories_per_second: f64,
|
pub current_cycle_directories_per_second: f64,
|
||||||
/// Bucket-drive scan rate for the currently running scanner cycle
|
/// Bucket-drive scan rate for the currently running scanner cycle
|
||||||
pub current_cycle_bucket_drive_scans_per_second: f64,
|
pub current_cycle_bucket_drive_scans_per_second: f64,
|
||||||
|
/// Number of scanner self-throttle yield events in the current scanner cycle
|
||||||
|
pub current_cycle_yield_events: u64,
|
||||||
|
/// Total scanner self-throttle yield duration in seconds for the current scanner cycle
|
||||||
|
pub current_cycle_yield_duration_seconds: f64,
|
||||||
|
/// Number of lifecycle actions applied by the current scanner cycle
|
||||||
|
pub current_cycle_ilm_actions: u64,
|
||||||
|
/// Number of object heal candidates enqueued by the current scanner cycle
|
||||||
|
pub current_cycle_heal_objects: u64,
|
||||||
|
/// Number of replication heal checks run by the current scanner cycle
|
||||||
|
pub current_cycle_replication_checks: u64,
|
||||||
|
/// Number of data-usage save operations run by the current scanner cycle
|
||||||
|
pub current_cycle_usage_saves: u64,
|
||||||
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
|
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
|
||||||
pub current_scan_mode: u64,
|
pub current_scan_mode: u64,
|
||||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error
|
/// Last scanner cycle result: 0 unknown, 1 success, 2 error
|
||||||
@@ -79,12 +118,26 @@ pub struct ScannerStats {
|
|||||||
pub last_cycle_directories_scanned: u64,
|
pub last_cycle_directories_scanned: u64,
|
||||||
/// Number of bucket-drive scans finished by the last scanner cycle
|
/// Number of bucket-drive scans finished by the last scanner cycle
|
||||||
pub last_cycle_bucket_drive_scans: u64,
|
pub last_cycle_bucket_drive_scans: u64,
|
||||||
|
/// Number of bucket-drive scans that failed in the last finished scanner cycle
|
||||||
|
pub last_cycle_bucket_drive_failures: u64,
|
||||||
/// Object scan rate for the last finished scanner cycle
|
/// Object scan rate for the last finished scanner cycle
|
||||||
pub last_cycle_objects_per_second: f64,
|
pub last_cycle_objects_per_second: f64,
|
||||||
/// Directory scan rate for the last finished scanner cycle
|
/// Directory scan rate for the last finished scanner cycle
|
||||||
pub last_cycle_directories_per_second: f64,
|
pub last_cycle_directories_per_second: f64,
|
||||||
/// Bucket-drive scan rate for the last finished scanner cycle
|
/// Bucket-drive scan rate for the last finished scanner cycle
|
||||||
pub last_cycle_bucket_drive_scans_per_second: f64,
|
pub last_cycle_bucket_drive_scans_per_second: f64,
|
||||||
|
/// Number of scanner self-throttle yield events in the last finished scanner cycle
|
||||||
|
pub last_cycle_yield_events: u64,
|
||||||
|
/// Total scanner self-throttle yield duration in seconds for the last finished scanner cycle
|
||||||
|
pub last_cycle_yield_duration_seconds: f64,
|
||||||
|
/// Number of lifecycle actions applied by the last finished scanner cycle
|
||||||
|
pub last_cycle_ilm_actions: u64,
|
||||||
|
/// Number of object heal candidates enqueued by the last finished scanner cycle
|
||||||
|
pub last_cycle_heal_objects: u64,
|
||||||
|
/// Number of replication heal checks run by the last finished scanner cycle
|
||||||
|
pub last_cycle_replication_checks: u64,
|
||||||
|
/// Number of data-usage save operations run by the last finished scanner cycle
|
||||||
|
pub last_cycle_usage_saves: u64,
|
||||||
/// Number of scanner cycles that failed since server start
|
/// Number of scanner cycles that failed since server start
|
||||||
pub failed_cycles: u64,
|
pub failed_cycles: u64,
|
||||||
}
|
}
|
||||||
@@ -97,11 +150,19 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
|||||||
vec![
|
vec![
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, stats.throttle_idle_mode_enabled as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_SLEEP_FACTOR_MD, stats.throttle_sleep_factor),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, stats.throttle_max_sleep_seconds),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, stats.bitrot_cycle_enabled as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
|
||||||
@@ -114,6 +175,10 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
|||||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||||
stats.current_cycle_bucket_drive_scans as f64,
|
stats.current_cycle_bucket_drive_scans as f64,
|
||||||
),
|
),
|
||||||
|
PrometheusMetric::from_descriptor(
|
||||||
|
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||||
|
stats.current_cycle_bucket_drive_failures as f64,
|
||||||
|
),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
|
||||||
PrometheusMetric::from_descriptor(
|
PrometheusMetric::from_descriptor(
|
||||||
&SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
&SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||||
@@ -123,6 +188,18 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
|||||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||||
stats.current_cycle_bucket_drive_scans_per_second,
|
stats.current_cycle_bucket_drive_scans_per_second,
|
||||||
),
|
),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, stats.current_cycle_yield_events as f64),
|
||||||
|
PrometheusMetric::from_descriptor(
|
||||||
|
&SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD,
|
||||||
|
stats.current_cycle_yield_duration_seconds,
|
||||||
|
),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, stats.current_cycle_ilm_actions as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, stats.current_cycle_heal_objects as f64),
|
||||||
|
PrometheusMetric::from_descriptor(
|
||||||
|
&SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD,
|
||||||
|
stats.current_cycle_replication_checks as f64,
|
||||||
|
),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
|
||||||
@@ -132,12 +209,22 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
|||||||
stats.last_cycle_directories_scanned as f64,
|
stats.last_cycle_directories_scanned as f64,
|
||||||
),
|
),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
|
||||||
|
PrometheusMetric::from_descriptor(
|
||||||
|
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||||
|
stats.last_cycle_bucket_drive_failures as f64,
|
||||||
|
),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
|
||||||
PrometheusMetric::from_descriptor(
|
PrometheusMetric::from_descriptor(
|
||||||
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||||
stats.last_cycle_bucket_drive_scans_per_second,
|
stats.last_cycle_bucket_drive_scans_per_second,
|
||||||
),
|
),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, stats.last_cycle_yield_events as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, stats.last_cycle_yield_duration_seconds),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, stats.last_cycle_ilm_actions as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, stats.last_cycle_heal_objects as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
|
||||||
|
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
|
||||||
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
|
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -152,36 +239,58 @@ mod tests {
|
|||||||
let stats = ScannerStats {
|
let stats = ScannerStats {
|
||||||
bucket_scans_finished: 100,
|
bucket_scans_finished: 100,
|
||||||
bucket_scans_started: 100,
|
bucket_scans_started: 100,
|
||||||
|
bucket_scans_failed: 2,
|
||||||
directories_scanned: 50000,
|
directories_scanned: 50000,
|
||||||
objects_scanned: 1000000,
|
objects_scanned: 1000000,
|
||||||
versions_scanned: 1500000,
|
versions_scanned: 1500000,
|
||||||
last_activity_seconds: 30,
|
last_activity_seconds: 30,
|
||||||
active_paths: 4,
|
active_paths: 4,
|
||||||
|
throttle_idle_mode_enabled: 1,
|
||||||
|
throttle_sleep_factor: 10.0,
|
||||||
|
throttle_max_sleep_seconds: 15.0,
|
||||||
|
yield_every_n_objects: 128,
|
||||||
|
cycle_interval_seconds: 3600.0,
|
||||||
|
bitrot_cycle_enabled: 1,
|
||||||
|
bitrot_cycle_seconds: 86400.0,
|
||||||
current_cycle: 12,
|
current_cycle: 12,
|
||||||
completed_cycles: 11,
|
completed_cycles: 11,
|
||||||
current_cycle_age_seconds: 90,
|
current_cycle_age_seconds: 90,
|
||||||
current_cycle_objects_scanned: 250,
|
current_cycle_objects_scanned: 250,
|
||||||
current_cycle_directories_scanned: 20,
|
current_cycle_directories_scanned: 20,
|
||||||
current_cycle_bucket_drive_scans: 2,
|
current_cycle_bucket_drive_scans: 2,
|
||||||
|
current_cycle_bucket_drive_failures: 1,
|
||||||
current_cycle_objects_per_second: 12.5,
|
current_cycle_objects_per_second: 12.5,
|
||||||
current_cycle_directories_per_second: 1.0,
|
current_cycle_directories_per_second: 1.0,
|
||||||
current_cycle_bucket_drive_scans_per_second: 0.1,
|
current_cycle_bucket_drive_scans_per_second: 0.1,
|
||||||
|
current_cycle_yield_events: 8,
|
||||||
|
current_cycle_yield_duration_seconds: 1.25,
|
||||||
|
current_cycle_ilm_actions: 6,
|
||||||
|
current_cycle_heal_objects: 2,
|
||||||
|
current_cycle_replication_checks: 5,
|
||||||
|
current_cycle_usage_saves: 3,
|
||||||
current_scan_mode: 2,
|
current_scan_mode: 2,
|
||||||
last_cycle_result: 1,
|
last_cycle_result: 1,
|
||||||
last_cycle_duration_seconds: 42.5,
|
last_cycle_duration_seconds: 42.5,
|
||||||
last_cycle_objects_scanned: 900,
|
last_cycle_objects_scanned: 900,
|
||||||
last_cycle_directories_scanned: 80,
|
last_cycle_directories_scanned: 80,
|
||||||
last_cycle_bucket_drive_scans: 6,
|
last_cycle_bucket_drive_scans: 6,
|
||||||
|
last_cycle_bucket_drive_failures: 2,
|
||||||
last_cycle_objects_per_second: 18.0,
|
last_cycle_objects_per_second: 18.0,
|
||||||
last_cycle_directories_per_second: 1.6,
|
last_cycle_directories_per_second: 1.6,
|
||||||
last_cycle_bucket_drive_scans_per_second: 0.12,
|
last_cycle_bucket_drive_scans_per_second: 0.12,
|
||||||
|
last_cycle_yield_events: 30,
|
||||||
|
last_cycle_yield_duration_seconds: 9.5,
|
||||||
|
last_cycle_ilm_actions: 44,
|
||||||
|
last_cycle_heal_objects: 7,
|
||||||
|
last_cycle_replication_checks: 12,
|
||||||
|
last_cycle_usage_saves: 9,
|
||||||
failed_cycles: 3,
|
failed_cycles: 3,
|
||||||
};
|
};
|
||||||
|
|
||||||
let metrics = collect_scanner_metrics(&stats);
|
let metrics = collect_scanner_metrics(&stats);
|
||||||
report_metrics(&metrics);
|
report_metrics(&metrics);
|
||||||
|
|
||||||
assert_eq!(metrics.len(), 26);
|
assert_eq!(metrics.len(), 48);
|
||||||
|
|
||||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||||
assert!(objects.is_some());
|
assert!(objects.is_some());
|
||||||
@@ -194,6 +303,46 @@ mod tests {
|
|||||||
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
||||||
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
||||||
|
|
||||||
|
let bucket_scans_failed = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name());
|
||||||
|
assert_eq!(bucket_scans_failed.map(|m| m.value), Some(2.0));
|
||||||
|
|
||||||
|
let throttle_idle_mode_enabled = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD.get_full_metric_name());
|
||||||
|
assert_eq!(throttle_idle_mode_enabled.map(|m| m.value), Some(1.0));
|
||||||
|
|
||||||
|
let throttle_sleep_factor = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_THROTTLE_SLEEP_FACTOR_MD.get_full_metric_name());
|
||||||
|
assert_eq!(throttle_sleep_factor.map(|m| m.value), Some(10.0));
|
||||||
|
|
||||||
|
let throttle_max_sleep = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(throttle_max_sleep.map(|m| m.value), Some(15.0));
|
||||||
|
|
||||||
|
let yield_every_n_objects = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_YIELD_EVERY_N_OBJECTS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(yield_every_n_objects.map(|m| m.value), Some(128.0));
|
||||||
|
|
||||||
|
let cycle_interval_seconds = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CYCLE_INTERVAL_SECONDS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(cycle_interval_seconds.map(|m| m.value), Some(3600.0));
|
||||||
|
|
||||||
|
let bitrot_cycle_enabled = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name());
|
||||||
|
assert_eq!(bitrot_cycle_enabled.map(|m| m.value), Some(1.0));
|
||||||
|
|
||||||
|
let bitrot_cycle_seconds = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_BITROT_CYCLE_SECONDS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(bitrot_cycle_seconds.map(|m| m.value), Some(86400.0));
|
||||||
|
|
||||||
let current_cycle = metrics
|
let current_cycle = metrics
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
|
||||||
@@ -224,6 +373,11 @@ mod tests {
|
|||||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||||
assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
|
assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
|
||||||
|
|
||||||
|
let current_cycle_bucket_drive_failures = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_bucket_drive_failures.map(|m| m.value), Some(1.0));
|
||||||
|
|
||||||
let current_cycle_objects_rate = metrics
|
let current_cycle_objects_rate = metrics
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||||
@@ -239,6 +393,36 @@ mod tests {
|
|||||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||||
assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
|
assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
|
||||||
|
|
||||||
|
let current_cycle_yield_events = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_yield_events.map(|m| m.value), Some(8.0));
|
||||||
|
|
||||||
|
let current_cycle_yield_duration = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_yield_duration.map(|m| m.value), Some(1.25));
|
||||||
|
|
||||||
|
let current_cycle_ilm_actions = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_ilm_actions.map(|m| m.value), Some(6.0));
|
||||||
|
|
||||||
|
let current_cycle_heal_objects = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_heal_objects.map(|m| m.value), Some(2.0));
|
||||||
|
|
||||||
|
let current_cycle_replication_checks = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_replication_checks.map(|m| m.value), Some(5.0));
|
||||||
|
|
||||||
|
let current_cycle_usage_saves = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||||
|
assert_eq!(current_cycle_usage_saves.map(|m| m.value), Some(3.0));
|
||||||
|
|
||||||
let current_scan_mode = metrics
|
let current_scan_mode = metrics
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
|
||||||
@@ -269,6 +453,11 @@ mod tests {
|
|||||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||||
assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
|
assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
|
||||||
|
|
||||||
|
let last_cycle_bucket_drive_failures = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_bucket_drive_failures.map(|m| m.value), Some(2.0));
|
||||||
|
|
||||||
let last_cycle_objects_rate = metrics
|
let last_cycle_objects_rate = metrics
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||||
@@ -284,6 +473,36 @@ mod tests {
|
|||||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||||
assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
|
assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
|
||||||
|
|
||||||
|
let last_cycle_yield_events = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_yield_events.map(|m| m.value), Some(30.0));
|
||||||
|
|
||||||
|
let last_cycle_yield_duration = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_yield_duration.map(|m| m.value), Some(9.5));
|
||||||
|
|
||||||
|
let last_cycle_ilm_actions = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_ilm_actions.map(|m| m.value), Some(44.0));
|
||||||
|
|
||||||
|
let last_cycle_heal_objects = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_heal_objects.map(|m| m.value), Some(7.0));
|
||||||
|
|
||||||
|
let last_cycle_replication_checks = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_replication_checks.map(|m| m.value), Some(12.0));
|
||||||
|
|
||||||
|
let last_cycle_usage_saves = metrics
|
||||||
|
.iter()
|
||||||
|
.find(|m| m.name == SCANNER_LAST_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||||
|
assert_eq!(last_cycle_usage_saves.map(|m| m.value), Some(9.0));
|
||||||
|
|
||||||
let failed_cycles = metrics
|
let failed_cycles = metrics
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
|
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
|
||||||
@@ -295,7 +514,7 @@ mod tests {
|
|||||||
let stats = ScannerStats::default();
|
let stats = ScannerStats::default();
|
||||||
let metrics = collect_scanner_metrics(&stats);
|
let metrics = collect_scanner_metrics(&stats);
|
||||||
|
|
||||||
assert_eq!(metrics.len(), 26);
|
assert_eq!(metrics.len(), 48);
|
||||||
for metric in &metrics {
|
for metric in &metrics {
|
||||||
assert_eq!(metric.value, 0.0);
|
assert_eq!(metric.value, 0.0);
|
||||||
assert!(metric.labels.is_empty());
|
assert!(metric.labels.is_empty());
|
||||||
|
|||||||
@@ -279,24 +279,46 @@ pub enum MetricName {
|
|||||||
ScannerVersionsScanned,
|
ScannerVersionsScanned,
|
||||||
ScannerLastActivitySeconds,
|
ScannerLastActivitySeconds,
|
||||||
ScannerActivePaths,
|
ScannerActivePaths,
|
||||||
|
ScannerBucketScansFailed,
|
||||||
|
ScannerThrottleIdleModeEnabled,
|
||||||
|
ScannerThrottleSleepFactor,
|
||||||
|
ScannerThrottleMaxSleepSeconds,
|
||||||
|
ScannerYieldEveryNObjects,
|
||||||
|
ScannerCycleIntervalSeconds,
|
||||||
|
ScannerBitrotCycleEnabled,
|
||||||
|
ScannerBitrotCycleSeconds,
|
||||||
ScannerCurrentCycle,
|
ScannerCurrentCycle,
|
||||||
ScannerCompletedCycles,
|
ScannerCompletedCycles,
|
||||||
ScannerCurrentCycleAgeSeconds,
|
ScannerCurrentCycleAgeSeconds,
|
||||||
ScannerCurrentCycleObjectsScanned,
|
ScannerCurrentCycleObjectsScanned,
|
||||||
ScannerCurrentCycleDirectoriesScanned,
|
ScannerCurrentCycleDirectoriesScanned,
|
||||||
ScannerCurrentCycleBucketDriveScans,
|
ScannerCurrentCycleBucketDriveScans,
|
||||||
|
ScannerCurrentCycleBucketDriveFailures,
|
||||||
ScannerCurrentCycleObjectsPerSecond,
|
ScannerCurrentCycleObjectsPerSecond,
|
||||||
ScannerCurrentCycleDirectoriesPerSecond,
|
ScannerCurrentCycleDirectoriesPerSecond,
|
||||||
ScannerCurrentCycleBucketDriveScansPerSecond,
|
ScannerCurrentCycleBucketDriveScansPerSecond,
|
||||||
|
ScannerCurrentCycleYieldEvents,
|
||||||
|
ScannerCurrentCycleYieldDurationSeconds,
|
||||||
|
ScannerCurrentCycleIlmActions,
|
||||||
|
ScannerCurrentCycleHealObjects,
|
||||||
|
ScannerCurrentCycleReplicationChecks,
|
||||||
|
ScannerCurrentCycleUsageSaves,
|
||||||
ScannerCurrentScanMode,
|
ScannerCurrentScanMode,
|
||||||
ScannerLastCycleResult,
|
ScannerLastCycleResult,
|
||||||
ScannerLastCycleDurationSeconds,
|
ScannerLastCycleDurationSeconds,
|
||||||
ScannerLastCycleObjectsScanned,
|
ScannerLastCycleObjectsScanned,
|
||||||
ScannerLastCycleDirectoriesScanned,
|
ScannerLastCycleDirectoriesScanned,
|
||||||
ScannerLastCycleBucketDriveScans,
|
ScannerLastCycleBucketDriveScans,
|
||||||
|
ScannerLastCycleBucketDriveFailures,
|
||||||
ScannerLastCycleObjectsPerSecond,
|
ScannerLastCycleObjectsPerSecond,
|
||||||
ScannerLastCycleDirectoriesPerSecond,
|
ScannerLastCycleDirectoriesPerSecond,
|
||||||
ScannerLastCycleBucketDriveScansPerSecond,
|
ScannerLastCycleBucketDriveScansPerSecond,
|
||||||
|
ScannerLastCycleYieldEvents,
|
||||||
|
ScannerLastCycleYieldDurationSeconds,
|
||||||
|
ScannerLastCycleIlmActions,
|
||||||
|
ScannerLastCycleHealObjects,
|
||||||
|
ScannerLastCycleReplicationChecks,
|
||||||
|
ScannerLastCycleUsageSaves,
|
||||||
ScannerFailedCycles,
|
ScannerFailedCycles,
|
||||||
|
|
||||||
// CPU system-related metrics
|
// CPU system-related metrics
|
||||||
@@ -639,24 +661,46 @@ impl MetricName {
|
|||||||
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
|
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
|
||||||
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
|
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
|
||||||
Self::ScannerActivePaths => "active_paths".to_string(),
|
Self::ScannerActivePaths => "active_paths".to_string(),
|
||||||
|
Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(),
|
||||||
|
Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(),
|
||||||
|
Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(),
|
||||||
|
Self::ScannerThrottleMaxSleepSeconds => "throttle_max_sleep_seconds".to_string(),
|
||||||
|
Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(),
|
||||||
|
Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(),
|
||||||
|
Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(),
|
||||||
|
Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(),
|
||||||
Self::ScannerCurrentCycle => "current_cycle".to_string(),
|
Self::ScannerCurrentCycle => "current_cycle".to_string(),
|
||||||
Self::ScannerCompletedCycles => "completed_cycles".to_string(),
|
Self::ScannerCompletedCycles => "completed_cycles".to_string(),
|
||||||
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
|
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
|
||||||
Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
|
Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
|
||||||
Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
|
Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
|
||||||
Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
|
Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
|
||||||
|
Self::ScannerCurrentCycleBucketDriveFailures => "current_cycle_bucket_drive_failures".to_string(),
|
||||||
Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
|
Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
|
||||||
Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
|
Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
|
||||||
Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
|
Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
|
||||||
|
Self::ScannerCurrentCycleYieldEvents => "current_cycle_yield_events".to_string(),
|
||||||
|
Self::ScannerCurrentCycleYieldDurationSeconds => "current_cycle_yield_duration_seconds".to_string(),
|
||||||
|
Self::ScannerCurrentCycleIlmActions => "current_cycle_ilm_actions".to_string(),
|
||||||
|
Self::ScannerCurrentCycleHealObjects => "current_cycle_heal_objects".to_string(),
|
||||||
|
Self::ScannerCurrentCycleReplicationChecks => "current_cycle_replication_checks".to_string(),
|
||||||
|
Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(),
|
||||||
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
|
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
|
||||||
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
|
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
|
||||||
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
|
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
|
||||||
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
|
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
|
||||||
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
|
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
|
||||||
Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
|
Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
|
||||||
|
Self::ScannerLastCycleBucketDriveFailures => "last_cycle_bucket_drive_failures".to_string(),
|
||||||
Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
|
Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
|
||||||
Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
|
Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
|
||||||
Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
|
Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
|
||||||
|
Self::ScannerLastCycleYieldEvents => "last_cycle_yield_events".to_string(),
|
||||||
|
Self::ScannerLastCycleYieldDurationSeconds => "last_cycle_yield_duration_seconds".to_string(),
|
||||||
|
Self::ScannerLastCycleIlmActions => "last_cycle_ilm_actions".to_string(),
|
||||||
|
Self::ScannerLastCycleHealObjects => "last_cycle_heal_objects".to_string(),
|
||||||
|
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
|
||||||
|
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
|
||||||
Self::ScannerFailedCycles => "failed_cycles".to_string(),
|
Self::ScannerFailedCycles => "failed_cycles".to_string(),
|
||||||
|
|
||||||
// CPU system-related metrics
|
// CPU system-related metrics
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use std::sync::LazyLock;
|
|||||||
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_counter_md(
|
new_counter_md(
|
||||||
MetricName::ScannerBucketScansFinished,
|
MetricName::ScannerBucketScansFinished,
|
||||||
"Total number of bucket scans finished since server start",
|
"Total number of bucket-drive scans finished since server start",
|
||||||
&[],
|
&[],
|
||||||
subsystems::SCANNER,
|
subsystems::SCANNER,
|
||||||
)
|
)
|
||||||
@@ -29,7 +29,16 @@ pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLo
|
|||||||
pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_counter_md(
|
new_counter_md(
|
||||||
MetricName::ScannerBucketScansStarted,
|
MetricName::ScannerBucketScansStarted,
|
||||||
"Total number of bucket scans started since server start",
|
"Total number of bucket-drive scans started since server start",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_BUCKET_SCANS_FAILED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_counter_md(
|
||||||
|
MetricName::ScannerBucketScansFailed,
|
||||||
|
"Total number of bucket-drive scans that failed since server start",
|
||||||
&[],
|
&[],
|
||||||
subsystems::SCANNER,
|
subsystems::SCANNER,
|
||||||
)
|
)
|
||||||
@@ -80,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
|
|||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerThrottleIdleModeEnabled,
|
||||||
|
"Whether scanner idle-mode self-throttling is enabled: 1 enabled, 0 disabled.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_THROTTLE_SLEEP_FACTOR_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerThrottleSleepFactor,
|
||||||
|
"Effective scanner sleep factor used to compute proportional self-throttle sleeps.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerThrottleMaxSleepSeconds,
|
||||||
|
"Effective maximum scanner self-throttle sleep duration in seconds.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_YIELD_EVERY_N_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerYieldEveryNObjects,
|
||||||
|
"Current object interval for cooperative scanner runtime yields, or zero when disabled.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CYCLE_INTERVAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCycleIntervalSeconds,
|
||||||
|
"Effective scanner cycle interval in seconds.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerBitrotCycleEnabled,
|
||||||
|
"Whether periodic scanner bitrot deep scans are enabled: 1 enabled, 0 disabled.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_BITROT_CYCLE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerBitrotCycleSeconds,
|
||||||
|
"Effective scanner bitrot deep-scan interval in seconds; zero means disabled or deep-scan every cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_gauge_md(
|
new_gauge_md(
|
||||||
MetricName::ScannerCurrentCycle,
|
MetricName::ScannerCurrentCycle,
|
||||||
@@ -134,6 +206,15 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescripto
|
|||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleBucketDriveFailures,
|
||||||
|
"Number of bucket-drive scans that failed in the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_gauge_md(
|
new_gauge_md(
|
||||||
MetricName::ScannerCurrentCycleObjectsPerSecond,
|
MetricName::ScannerCurrentCycleObjectsPerSecond,
|
||||||
@@ -161,6 +242,60 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<Metr
|
|||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleYieldEvents,
|
||||||
|
"Number of scanner self-throttle yield events in the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleYieldDurationSeconds,
|
||||||
|
"Total scanner self-throttle yield duration in seconds for the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleIlmActions,
|
||||||
|
"Number of lifecycle actions applied by the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleHealObjects,
|
||||||
|
"Number of object heal candidates enqueued by the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleReplicationChecks,
|
||||||
|
"Number of replication heal checks run by the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerCurrentCycleUsageSaves,
|
||||||
|
"Number of data-usage save operations run by the currently running scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_gauge_md(
|
new_gauge_md(
|
||||||
MetricName::ScannerCurrentScanMode,
|
MetricName::ScannerCurrentScanMode,
|
||||||
@@ -215,6 +350,15 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor>
|
|||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleBucketDriveFailures,
|
||||||
|
"Number of bucket-drive scans that failed in the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_gauge_md(
|
new_gauge_md(
|
||||||
MetricName::ScannerLastCycleObjectsPerSecond,
|
MetricName::ScannerLastCycleObjectsPerSecond,
|
||||||
@@ -242,6 +386,60 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricD
|
|||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleYieldEvents,
|
||||||
|
"Number of scanner self-throttle yield events in the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleYieldDurationSeconds,
|
||||||
|
"Total scanner self-throttle yield duration in seconds for the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleIlmActions,
|
||||||
|
"Number of lifecycle actions applied by the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleHealObjects,
|
||||||
|
"Number of object heal candidates enqueued by the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleReplicationChecks,
|
||||||
|
"Number of replication heal checks run by the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
pub static SCANNER_LAST_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
|
new_gauge_md(
|
||||||
|
MetricName::ScannerLastCycleUsageSaves,
|
||||||
|
"Number of data-usage save operations run by the last finished scanner cycle.",
|
||||||
|
&[],
|
||||||
|
subsystems::SCANNER,
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||||
new_counter_md(
|
new_counter_md(
|
||||||
MetricName::ScannerFailedCycles,
|
MetricName::ScannerFailedCycles,
|
||||||
|
|||||||
@@ -896,10 +896,23 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
|
|||||||
///
|
///
|
||||||
/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
|
/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
|
||||||
/// rustfs-obs scanner collector shape.
|
/// rustfs-obs scanner collector shape.
|
||||||
|
fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_scans_finished: u64) -> u64 {
|
||||||
|
life_time_ops
|
||||||
|
.get("scan_bucket_drive_start")
|
||||||
|
.copied()
|
||||||
|
.unwrap_or(bucket_scans_finished)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||||
let metrics = global_metrics().report().await;
|
let metrics = global_metrics().report().await;
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
|
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
|
||||||
|
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
|
||||||
|
let bucket_scans_failed = metrics
|
||||||
|
.life_time_ops
|
||||||
|
.get("scan_bucket_drive_failure")
|
||||||
|
.copied()
|
||||||
|
.unwrap_or_default();
|
||||||
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
|
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
|
||||||
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
|
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
|
||||||
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
|
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
|
||||||
@@ -914,21 +927,27 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
|||||||
|
|
||||||
Some(ScannerStats {
|
Some(ScannerStats {
|
||||||
bucket_scans_finished,
|
bucket_scans_finished,
|
||||||
// `global_metrics()` currently tracks completed bucket-drive scans, not a
|
bucket_scans_started,
|
||||||
// separate started counter. Mirror the finished count until Task 5/Task 10
|
bucket_scans_failed,
|
||||||
// expands the scanner runtime source shape.
|
|
||||||
bucket_scans_started: bucket_scans_finished,
|
|
||||||
directories_scanned,
|
directories_scanned,
|
||||||
objects_scanned,
|
objects_scanned,
|
||||||
versions_scanned,
|
versions_scanned,
|
||||||
last_activity_seconds,
|
last_activity_seconds,
|
||||||
active_paths,
|
active_paths,
|
||||||
|
throttle_idle_mode_enabled: u64::from(metrics.throttle_idle_mode_enabled),
|
||||||
|
throttle_sleep_factor: metrics.throttle_sleep_factor,
|
||||||
|
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
|
||||||
|
yield_every_n_objects: metrics.yield_every_n_objects,
|
||||||
|
cycle_interval_seconds: metrics.cycle_interval_seconds,
|
||||||
|
bitrot_cycle_enabled: u64::from(metrics.bitrot_cycle_enabled),
|
||||||
|
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||||
current_cycle: metrics.current_cycle,
|
current_cycle: metrics.current_cycle,
|
||||||
completed_cycles,
|
completed_cycles,
|
||||||
current_cycle_age_seconds,
|
current_cycle_age_seconds,
|
||||||
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
|
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
|
||||||
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
|
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
|
||||||
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
|
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
|
||||||
|
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
|
||||||
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
|
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
|
||||||
current_cycle_directories_per_second: scanner_work_rate_per_second(
|
current_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||||
metrics.current_cycle_directories_scanned,
|
metrics.current_cycle_directories_scanned,
|
||||||
@@ -938,12 +957,19 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
|||||||
metrics.current_cycle_bucket_drive_scans,
|
metrics.current_cycle_bucket_drive_scans,
|
||||||
current_cycle_age,
|
current_cycle_age,
|
||||||
),
|
),
|
||||||
|
current_cycle_yield_events: metrics.current_cycle_yield_events,
|
||||||
|
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
|
||||||
|
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
|
||||||
|
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
|
||||||
|
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
|
||||||
|
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
|
||||||
current_scan_mode,
|
current_scan_mode,
|
||||||
last_cycle_result: metrics.last_cycle_result_code,
|
last_cycle_result: metrics.last_cycle_result_code,
|
||||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||||
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
|
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
|
||||||
|
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
|
||||||
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
|
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
|
||||||
last_cycle_directories_per_second: scanner_work_rate_per_second(
|
last_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||||
metrics.last_cycle_directories_scanned,
|
metrics.last_cycle_directories_scanned,
|
||||||
@@ -953,6 +979,12 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
|||||||
metrics.last_cycle_bucket_drive_scans,
|
metrics.last_cycle_bucket_drive_scans,
|
||||||
last_cycle_duration,
|
last_cycle_duration,
|
||||||
),
|
),
|
||||||
|
last_cycle_yield_events: metrics.last_cycle_yield_events,
|
||||||
|
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
|
||||||
|
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
|
||||||
|
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
|
||||||
|
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
|
||||||
|
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
|
||||||
failed_cycles: metrics.failed_cycles,
|
failed_cycles: metrics.failed_cycles,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -1056,6 +1088,21 @@ mod tests {
|
|||||||
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
|
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_bucket_scans_started_uses_explicit_started_count() {
|
||||||
|
let mut life_time_ops = HashMap::new();
|
||||||
|
life_time_ops.insert("scan_bucket_drive_start".to_string(), 7);
|
||||||
|
|
||||||
|
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_bucket_scans_started_falls_back_to_finished_count() {
|
||||||
|
let life_time_ops = HashMap::new();
|
||||||
|
|
||||||
|
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_work_rate_per_second_reports_rate() {
|
fn scanner_work_rate_per_second_reports_rate() {
|
||||||
assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
|
assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
|
||||||
|
|||||||
@@ -266,8 +266,13 @@ fn bitrot_scan_cycle() -> Option<Duration> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_cycle_scan_mode(current_cycle: u64, bitrot_start_cycle: u64, bitrot_start_time: Option<DateTime<Utc>>) -> HealScanMode {
|
fn get_cycle_scan_mode(
|
||||||
let Some(bitrot_cycle) = bitrot_scan_cycle() else {
|
current_cycle: u64,
|
||||||
|
bitrot_start_cycle: u64,
|
||||||
|
bitrot_start_time: Option<DateTime<Utc>>,
|
||||||
|
bitrot_cycle: Option<Duration>,
|
||||||
|
) -> HealScanMode {
|
||||||
|
let Some(bitrot_cycle) = bitrot_cycle else {
|
||||||
return HealScanMode::Normal;
|
return HealScanMode::Normal;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -299,8 +304,10 @@ fn background_heal_info_for_scan_start(
|
|||||||
current_cycle: u64,
|
current_cycle: u64,
|
||||||
scan_mode: HealScanMode,
|
scan_mode: HealScanMode,
|
||||||
now: DateTime<Utc>,
|
now: DateTime<Utc>,
|
||||||
|
bitrot_cycle: Option<Duration>,
|
||||||
) -> Option<BackgroundHealInfo> {
|
) -> Option<BackgroundHealInfo> {
|
||||||
let reset_bitrot_start = scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now);
|
let reset_bitrot_start =
|
||||||
|
scan_mode == HealScanMode::Deep && should_reset_bitrot_start(&info, current_cycle, now, bitrot_cycle);
|
||||||
if info.current_scan_mode == scan_mode && !reset_bitrot_start {
|
if info.current_scan_mode == scan_mode && !reset_bitrot_start {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
@@ -314,12 +321,17 @@ fn background_heal_info_for_scan_start(
|
|||||||
Some(info)
|
Some(info)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_reset_bitrot_start(info: &BackgroundHealInfo, current_cycle: u64, now: DateTime<Utc>) -> bool {
|
fn should_reset_bitrot_start(
|
||||||
|
info: &BackgroundHealInfo,
|
||||||
|
current_cycle: u64,
|
||||||
|
now: DateTime<Utc>,
|
||||||
|
bitrot_cycle: Option<Duration>,
|
||||||
|
) -> bool {
|
||||||
let Some(bitrot_start_time) = info.bitrot_start_time else {
|
let Some(bitrot_start_time) = info.bitrot_start_time else {
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
let Some(bitrot_cycle) = bitrot_scan_cycle() else {
|
let Some(bitrot_cycle) = bitrot_cycle else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -424,6 +436,9 @@ fn get_lock_acquire_timeout() -> Duration {
|
|||||||
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>, cycle_info: &mut CurrentCycle) {
|
||||||
let _activity_guard = ScannerActivityGuard::new();
|
let _activity_guard = ScannerActivityGuard::new();
|
||||||
SCANNER_SLEEPER.refresh_from_env();
|
SCANNER_SLEEPER.refresh_from_env();
|
||||||
|
let configured_cycle_interval = cycle_interval();
|
||||||
|
let configured_bitrot_cycle = bitrot_scan_cycle();
|
||||||
|
global_metrics().record_scanner_cycle_config(configured_cycle_interval, configured_bitrot_cycle);
|
||||||
info!("Start run data scanner cycle");
|
info!("Start run data scanner cycle");
|
||||||
cycle_info.current = cycle_info.next;
|
cycle_info.current = cycle_info.next;
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
@@ -437,11 +452,16 @@ async fn run_data_scanner_cycle(ctx: &CancellationToken, storeapi: &Arc<ECStore>
|
|||||||
cycle_info.current,
|
cycle_info.current,
|
||||||
background_heal_info.bitrot_start_cycle,
|
background_heal_info.bitrot_start_cycle,
|
||||||
background_heal_info.bitrot_start_time,
|
background_heal_info.bitrot_start_time,
|
||||||
|
configured_bitrot_cycle,
|
||||||
);
|
);
|
||||||
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
let _scan_mode_guard = ScannerScanModeGuard::new(scan_mode);
|
||||||
if let Some(new_heal_info) =
|
if let Some(new_heal_info) = background_heal_info_for_scan_start(
|
||||||
background_heal_info_for_scan_start(background_heal_info.clone(), cycle_info.current, scan_mode, Utc::now())
|
background_heal_info.clone(),
|
||||||
{
|
cycle_info.current,
|
||||||
|
scan_mode,
|
||||||
|
Utc::now(),
|
||||||
|
configured_bitrot_cycle,
|
||||||
|
) {
|
||||||
background_heal_info = new_heal_info.clone();
|
background_heal_info = new_heal_info.clone();
|
||||||
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
save_background_heal_info(storeapi.clone(), new_heal_info).await;
|
||||||
}
|
}
|
||||||
@@ -600,18 +620,22 @@ pub async fn store_data_usage_in_backend(
|
|||||||
// Save a backup every 10th update
|
// Save a backup every 10th update
|
||||||
if attempts > 10 {
|
if attempts > 10 {
|
||||||
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
|
if let Err(e) = save_config(storeapi.clone(), &backup_path, data.clone()).await {
|
||||||
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
|
warn!("Failed to save data usage backup to {}: {}", backup_path, e);
|
||||||
}
|
}
|
||||||
|
done_save();
|
||||||
attempts = 1;
|
attempts = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save main configuration
|
// Save main configuration
|
||||||
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
|
if let Err(e) = save_config(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data).await {
|
||||||
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
error!("Failed to save data usage info to {}: {e}", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||||
} else {
|
} else {
|
||||||
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
rustfs_ecstore::data_usage::replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||||
}
|
}
|
||||||
|
done_save();
|
||||||
|
|
||||||
attempts += 1;
|
attempts += 1;
|
||||||
}
|
}
|
||||||
@@ -866,7 +890,7 @@ mod tests {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
fn test_get_cycle_scan_mode_runs_deep_until_selection_window_completes() {
|
||||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("3600"), || {
|
||||||
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()));
|
let mode = get_cycle_scan_mode(10, 0, Some(Utc::now()), bitrot_scan_cycle());
|
||||||
assert_eq!(mode, HealScanMode::Deep);
|
assert_eq!(mode, HealScanMode::Deep);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -878,8 +902,8 @@ mod tests {
|
|||||||
let recent = Utc::now() - chrono::Duration::minutes(30);
|
let recent = Utc::now() - chrono::Duration::minutes(30);
|
||||||
let old = Utc::now() - chrono::Duration::hours(2);
|
let old = Utc::now() - chrono::Duration::hours(2);
|
||||||
|
|
||||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent)), HealScanMode::Normal);
|
assert_eq!(get_cycle_scan_mode(2048, 0, Some(recent), bitrot_scan_cycle()), HealScanMode::Normal);
|
||||||
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old)), HealScanMode::Deep);
|
assert_eq!(get_cycle_scan_mode(2048, 0, Some(old), bitrot_scan_cycle()), HealScanMode::Deep);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -887,7 +911,7 @@ mod tests {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
fn test_get_cycle_scan_mode_can_disable_periodic_deep_scan() {
|
||||||
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
with_var(ENV_SCANNER_BITROT_CYCLE_SECS, Some("off"), || {
|
||||||
assert_eq!(get_cycle_scan_mode(1, 0, None), HealScanMode::Normal);
|
assert_eq!(get_cycle_scan_mode(1, 0, None, bitrot_scan_cycle()), HealScanMode::Normal);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -895,8 +919,9 @@ mod tests {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
fn test_background_heal_info_for_scan_start_marks_deep_active() {
|
||||||
let now = Utc::now();
|
let now = Utc::now();
|
||||||
let info = background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now)
|
let info =
|
||||||
.expect("deep scan should update background heal info");
|
background_heal_info_for_scan_start(BackgroundHealInfo::default(), 7, HealScanMode::Deep, now, bitrot_scan_cycle())
|
||||||
|
.expect("deep scan should update background heal info");
|
||||||
|
|
||||||
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
||||||
assert_eq!(info.bitrot_start_cycle, 7);
|
assert_eq!(info.bitrot_start_cycle, 7);
|
||||||
@@ -914,7 +939,7 @@ mod tests {
|
|||||||
current_scan_mode: HealScanMode::Normal,
|
current_scan_mode: HealScanMode::Normal,
|
||||||
};
|
};
|
||||||
|
|
||||||
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now())
|
let info = background_heal_info_for_scan_start(info, 8, HealScanMode::Deep, Utc::now(), bitrot_scan_cycle())
|
||||||
.expect("deep scan should mark active status");
|
.expect("deep scan should mark active status");
|
||||||
|
|
||||||
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
assert_eq!(info.current_scan_mode, HealScanMode::Deep);
|
||||||
|
|||||||
@@ -16,19 +16,19 @@ use std::collections::HashSet;
|
|||||||
use std::fs::FileType;
|
use std::fs::FileType;
|
||||||
use std::io::ErrorKind;
|
use std::io::ErrorKind;
|
||||||
use std::sync::{Arc, Once};
|
use std::sync::{Arc, Once};
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, Instant, SystemTime};
|
||||||
|
|
||||||
use crate::ReplTargetSizeSummary;
|
use crate::ReplTargetSizeSummary;
|
||||||
use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path};
|
use crate::data_usage_define::{DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, SizeSummary, hash_path};
|
||||||
use crate::error::ScannerError;
|
use crate::error::ScannerError;
|
||||||
use crate::scanner_io::ScannerIODisk as _;
|
use crate::scanner_io::ScannerIODisk as _;
|
||||||
use crate::sleeper::DynamicSleeper;
|
use crate::sleeper::{DynamicSleeper, scanner_yield_every_n_objects};
|
||||||
use metrics::{counter, describe_counter};
|
use metrics::{counter, describe_counter};
|
||||||
use rustfs_common::heal_channel::{
|
use rustfs_common::heal_channel::{
|
||||||
HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode,
|
HEAL_DELETE_DANGLING, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HealScanMode,
|
||||||
send_heal_request_with_admission,
|
send_heal_request_with_admission,
|
||||||
};
|
};
|
||||||
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater};
|
use rustfs_common::metrics::{IlmAction, Metric, Metrics, UpdateCurrentPathFn, current_path_updater, global_metrics};
|
||||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||||
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
|
use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, apply_expiry_rule};
|
||||||
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
|
use rustfs_ecstore::bucket::lifecycle::evaluator::Evaluator;
|
||||||
@@ -143,13 +143,6 @@ fn scanner_excess_folders_threshold() -> u64 {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn scanner_yield_every_n_objects() -> u64 {
|
|
||||||
rustfs_utils::get_env_u64(
|
|
||||||
rustfs_config::ENV_SCANNER_YIELD_EVERY_N_OBJECTS,
|
|
||||||
rustfs_config::DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
|
fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
|
||||||
yield_every > 0 && object_count.is_multiple_of(yield_every)
|
yield_every > 0 && object_count.is_multiple_of(yield_every)
|
||||||
}
|
}
|
||||||
@@ -485,6 +478,7 @@ impl ScannerItem {
|
|||||||
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
||||||
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||||
done_ilm(1)();
|
done_ilm(1)();
|
||||||
|
global_metrics().record_scanner_ilm_action(1);
|
||||||
break 'eventLoop;
|
break 'eventLoop;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -497,6 +491,7 @@ impl ScannerItem {
|
|||||||
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
|
||||||
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||||
done_ilm(1)();
|
done_ilm(1)();
|
||||||
|
global_metrics().record_scanner_ilm_action(1);
|
||||||
}
|
}
|
||||||
IlmAction::DeleteVersionAction => {
|
IlmAction::DeleteVersionAction => {
|
||||||
remaining_versions -= 1;
|
remaining_versions -= 1;
|
||||||
@@ -510,11 +505,13 @@ impl ScannerItem {
|
|||||||
}
|
}
|
||||||
noncurrent_events.push(event.clone());
|
noncurrent_events.push(event.clone());
|
||||||
done_ilm(1)();
|
done_ilm(1)();
|
||||||
|
global_metrics().record_scanner_ilm_action(1);
|
||||||
}
|
}
|
||||||
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
||||||
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
|
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
|
||||||
apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
|
apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||||
done_ilm(1)();
|
done_ilm(1)();
|
||||||
|
global_metrics().record_scanner_ilm_action(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
IlmAction::NoneAction | IlmAction::ActionCount => {
|
IlmAction::NoneAction | IlmAction::ActionCount => {
|
||||||
@@ -560,7 +557,9 @@ impl ScannerItem {
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let done_replication = Metrics::time(Metric::CheckReplication);
|
||||||
let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
|
let roi = queue_replication_heal_internal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
|
||||||
|
done_replication();
|
||||||
if !Self::should_account_replication_stats(oi) {
|
if !Self::should_account_replication_stats(oi) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1110,7 +1109,9 @@ impl FolderScanner {
|
|||||||
timer.sleep().await;
|
timer.sleep().await;
|
||||||
|
|
||||||
if should_yield_after_object(object_count, yield_every_objects) {
|
if should_yield_after_object(object_count, yield_every_objects) {
|
||||||
|
let yield_start = Instant::now();
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
global_metrics().record_scanner_yield(yield_start.elapsed());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ use futures::future::join_all;
|
|||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rand::seq::SliceRandom as _;
|
use rand::seq::SliceRandom as _;
|
||||||
use rustfs_common::heal_channel::HealScanMode;
|
use rustfs_common::heal_channel::HealScanMode;
|
||||||
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete};
|
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, global_metrics};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
||||||
ENV_SCANNER_MAX_CONCURRENT_SET_SCANS,
|
ENV_SCANNER_MAX_CONCURRENT_SET_SCANS,
|
||||||
@@ -116,6 +116,28 @@ impl Drop for DiskBucketScanActiveGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct BucketDriveFailureGuard {
|
||||||
|
failed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BucketDriveFailureGuard {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self { failed: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mark_success(&mut self) {
|
||||||
|
self.failed = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for BucketDriveFailureGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.failed {
|
||||||
|
global_metrics().record_scan_bucket_drive_failure();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
|
fn decrement_atomic_usize(counter: &AtomicUsize) -> usize {
|
||||||
counter
|
counter
|
||||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
|
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(1)))
|
||||||
@@ -217,9 +239,11 @@ async fn persist_and_publish_cache_snapshot<S: StorageAPI>(
|
|||||||
) -> Option<SystemTime> {
|
) -> Option<SystemTime> {
|
||||||
let last_update = cache_snapshot.info.last_update;
|
let last_update = cache_snapshot.info.last_update;
|
||||||
|
|
||||||
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
|
if let Err(e) = cache_snapshot.save(store, DATA_USAGE_CACHE_NAME).await {
|
||||||
error!("Failed to save data usage cache: {}", e);
|
error!("Failed to save data usage cache: {}", e);
|
||||||
}
|
}
|
||||||
|
done_save();
|
||||||
|
|
||||||
if let Err(e) = updates.send(cache_snapshot).await {
|
if let Err(e) = updates.send(cache_snapshot).await {
|
||||||
error!("Failed to send data usage cache: {}", e);
|
error!("Failed to send data usage cache: {}", e);
|
||||||
@@ -732,9 +756,12 @@ impl ScannerIOCache for SetDisks {
|
|||||||
|
|
||||||
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
|
if let (Some(last_update), Some(before_update)) = (cache.info.last_update, before)
|
||||||
&& last_update > before_update
|
&& last_update > before_update
|
||||||
&& let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await
|
|
||||||
{
|
{
|
||||||
error!("Failed to save data usage cache: {}", e);
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
|
if let Err(e) = cache.save(store_clone_clone.clone(), cache_name.as_str()).await {
|
||||||
|
error!("Failed to save data usage cache: {}", e);
|
||||||
|
}
|
||||||
|
done_save();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = update_fut.await {
|
if let Err(e) = update_fut.await {
|
||||||
@@ -775,9 +802,11 @@ impl ScannerIOCache for SetDisks {
|
|||||||
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
|
error!("nsscanner_disk: Failed to send data usage entry info: {}", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let done_save = Metrics::time(Metric::SaveUsage);
|
||||||
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
|
if let Err(e) = cache.save(store_clone_clone.clone(), &cache_name).await {
|
||||||
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
|
error!("nsscanner_disk: Failed to save data usage cache: {}", e);
|
||||||
}
|
}
|
||||||
|
done_save();
|
||||||
}
|
}
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
@@ -900,6 +929,8 @@ impl ScannerIODisk for Disk {
|
|||||||
let drive_start = std::time::Instant::now();
|
let drive_start = std::time::Instant::now();
|
||||||
let bucket = cache.info.name.clone();
|
let bucket = cache.info.name.clone();
|
||||||
let disk_path = self.path().to_string_lossy().to_string();
|
let disk_path = self.path().to_string_lossy().to_string();
|
||||||
|
global_metrics().record_scan_bucket_drive_start();
|
||||||
|
let mut failure_guard = BucketDriveFailureGuard::new();
|
||||||
let _guard = self.start_scan();
|
let _guard = self.start_scan();
|
||||||
|
|
||||||
let mut cache = cache;
|
let mut cache = cache;
|
||||||
@@ -966,9 +997,11 @@ impl ScannerIODisk for Disk {
|
|||||||
done_drive();
|
done_drive();
|
||||||
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
|
emit_scan_bucket_drive_complete(true, &bucket, &disk_path, drive_start.elapsed());
|
||||||
data_usage_info.info.last_update = Some(SystemTime::now());
|
data_usage_info.info.last_update = Some(SystemTime::now());
|
||||||
|
failure_guard.mark_success();
|
||||||
Ok(data_usage_info)
|
Ok(data_usage_info)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
done_drive();
|
||||||
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
|
emit_scan_bucket_drive_complete(false, &bucket, &disk_path, drive_start.elapsed());
|
||||||
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
|
Err(StorageError::other(format!("Failed to scan data folder: {e}")))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
|
|||||||
use std::sync::{Arc, LazyLock, RwLock};
|
use std::sync::{Arc, LazyLock, RwLock};
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
use rustfs_config::{DEFAULT_SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, ScannerSpeed};
|
use rustfs_common::metrics::global_metrics;
|
||||||
|
use rustfs_config::{
|
||||||
|
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
|
||||||
|
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||||
|
};
|
||||||
use tokio::time::Duration;
|
use tokio::time::Duration;
|
||||||
|
|
||||||
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
||||||
@@ -64,6 +68,10 @@ fn scanner_env_config() -> (ScannerSpeed, bool) {
|
|||||||
(speed, idle_mode)
|
(speed, idle_mode)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn scanner_yield_every_n_objects() -> u64 {
|
||||||
|
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
|
||||||
|
}
|
||||||
|
|
||||||
/// When `true` (default), the scanner throttles itself between operations.
|
/// When `true` (default), the scanner throttles itself between operations.
|
||||||
/// When `false`, all sleeps are skipped and the scanner runs at full speed.
|
/// When `false`, all sleeps are skipped and the scanner runs at full speed.
|
||||||
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
|
pub static SCANNER_IDLE_MODE: AtomicBool = AtomicBool::new(DEFAULT_SCANNER_IDLE_MODE);
|
||||||
@@ -74,7 +82,9 @@ pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
|
|||||||
let (speed, idle_mode) = scanner_env_config();
|
let (speed, idle_mode) = scanner_env_config();
|
||||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||||
|
|
||||||
DynamicSleeper::new(speed)
|
let sleeper = DynamicSleeper::new(speed);
|
||||||
|
sleeper.record_throttle_config();
|
||||||
|
sleeper
|
||||||
});
|
});
|
||||||
|
|
||||||
/// Proportional-backoff sleeper for the data scanner.
|
/// Proportional-backoff sleeper for the data scanner.
|
||||||
@@ -124,6 +134,7 @@ impl DynamicSleeper {
|
|||||||
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep);
|
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor).min(max_sleep);
|
||||||
if !sleep_dur.is_zero() {
|
if !sleep_dur.is_zero() {
|
||||||
tokio::time::sleep(sleep_dur).await;
|
tokio::time::sleep(sleep_dur).await;
|
||||||
|
global_metrics().record_scanner_yield(sleep_dur);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -150,6 +161,17 @@ impl DynamicSleeper {
|
|||||||
let (speed, idle_mode) = scanner_env_config();
|
let (speed, idle_mode) = scanner_env_config();
|
||||||
self.update(speed);
|
self.update(speed);
|
||||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||||
|
self.record_throttle_config();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_throttle_config(&self) {
|
||||||
|
let (factor, max_sleep) = self.read_params();
|
||||||
|
global_metrics().record_scanner_throttle_config(
|
||||||
|
SCANNER_IDLE_MODE.load(Ordering::Relaxed),
|
||||||
|
factor,
|
||||||
|
max_sleep,
|
||||||
|
scanner_yield_every_n_objects(),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -177,6 +199,7 @@ impl SleepTimer {
|
|||||||
.min(max_sleep);
|
.min(max_sleep);
|
||||||
if !sleep_dur.is_zero() {
|
if !sleep_dur.is_zero() {
|
||||||
tokio::time::sleep(sleep_dur).await;
|
tokio::time::sleep(sleep_dur).await;
|
||||||
|
global_metrics().record_scanner_yield(sleep_dur);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user