mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
feat(scanner): expand scanner observability metrics (#3159)
* feat(scanner): expand scanner observability metrics * chore(scanner): align bucket-drive metric wording --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
@@ -16,29 +16,40 @@
|
||||
|
||||
//! Scanner metrics collector.
|
||||
//!
|
||||
//! Collects background scanner metrics including bucket scans,
|
||||
//! Collects background scanner metrics including bucket-drive scans,
|
||||
//! directory scans, and object scans.
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::scanner::{
|
||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_CURRENT_CYCLE_MD, SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
|
||||
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD,
|
||||
SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD,
|
||||
SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_VERSIONS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
|
||||
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD,
|
||||
SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD,
|
||||
SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||
};
|
||||
|
||||
/// Scanner statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ScannerStats {
|
||||
/// Number of bucket scans finished
|
||||
/// Number of bucket-drive scans finished
|
||||
pub bucket_scans_finished: u64,
|
||||
/// Number of bucket scans started
|
||||
/// Number of bucket-drive scans started
|
||||
pub bucket_scans_started: u64,
|
||||
/// Number of bucket-drive scans that failed
|
||||
pub bucket_scans_failed: u64,
|
||||
/// Number of directories scanned
|
||||
pub directories_scanned: u64,
|
||||
/// Number of objects scanned
|
||||
@@ -49,6 +60,20 @@ pub struct ScannerStats {
|
||||
pub last_activity_seconds: u64,
|
||||
/// Number of scanner paths currently being processed
|
||||
pub active_paths: u64,
|
||||
/// Whether scanner idle-mode self-throttling is enabled
|
||||
pub throttle_idle_mode_enabled: u64,
|
||||
/// Effective scanner sleep factor
|
||||
pub throttle_sleep_factor: f64,
|
||||
/// Effective scanner maximum self-throttle sleep duration in seconds
|
||||
pub throttle_max_sleep_seconds: f64,
|
||||
/// Object interval for cooperative scanner runtime yields
|
||||
pub yield_every_n_objects: u64,
|
||||
/// Effective scanner cycle interval in seconds
|
||||
pub cycle_interval_seconds: f64,
|
||||
/// Whether periodic scanner bitrot deep scans are enabled
|
||||
pub bitrot_cycle_enabled: u64,
|
||||
/// Effective scanner bitrot deep-scan interval in seconds
|
||||
pub bitrot_cycle_seconds: f64,
|
||||
/// Current scanner cycle number, or zero when idle
|
||||
pub current_cycle: u64,
|
||||
/// Number of scanner cycles completed since server start
|
||||
@@ -61,12 +86,26 @@ pub struct ScannerStats {
|
||||
pub current_cycle_directories_scanned: u64,
|
||||
/// Number of bucket-drive scans finished by the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_scans: u64,
|
||||
/// Number of bucket-drive scans that failed in the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_failures: u64,
|
||||
/// Object scan rate for the currently running scanner cycle
|
||||
pub current_cycle_objects_per_second: f64,
|
||||
/// Directory scan rate for the currently running scanner cycle
|
||||
pub current_cycle_directories_per_second: f64,
|
||||
/// Bucket-drive scan rate for the currently running scanner cycle
|
||||
pub current_cycle_bucket_drive_scans_per_second: f64,
|
||||
/// Number of scanner self-throttle yield events in the current scanner cycle
|
||||
pub current_cycle_yield_events: u64,
|
||||
/// Total scanner self-throttle yield duration in seconds for the current scanner cycle
|
||||
pub current_cycle_yield_duration_seconds: f64,
|
||||
/// Number of lifecycle actions applied by the current scanner cycle
|
||||
pub current_cycle_ilm_actions: u64,
|
||||
/// Number of object heal candidates enqueued by the current scanner cycle
|
||||
pub current_cycle_heal_objects: u64,
|
||||
/// Number of replication heal checks run by the current scanner cycle
|
||||
pub current_cycle_replication_checks: u64,
|
||||
/// Number of data-usage save operations run by the current scanner cycle
|
||||
pub current_cycle_usage_saves: u64,
|
||||
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
|
||||
pub current_scan_mode: u64,
|
||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error
|
||||
@@ -79,12 +118,26 @@ pub struct ScannerStats {
|
||||
pub last_cycle_directories_scanned: u64,
|
||||
/// Number of bucket-drive scans finished by the last scanner cycle
|
||||
pub last_cycle_bucket_drive_scans: u64,
|
||||
/// Number of bucket-drive scans that failed in the last finished scanner cycle
|
||||
pub last_cycle_bucket_drive_failures: u64,
|
||||
/// Object scan rate for the last finished scanner cycle
|
||||
pub last_cycle_objects_per_second: f64,
|
||||
/// Directory scan rate for the last finished scanner cycle
|
||||
pub last_cycle_directories_per_second: f64,
|
||||
/// Bucket-drive scan rate for the last finished scanner cycle
|
||||
pub last_cycle_bucket_drive_scans_per_second: f64,
|
||||
/// Number of scanner self-throttle yield events in the last finished scanner cycle
|
||||
pub last_cycle_yield_events: u64,
|
||||
/// Total scanner self-throttle yield duration in seconds for the last finished scanner cycle
|
||||
pub last_cycle_yield_duration_seconds: f64,
|
||||
/// Number of lifecycle actions applied by the last finished scanner cycle
|
||||
pub last_cycle_ilm_actions: u64,
|
||||
/// Number of object heal candidates enqueued by the last finished scanner cycle
|
||||
pub last_cycle_heal_objects: u64,
|
||||
/// Number of replication heal checks run by the last finished scanner cycle
|
||||
pub last_cycle_replication_checks: u64,
|
||||
/// Number of data-usage save operations run by the last finished scanner cycle
|
||||
pub last_cycle_usage_saves: u64,
|
||||
/// Number of scanner cycles that failed since server start
|
||||
pub failed_cycles: u64,
|
||||
}
|
||||
@@ -97,11 +150,19 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
vec![
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_DIRECTORIES_SCANNED_MD, stats.directories_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_OBJECTS_SCANNED_MD, stats.objects_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, stats.throttle_idle_mode_enabled as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_SLEEP_FACTOR_MD, stats.throttle_sleep_factor),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, stats.throttle_max_sleep_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, stats.bitrot_cycle_enabled as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_COMPLETED_CYCLES_MD, stats.completed_cycles as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, stats.current_cycle_age_seconds as f64),
|
||||
@@ -114,6 +175,10 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
stats.current_cycle_bucket_drive_scans as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
stats.current_cycle_bucket_drive_failures as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, stats.current_cycle_objects_per_second),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
@@ -123,6 +188,18 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
&SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
stats.current_cycle_bucket_drive_scans_per_second,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, stats.current_cycle_yield_events as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD,
|
||||
stats.current_cycle_yield_duration_seconds,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, stats.current_cycle_ilm_actions as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, stats.current_cycle_heal_objects as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD,
|
||||
stats.current_cycle_replication_checks as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
|
||||
@@ -132,12 +209,22 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
stats.last_cycle_directories_scanned as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, stats.last_cycle_bucket_drive_scans as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
stats.last_cycle_bucket_drive_failures as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, stats.last_cycle_objects_per_second),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, stats.last_cycle_directories_per_second),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
stats.last_cycle_bucket_drive_scans_per_second,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, stats.last_cycle_yield_events as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, stats.last_cycle_yield_duration_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, stats.last_cycle_ilm_actions as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, stats.last_cycle_heal_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, stats.last_cycle_replication_checks as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
|
||||
]
|
||||
}
|
||||
@@ -152,36 +239,58 @@ mod tests {
|
||||
let stats = ScannerStats {
|
||||
bucket_scans_finished: 100,
|
||||
bucket_scans_started: 100,
|
||||
bucket_scans_failed: 2,
|
||||
directories_scanned: 50000,
|
||||
objects_scanned: 1000000,
|
||||
versions_scanned: 1500000,
|
||||
last_activity_seconds: 30,
|
||||
active_paths: 4,
|
||||
throttle_idle_mode_enabled: 1,
|
||||
throttle_sleep_factor: 10.0,
|
||||
throttle_max_sleep_seconds: 15.0,
|
||||
yield_every_n_objects: 128,
|
||||
cycle_interval_seconds: 3600.0,
|
||||
bitrot_cycle_enabled: 1,
|
||||
bitrot_cycle_seconds: 86400.0,
|
||||
current_cycle: 12,
|
||||
completed_cycles: 11,
|
||||
current_cycle_age_seconds: 90,
|
||||
current_cycle_objects_scanned: 250,
|
||||
current_cycle_directories_scanned: 20,
|
||||
current_cycle_bucket_drive_scans: 2,
|
||||
current_cycle_bucket_drive_failures: 1,
|
||||
current_cycle_objects_per_second: 12.5,
|
||||
current_cycle_directories_per_second: 1.0,
|
||||
current_cycle_bucket_drive_scans_per_second: 0.1,
|
||||
current_cycle_yield_events: 8,
|
||||
current_cycle_yield_duration_seconds: 1.25,
|
||||
current_cycle_ilm_actions: 6,
|
||||
current_cycle_heal_objects: 2,
|
||||
current_cycle_replication_checks: 5,
|
||||
current_cycle_usage_saves: 3,
|
||||
current_scan_mode: 2,
|
||||
last_cycle_result: 1,
|
||||
last_cycle_duration_seconds: 42.5,
|
||||
last_cycle_objects_scanned: 900,
|
||||
last_cycle_directories_scanned: 80,
|
||||
last_cycle_bucket_drive_scans: 6,
|
||||
last_cycle_bucket_drive_failures: 2,
|
||||
last_cycle_objects_per_second: 18.0,
|
||||
last_cycle_directories_per_second: 1.6,
|
||||
last_cycle_bucket_drive_scans_per_second: 0.12,
|
||||
last_cycle_yield_events: 30,
|
||||
last_cycle_yield_duration_seconds: 9.5,
|
||||
last_cycle_ilm_actions: 44,
|
||||
last_cycle_heal_objects: 7,
|
||||
last_cycle_replication_checks: 12,
|
||||
last_cycle_usage_saves: 9,
|
||||
failed_cycles: 3,
|
||||
};
|
||||
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 26);
|
||||
assert_eq!(metrics.len(), 48);
|
||||
|
||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(objects.is_some());
|
||||
@@ -194,6 +303,46 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
||||
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
||||
|
||||
let bucket_scans_failed = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name());
|
||||
assert_eq!(bucket_scans_failed.map(|m| m.value), Some(2.0));
|
||||
|
||||
let throttle_idle_mode_enabled = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_idle_mode_enabled.map(|m| m.value), Some(1.0));
|
||||
|
||||
let throttle_sleep_factor = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_SLEEP_FACTOR_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_sleep_factor.map(|m| m.value), Some(10.0));
|
||||
|
||||
let throttle_max_sleep = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(throttle_max_sleep.map(|m| m.value), Some(15.0));
|
||||
|
||||
let yield_every_n_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_YIELD_EVERY_N_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(yield_every_n_objects.map(|m| m.value), Some(128.0));
|
||||
|
||||
let cycle_interval_seconds = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CYCLE_INTERVAL_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(cycle_interval_seconds.map(|m| m.value), Some(3600.0));
|
||||
|
||||
let bitrot_cycle_enabled = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name());
|
||||
assert_eq!(bitrot_cycle_enabled.map(|m| m.value), Some(1.0));
|
||||
|
||||
let bitrot_cycle_seconds = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BITROT_CYCLE_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(bitrot_cycle_seconds.map(|m| m.value), Some(86400.0));
|
||||
|
||||
let current_cycle = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_MD.get_full_metric_name());
|
||||
@@ -224,6 +373,11 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_scans.map(|m| m.value), Some(2.0));
|
||||
|
||||
let current_cycle_bucket_drive_failures = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_failures.map(|m| m.value), Some(1.0));
|
||||
|
||||
let current_cycle_objects_rate = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||
@@ -239,6 +393,36 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.1));
|
||||
|
||||
let current_cycle_yield_events = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_yield_events.map(|m| m.value), Some(8.0));
|
||||
|
||||
let current_cycle_yield_duration = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_yield_duration.map(|m| m.value), Some(1.25));
|
||||
|
||||
let current_cycle_ilm_actions = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_ilm_actions.map(|m| m.value), Some(6.0));
|
||||
|
||||
let current_cycle_heal_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_heal_objects.map(|m| m.value), Some(2.0));
|
||||
|
||||
let current_cycle_replication_checks = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_replication_checks.map(|m| m.value), Some(5.0));
|
||||
|
||||
let current_cycle_usage_saves = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||
assert_eq!(current_cycle_usage_saves.map(|m| m.value), Some(3.0));
|
||||
|
||||
let current_scan_mode = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_SCAN_MODE_MD.get_full_metric_name());
|
||||
@@ -269,6 +453,11 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_scans.map(|m| m.value), Some(6.0));
|
||||
|
||||
let last_cycle_bucket_drive_failures = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_failures.map(|m| m.value), Some(2.0));
|
||||
|
||||
let last_cycle_objects_rate = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD.get_full_metric_name());
|
||||
@@ -284,6 +473,36 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_bucket_drive_scans_rate.map(|m| m.value), Some(0.12));
|
||||
|
||||
let last_cycle_yield_events = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_EVENTS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_yield_events.map(|m| m.value), Some(30.0));
|
||||
|
||||
let last_cycle_yield_duration = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_yield_duration.map(|m| m.value), Some(9.5));
|
||||
|
||||
let last_cycle_ilm_actions = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_ILM_ACTIONS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_ilm_actions.map(|m| m.value), Some(44.0));
|
||||
|
||||
let last_cycle_heal_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_heal_objects.map(|m| m.value), Some(7.0));
|
||||
|
||||
let last_cycle_replication_checks = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_replication_checks.map(|m| m.value), Some(12.0));
|
||||
|
||||
let last_cycle_usage_saves = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_USAGE_SAVES_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_usage_saves.map(|m| m.value), Some(9.0));
|
||||
|
||||
let failed_cycles = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_FAILED_CYCLES_MD.get_full_metric_name());
|
||||
@@ -295,7 +514,7 @@ mod tests {
|
||||
let stats = ScannerStats::default();
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 26);
|
||||
assert_eq!(metrics.len(), 48);
|
||||
for metric in &metrics {
|
||||
assert_eq!(metric.value, 0.0);
|
||||
assert!(metric.labels.is_empty());
|
||||
|
||||
@@ -279,24 +279,46 @@ pub enum MetricName {
|
||||
ScannerVersionsScanned,
|
||||
ScannerLastActivitySeconds,
|
||||
ScannerActivePaths,
|
||||
ScannerBucketScansFailed,
|
||||
ScannerThrottleIdleModeEnabled,
|
||||
ScannerThrottleSleepFactor,
|
||||
ScannerThrottleMaxSleepSeconds,
|
||||
ScannerYieldEveryNObjects,
|
||||
ScannerCycleIntervalSeconds,
|
||||
ScannerBitrotCycleEnabled,
|
||||
ScannerBitrotCycleSeconds,
|
||||
ScannerCurrentCycle,
|
||||
ScannerCompletedCycles,
|
||||
ScannerCurrentCycleAgeSeconds,
|
||||
ScannerCurrentCycleObjectsScanned,
|
||||
ScannerCurrentCycleDirectoriesScanned,
|
||||
ScannerCurrentCycleBucketDriveScans,
|
||||
ScannerCurrentCycleBucketDriveFailures,
|
||||
ScannerCurrentCycleObjectsPerSecond,
|
||||
ScannerCurrentCycleDirectoriesPerSecond,
|
||||
ScannerCurrentCycleBucketDriveScansPerSecond,
|
||||
ScannerCurrentCycleYieldEvents,
|
||||
ScannerCurrentCycleYieldDurationSeconds,
|
||||
ScannerCurrentCycleIlmActions,
|
||||
ScannerCurrentCycleHealObjects,
|
||||
ScannerCurrentCycleReplicationChecks,
|
||||
ScannerCurrentCycleUsageSaves,
|
||||
ScannerCurrentScanMode,
|
||||
ScannerLastCycleResult,
|
||||
ScannerLastCycleDurationSeconds,
|
||||
ScannerLastCycleObjectsScanned,
|
||||
ScannerLastCycleDirectoriesScanned,
|
||||
ScannerLastCycleBucketDriveScans,
|
||||
ScannerLastCycleBucketDriveFailures,
|
||||
ScannerLastCycleObjectsPerSecond,
|
||||
ScannerLastCycleDirectoriesPerSecond,
|
||||
ScannerLastCycleBucketDriveScansPerSecond,
|
||||
ScannerLastCycleYieldEvents,
|
||||
ScannerLastCycleYieldDurationSeconds,
|
||||
ScannerLastCycleIlmActions,
|
||||
ScannerLastCycleHealObjects,
|
||||
ScannerLastCycleReplicationChecks,
|
||||
ScannerLastCycleUsageSaves,
|
||||
ScannerFailedCycles,
|
||||
|
||||
// CPU system-related metrics
|
||||
@@ -639,24 +661,46 @@ impl MetricName {
|
||||
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
|
||||
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
|
||||
Self::ScannerActivePaths => "active_paths".to_string(),
|
||||
Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(),
|
||||
Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(),
|
||||
Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(),
|
||||
Self::ScannerThrottleMaxSleepSeconds => "throttle_max_sleep_seconds".to_string(),
|
||||
Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(),
|
||||
Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(),
|
||||
Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(),
|
||||
Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(),
|
||||
Self::ScannerCurrentCycle => "current_cycle".to_string(),
|
||||
Self::ScannerCompletedCycles => "completed_cycles".to_string(),
|
||||
Self::ScannerCurrentCycleAgeSeconds => "current_cycle_age_seconds".to_string(),
|
||||
Self::ScannerCurrentCycleObjectsScanned => "current_cycle_objects_scanned".to_string(),
|
||||
Self::ScannerCurrentCycleDirectoriesScanned => "current_cycle_directories_scanned".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveScans => "current_cycle_bucket_drive_scans".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveFailures => "current_cycle_bucket_drive_failures".to_string(),
|
||||
Self::ScannerCurrentCycleObjectsPerSecond => "current_cycle_objects_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleDirectoriesPerSecond => "current_cycle_directories_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleBucketDriveScansPerSecond => "current_cycle_bucket_drive_scans_per_second".to_string(),
|
||||
Self::ScannerCurrentCycleYieldEvents => "current_cycle_yield_events".to_string(),
|
||||
Self::ScannerCurrentCycleYieldDurationSeconds => "current_cycle_yield_duration_seconds".to_string(),
|
||||
Self::ScannerCurrentCycleIlmActions => "current_cycle_ilm_actions".to_string(),
|
||||
Self::ScannerCurrentCycleHealObjects => "current_cycle_heal_objects".to_string(),
|
||||
Self::ScannerCurrentCycleReplicationChecks => "current_cycle_replication_checks".to_string(),
|
||||
Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(),
|
||||
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
|
||||
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
|
||||
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
|
||||
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
|
||||
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveScans => "last_cycle_bucket_drive_scans".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveFailures => "last_cycle_bucket_drive_failures".to_string(),
|
||||
Self::ScannerLastCycleObjectsPerSecond => "last_cycle_objects_per_second".to_string(),
|
||||
Self::ScannerLastCycleDirectoriesPerSecond => "last_cycle_directories_per_second".to_string(),
|
||||
Self::ScannerLastCycleBucketDriveScansPerSecond => "last_cycle_bucket_drive_scans_per_second".to_string(),
|
||||
Self::ScannerLastCycleYieldEvents => "last_cycle_yield_events".to_string(),
|
||||
Self::ScannerLastCycleYieldDurationSeconds => "last_cycle_yield_duration_seconds".to_string(),
|
||||
Self::ScannerLastCycleIlmActions => "last_cycle_ilm_actions".to_string(),
|
||||
Self::ScannerLastCycleHealObjects => "last_cycle_heal_objects".to_string(),
|
||||
Self::ScannerLastCycleReplicationChecks => "last_cycle_replication_checks".to_string(),
|
||||
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
|
||||
Self::ScannerFailedCycles => "failed_cycles".to_string(),
|
||||
|
||||
// CPU system-related metrics
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::sync::LazyLock;
|
||||
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansFinished,
|
||||
"Total number of bucket scans finished since server start",
|
||||
"Total number of bucket-drive scans finished since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
@@ -29,7 +29,16 @@ pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLo
|
||||
pub static SCANNER_BUCKET_SCANS_STARTED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansStarted,
|
||||
"Total number of bucket scans started since server start",
|
||||
"Total number of bucket-drive scans started since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BUCKET_SCANS_FAILED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansFailed,
|
||||
"Total number of bucket-drive scans that failed since server start",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
@@ -80,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleIdleModeEnabled,
|
||||
"Whether scanner idle-mode self-throttling is enabled: 1 enabled, 0 disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_SLEEP_FACTOR_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleSleepFactor,
|
||||
"Effective scanner sleep factor used to compute proportional self-throttle sleeps.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleMaxSleepSeconds,
|
||||
"Effective maximum scanner self-throttle sleep duration in seconds.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_YIELD_EVERY_N_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerYieldEveryNObjects,
|
||||
"Current object interval for cooperative scanner runtime yields, or zero when disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_INTERVAL_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCycleIntervalSeconds,
|
||||
"Effective scanner cycle interval in seconds.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerBitrotCycleEnabled,
|
||||
"Whether periodic scanner bitrot deep scans are enabled: 1 enabled, 0 disabled.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BITROT_CYCLE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerBitrotCycleSeconds,
|
||||
"Effective scanner bitrot deep-scan interval in seconds; zero means disabled or deep-scan every cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycle,
|
||||
@@ -134,6 +206,15 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescripto
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleBucketDriveFailures,
|
||||
"Number of bucket-drive scans that failed in the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleObjectsPerSecond,
|
||||
@@ -161,6 +242,60 @@ pub static SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<Metr
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleYieldEvents,
|
||||
"Number of scanner self-throttle yield events in the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleYieldDurationSeconds,
|
||||
"Total scanner self-throttle yield duration in seconds for the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleIlmActions,
|
||||
"Number of lifecycle actions applied by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleHealObjects,
|
||||
"Number of object heal candidates enqueued by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleReplicationChecks,
|
||||
"Number of replication heal checks run by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentCycleUsageSaves,
|
||||
"Number of data-usage save operations run by the currently running scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentScanMode,
|
||||
@@ -215,6 +350,15 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD: LazyLock<MetricDescriptor>
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleBucketDriveFailures,
|
||||
"Number of bucket-drive scans that failed in the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleObjectsPerSecond,
|
||||
@@ -242,6 +386,60 @@ pub static SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD: LazyLock<MetricD
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_YIELD_EVENTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleYieldEvents,
|
||||
"Number of scanner self-throttle yield events in the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleYieldDurationSeconds,
|
||||
"Total scanner self-throttle yield duration in seconds for the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_ILM_ACTIONS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleIlmActions,
|
||||
"Number of lifecycle actions applied by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleHealObjects,
|
||||
"Number of object heal candidates enqueued by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleReplicationChecks,
|
||||
"Number of replication heal checks run by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_USAGE_SAVES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleUsageSaves,
|
||||
"Number of data-usage save operations run by the last finished scanner cycle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerFailedCycles,
|
||||
|
||||
@@ -896,10 +896,23 @@ pub async fn collect_ilm_metric_stats() -> Option<IlmStats> {
|
||||
///
|
||||
/// Task 5 maps scanner runtime snapshots from `global_metrics()` into the
|
||||
/// rustfs-obs scanner collector shape.
|
||||
fn scanner_bucket_scans_started(life_time_ops: &HashMap<String, u64>, bucket_scans_finished: u64) -> u64 {
|
||||
life_time_ops
|
||||
.get("scan_bucket_drive_start")
|
||||
.copied()
|
||||
.unwrap_or(bucket_scans_finished)
|
||||
}
|
||||
|
||||
pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
let metrics = global_metrics().report().await;
|
||||
let now = Utc::now();
|
||||
let bucket_scans_finished = metrics.life_time_ops.get("scan_bucket_drive").copied().unwrap_or_default();
|
||||
let bucket_scans_started = scanner_bucket_scans_started(&metrics.life_time_ops, bucket_scans_finished);
|
||||
let bucket_scans_failed = metrics
|
||||
.life_time_ops
|
||||
.get("scan_bucket_drive_failure")
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let completed_cycles = metrics.life_time_ops.get("scan_cycle").copied().unwrap_or_default();
|
||||
let directories_scanned = metrics.life_time_ops.get("scan_folder").copied().unwrap_or_default();
|
||||
let objects_scanned = metrics.life_time_ops.get("scan_object").copied().unwrap_or_default();
|
||||
@@ -914,21 +927,27 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
|
||||
Some(ScannerStats {
|
||||
bucket_scans_finished,
|
||||
// `global_metrics()` currently tracks completed bucket-drive scans, not a
|
||||
// separate started counter. Mirror the finished count until Task 5/Task 10
|
||||
// expands the scanner runtime source shape.
|
||||
bucket_scans_started: bucket_scans_finished,
|
||||
bucket_scans_started,
|
||||
bucket_scans_failed,
|
||||
directories_scanned,
|
||||
objects_scanned,
|
||||
versions_scanned,
|
||||
last_activity_seconds,
|
||||
active_paths,
|
||||
throttle_idle_mode_enabled: u64::from(metrics.throttle_idle_mode_enabled),
|
||||
throttle_sleep_factor: metrics.throttle_sleep_factor,
|
||||
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
|
||||
yield_every_n_objects: metrics.yield_every_n_objects,
|
||||
cycle_interval_seconds: metrics.cycle_interval_seconds,
|
||||
bitrot_cycle_enabled: u64::from(metrics.bitrot_cycle_enabled),
|
||||
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||
current_cycle: metrics.current_cycle,
|
||||
completed_cycles,
|
||||
current_cycle_age_seconds,
|
||||
current_cycle_objects_scanned: metrics.current_cycle_objects_scanned,
|
||||
current_cycle_directories_scanned: metrics.current_cycle_directories_scanned,
|
||||
current_cycle_bucket_drive_scans: metrics.current_cycle_bucket_drive_scans,
|
||||
current_cycle_bucket_drive_failures: metrics.current_cycle_bucket_drive_failures,
|
||||
current_cycle_objects_per_second: scanner_work_rate_per_second(metrics.current_cycle_objects_scanned, current_cycle_age),
|
||||
current_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||
metrics.current_cycle_directories_scanned,
|
||||
@@ -938,12 +957,19 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
metrics.current_cycle_bucket_drive_scans,
|
||||
current_cycle_age,
|
||||
),
|
||||
current_cycle_yield_events: metrics.current_cycle_yield_events,
|
||||
current_cycle_yield_duration_seconds: metrics.current_cycle_yield_duration_seconds,
|
||||
current_cycle_ilm_actions: metrics.current_cycle_ilm_actions,
|
||||
current_cycle_heal_objects: metrics.current_cycle_heal_objects,
|
||||
current_cycle_replication_checks: metrics.current_cycle_replication_checks,
|
||||
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
|
||||
current_scan_mode,
|
||||
last_cycle_result: metrics.last_cycle_result_code,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
last_cycle_bucket_drive_scans: metrics.last_cycle_bucket_drive_scans,
|
||||
last_cycle_bucket_drive_failures: metrics.last_cycle_bucket_drive_failures,
|
||||
last_cycle_objects_per_second: scanner_work_rate_per_second(metrics.last_cycle_objects_scanned, last_cycle_duration),
|
||||
last_cycle_directories_per_second: scanner_work_rate_per_second(
|
||||
metrics.last_cycle_directories_scanned,
|
||||
@@ -953,6 +979,12 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
metrics.last_cycle_bucket_drive_scans,
|
||||
last_cycle_duration,
|
||||
),
|
||||
last_cycle_yield_events: metrics.last_cycle_yield_events,
|
||||
last_cycle_yield_duration_seconds: metrics.last_cycle_yield_duration_seconds,
|
||||
last_cycle_ilm_actions: metrics.last_cycle_ilm_actions,
|
||||
last_cycle_heal_objects: metrics.last_cycle_heal_objects,
|
||||
last_cycle_replication_checks: metrics.last_cycle_replication_checks,
|
||||
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
|
||||
failed_cycles: metrics.failed_cycles,
|
||||
})
|
||||
}
|
||||
@@ -1056,6 +1088,21 @@ mod tests {
|
||||
assert_eq!(scanner_scan_mode_code(""), HealScanMode::Unknown as u8 as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_bucket_scans_started_uses_explicit_started_count() {
|
||||
let mut life_time_ops = HashMap::new();
|
||||
life_time_ops.insert("scan_bucket_drive_start".to_string(), 7);
|
||||
|
||||
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_bucket_scans_started_falls_back_to_finished_count() {
|
||||
let life_time_ops = HashMap::new();
|
||||
|
||||
assert_eq!(scanner_bucket_scans_started(&life_time_ops, 5), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_work_rate_per_second_reports_rate() {
|
||||
assert_eq!(scanner_work_rate_per_second(90, 45.0), 2.0);
|
||||
|
||||
Reference in New Issue
Block a user