mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 11:32:19 +00:00
feat(scanner): add scanner budget progress controls (#3185)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
@@ -30,18 +30,22 @@ use crate::metrics::schema::scanner::{
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD,
|
||||
SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD, SCANNER_CURRENT_SCAN_MODE_MD,
|
||||
SCANNER_CYCLE_INTERVAL_SECONDS_MD, SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, SCANNER_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD, SCANNER_LAST_CYCLE_ILM_ACTIONS_MD,
|
||||
SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD,
|
||||
SCANNER_LAST_CYCLE_RESULT_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD,
|
||||
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD,
|
||||
SCANNER_LAST_CYCLE_YIELD_EVENTS_MD, SCANNER_OBJECTS_SCANNED_MD, SCANNER_PARTIAL_CYCLES_MD,
|
||||
SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD,
|
||||
SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD,
|
||||
SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD, SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD,
|
||||
SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD,
|
||||
SCANNER_CURRENT_SET_SCANS_ACTIVE_MD, SCANNER_CURRENT_SET_SCANS_QUEUED_MD, SCANNER_CYCLE_INTERVAL_SECONDS_MD,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES_MD, SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, SCANNER_CYCLE_MAX_OBJECTS_MD,
|
||||
SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
|
||||
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_PARTIAL_REASON_MD, SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD,
|
||||
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
|
||||
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
|
||||
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
|
||||
SCANNER_PARTIAL_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD, SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD,
|
||||
SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD, SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||
};
|
||||
|
||||
/// Scanner statistics.
|
||||
@@ -63,6 +67,20 @@ pub struct ScannerStats {
|
||||
pub last_activity_seconds: u64,
|
||||
/// Number of scanner paths currently being processed
|
||||
pub active_paths: u64,
|
||||
/// Age in seconds of the oldest active scanner path update
|
||||
pub oldest_active_path_age_seconds: u64,
|
||||
/// Current aggregate set scan concurrency limit
|
||||
pub current_set_scan_concurrency_limit: u64,
|
||||
/// Current number of queued set scans
|
||||
pub current_set_scans_queued: u64,
|
||||
/// Current number of active set scans
|
||||
pub current_set_scans_active: u64,
|
||||
/// Current aggregate disk-bucket scan concurrency limit
|
||||
pub current_disk_scan_concurrency_limit: u64,
|
||||
/// Current number of queued disk-bucket scans
|
||||
pub current_disk_bucket_scans_queued: u64,
|
||||
/// Current number of active disk-bucket scans
|
||||
pub current_disk_bucket_scans_active: u64,
|
||||
/// Whether scanner idle-mode self-throttling is enabled
|
||||
pub throttle_idle_mode_enabled: bool,
|
||||
/// Effective scanner sleep factor
|
||||
@@ -75,6 +93,10 @@ pub struct ScannerStats {
|
||||
pub cycle_interval_seconds: f64,
|
||||
/// Effective maximum scanner cycle runtime in seconds
|
||||
pub cycle_max_duration_seconds: f64,
|
||||
/// Effective maximum objects processed by one scanner cycle
|
||||
pub cycle_max_objects: u64,
|
||||
/// Effective maximum directories entered by one scanner cycle
|
||||
pub cycle_max_directories: u64,
|
||||
/// Whether periodic scanner bitrot deep scans are enabled
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
/// Effective scanner bitrot deep-scan interval in seconds
|
||||
@@ -119,6 +141,8 @@ pub struct ScannerStats {
|
||||
pub current_scan_mode: u64,
|
||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial
|
||||
pub last_cycle_result: u64,
|
||||
/// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
|
||||
pub last_cycle_partial_reason: u64,
|
||||
/// Duration in seconds of the last finished scanner cycle
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
/// Number of objects scanned by the last finished scanner cycle
|
||||
@@ -155,6 +179,14 @@ pub struct ScannerStats {
|
||||
pub failed_cycles: u64,
|
||||
/// Number of scanner cycles stopped by runtime budget since server start
|
||||
pub partial_cycles: u64,
|
||||
/// Number of scanner cycles stopped by an unknown budget reason
|
||||
pub partial_cycles_unknown: u64,
|
||||
/// Number of scanner cycles stopped by runtime budget
|
||||
pub partial_cycles_runtime: u64,
|
||||
/// Number of scanner cycles stopped by object budget
|
||||
pub partial_cycles_objects: u64,
|
||||
/// Number of scanner cycles stopped by directory budget
|
||||
pub partial_cycles_directories: u64,
|
||||
}
|
||||
|
||||
/// Collects scanner metrics from the given stats.
|
||||
@@ -171,6 +203,28 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
PrometheusMetric::from_descriptor(&SCANNER_VERSIONS_SCANNED_MD, stats.versions_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_ACTIVITY_SECONDS_MD, stats.last_activity_seconds as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_ACTIVE_PATHS_MD, stats.active_paths as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD,
|
||||
stats.oldest_active_path_age_seconds as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD,
|
||||
stats.current_set_scan_concurrency_limit as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SET_SCANS_QUEUED_MD, stats.current_set_scans_queued as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SET_SCANS_ACTIVE_MD, stats.current_set_scans_active as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD,
|
||||
stats.current_disk_scan_concurrency_limit as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD,
|
||||
stats.current_disk_bucket_scans_queued as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD,
|
||||
stats.current_disk_bucket_scans_active as f64,
|
||||
),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
|
||||
bool_metric_value(stats.throttle_idle_mode_enabled),
|
||||
@@ -180,6 +234,8 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
PrometheusMetric::from_descriptor(&SCANNER_YIELD_EVERY_N_OBJECTS_MD, stats.yield_every_n_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_INTERVAL_SECONDS_MD, stats.cycle_interval_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, stats.cycle_max_duration_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_MAX_OBJECTS_MD, stats.cycle_max_objects as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CYCLE_MAX_DIRECTORIES_MD, stats.cycle_max_directories as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_ENABLED_MD, bool_metric_value(stats.bitrot_cycle_enabled)),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BITROT_CYCLE_SECONDS_MD, stats.bitrot_cycle_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_MD, stats.current_cycle as f64),
|
||||
@@ -229,6 +285,7 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD, stats.current_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_CURRENT_SCAN_MODE_MD, stats.current_scan_mode as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_RESULT_MD, stats.last_cycle_result as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_PARTIAL_REASON_MD, stats.last_cycle_partial_reason as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, stats.last_cycle_duration_seconds),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD, stats.last_cycle_objects_scanned as f64),
|
||||
PrometheusMetric::from_descriptor(
|
||||
@@ -262,6 +319,14 @@ pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
PrometheusMetric::from_descriptor(&SCANNER_LAST_CYCLE_USAGE_SAVES_MD, stats.last_cycle_usage_saves as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_FAILED_CYCLES_MD, stats.failed_cycles as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_MD, stats.partial_cycles as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_unknown as f64)
|
||||
.with_label("reason", "unknown"),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_runtime as f64)
|
||||
.with_label("reason", "runtime"),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_objects as f64)
|
||||
.with_label("reason", "objects"),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_directories as f64)
|
||||
.with_label("reason", "directories"),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -285,12 +350,21 @@ mod tests {
|
||||
versions_scanned: 1500000,
|
||||
last_activity_seconds: 30,
|
||||
active_paths: 4,
|
||||
oldest_active_path_age_seconds: 17,
|
||||
current_set_scan_concurrency_limit: 3,
|
||||
current_set_scans_queued: 5,
|
||||
current_set_scans_active: 2,
|
||||
current_disk_scan_concurrency_limit: 6,
|
||||
current_disk_bucket_scans_queued: 18,
|
||||
current_disk_bucket_scans_active: 4,
|
||||
throttle_idle_mode_enabled: true,
|
||||
throttle_sleep_factor: 10.0,
|
||||
throttle_max_sleep_seconds: 15.0,
|
||||
yield_every_n_objects: 128,
|
||||
cycle_interval_seconds: 3600.0,
|
||||
cycle_max_duration_seconds: 1800.0,
|
||||
cycle_max_objects: 1_000_000,
|
||||
cycle_max_directories: 100_000,
|
||||
bitrot_cycle_enabled: true,
|
||||
bitrot_cycle_seconds: 86400.0,
|
||||
current_cycle: 12,
|
||||
@@ -313,6 +387,7 @@ mod tests {
|
||||
current_cycle_usage_saves: 3,
|
||||
current_scan_mode: 2,
|
||||
last_cycle_result: 1,
|
||||
last_cycle_partial_reason: 3,
|
||||
last_cycle_duration_seconds: 42.5,
|
||||
last_cycle_objects_scanned: 900,
|
||||
last_cycle_directories_scanned: 80,
|
||||
@@ -330,13 +405,17 @@ mod tests {
|
||||
last_cycle_replication_checks: 12,
|
||||
last_cycle_usage_saves: 9,
|
||||
failed_cycles: 3,
|
||||
partial_cycles: 2,
|
||||
partial_cycles: 10,
|
||||
partial_cycles_unknown: 1,
|
||||
partial_cycles_runtime: 2,
|
||||
partial_cycles_objects: 3,
|
||||
partial_cycles_directories: 4,
|
||||
};
|
||||
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 54);
|
||||
assert_eq!(metrics.len(), 68);
|
||||
|
||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(objects.is_some());
|
||||
@@ -349,6 +428,41 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
||||
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
||||
|
||||
let oldest_active_path_age = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(oldest_active_path_age.map(|m| m.value), Some(17.0));
|
||||
|
||||
let current_set_scan_concurrency_limit = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD.get_full_metric_name());
|
||||
assert_eq!(current_set_scan_concurrency_limit.map(|m| m.value), Some(3.0));
|
||||
|
||||
let current_set_scans_queued = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_SET_SCANS_QUEUED_MD.get_full_metric_name());
|
||||
assert_eq!(current_set_scans_queued.map(|m| m.value), Some(5.0));
|
||||
|
||||
let current_set_scans_active = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_SET_SCANS_ACTIVE_MD.get_full_metric_name());
|
||||
assert_eq!(current_set_scans_active.map(|m| m.value), Some(2.0));
|
||||
|
||||
let current_disk_scan_concurrency_limit = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD.get_full_metric_name());
|
||||
assert_eq!(current_disk_scan_concurrency_limit.map(|m| m.value), Some(6.0));
|
||||
|
||||
let current_disk_bucket_scans_queued = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD.get_full_metric_name());
|
||||
assert_eq!(current_disk_bucket_scans_queued.map(|m| m.value), Some(18.0));
|
||||
|
||||
let current_disk_bucket_scans_active = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD.get_full_metric_name());
|
||||
assert_eq!(current_disk_bucket_scans_active.map(|m| m.value), Some(4.0));
|
||||
|
||||
let bucket_scans_failed = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BUCKET_SCANS_FAILED_MD.get_full_metric_name());
|
||||
@@ -384,6 +498,16 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_CYCLE_MAX_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
assert_eq!(cycle_max_duration_seconds.map(|m| m.value), Some(1800.0));
|
||||
|
||||
let cycle_max_objects = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CYCLE_MAX_OBJECTS_MD.get_full_metric_name());
|
||||
assert_eq!(cycle_max_objects.map(|m| m.value), Some(1_000_000.0));
|
||||
|
||||
let cycle_max_directories = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_CYCLE_MAX_DIRECTORIES_MD.get_full_metric_name());
|
||||
assert_eq!(cycle_max_directories.map(|m| m.value), Some(100_000.0));
|
||||
|
||||
let bitrot_cycle_enabled = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BITROT_CYCLE_ENABLED_MD.get_full_metric_name());
|
||||
@@ -494,6 +618,11 @@ mod tests {
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_RESULT_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_result.map(|m| m.value), Some(1.0));
|
||||
|
||||
let last_cycle_partial_reason = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_PARTIAL_REASON_MD.get_full_metric_name());
|
||||
assert_eq!(last_cycle_partial_reason.map(|m| m.value), Some(3.0));
|
||||
|
||||
let last_cycle_duration = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_LAST_CYCLE_DURATION_SECONDS_MD.get_full_metric_name());
|
||||
@@ -582,7 +711,31 @@ mod tests {
|
||||
let partial_cycles = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_PARTIAL_CYCLES_MD.get_full_metric_name());
|
||||
assert_eq!(partial_cycles.map(|m| m.value), Some(2.0));
|
||||
assert_eq!(partial_cycles.map(|m| m.value), Some(10.0));
|
||||
|
||||
let partial_cycles_runtime = metrics.iter().find(|m| {
|
||||
m.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == "reason" && value.as_ref() == "runtime")
|
||||
});
|
||||
assert_eq!(partial_cycles_runtime.map(|m| m.value), Some(2.0));
|
||||
|
||||
let partial_cycles_objects = metrics.iter().find(|m| {
|
||||
m.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == "reason" && value.as_ref() == "objects")
|
||||
});
|
||||
assert_eq!(partial_cycles_objects.map(|m| m.value), Some(3.0));
|
||||
|
||||
let partial_cycles_directories = metrics.iter().find(|m| {
|
||||
m.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == "reason" && value.as_ref() == "directories")
|
||||
});
|
||||
assert_eq!(partial_cycles_directories.map(|m| m.value), Some(4.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -590,10 +743,16 @@ mod tests {
|
||||
let stats = ScannerStats::default();
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 54);
|
||||
assert_eq!(metrics.len(), 68);
|
||||
for metric in &metrics {
|
||||
assert_eq!(metric.value, 0.0);
|
||||
assert!(metric.labels.is_empty());
|
||||
if metric.name == SCANNER_PARTIAL_CYCLES_BY_REASON_MD.get_full_metric_name() {
|
||||
assert_eq!(metric.labels.len(), 1);
|
||||
assert_eq!(metric.labels[0].0, "reason");
|
||||
assert!(matches!(metric.labels[0].1.as_ref(), "unknown" | "runtime" | "objects" | "directories"));
|
||||
} else {
|
||||
assert!(metric.labels.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,6 +279,13 @@ pub enum MetricName {
|
||||
ScannerVersionsScanned,
|
||||
ScannerLastActivitySeconds,
|
||||
ScannerActivePaths,
|
||||
ScannerOldestActivePathAgeSeconds,
|
||||
ScannerCurrentSetScanConcurrencyLimit,
|
||||
ScannerCurrentSetScansQueued,
|
||||
ScannerCurrentSetScansActive,
|
||||
ScannerCurrentDiskScanConcurrencyLimit,
|
||||
ScannerCurrentDiskBucketScansQueued,
|
||||
ScannerCurrentDiskBucketScansActive,
|
||||
ScannerBucketScansFailed,
|
||||
ScannerThrottleIdleModeEnabled,
|
||||
ScannerThrottleSleepFactor,
|
||||
@@ -286,6 +293,8 @@ pub enum MetricName {
|
||||
ScannerYieldEveryNObjects,
|
||||
ScannerCycleIntervalSeconds,
|
||||
ScannerCycleMaxDurationSeconds,
|
||||
ScannerCycleMaxObjects,
|
||||
ScannerCycleMaxDirectories,
|
||||
ScannerBitrotCycleEnabled,
|
||||
ScannerBitrotCycleSeconds,
|
||||
ScannerCurrentCycle,
|
||||
@@ -308,6 +317,7 @@ pub enum MetricName {
|
||||
ScannerCurrentCycleUsageSaves,
|
||||
ScannerCurrentScanMode,
|
||||
ScannerLastCycleResult,
|
||||
ScannerLastCyclePartialReason,
|
||||
ScannerLastCycleDurationSeconds,
|
||||
ScannerLastCycleObjectsScanned,
|
||||
ScannerLastCycleDirectoriesScanned,
|
||||
@@ -326,6 +336,7 @@ pub enum MetricName {
|
||||
ScannerLastCycleUsageSaves,
|
||||
ScannerFailedCycles,
|
||||
ScannerPartialCycles,
|
||||
ScannerPartialCyclesByReason,
|
||||
|
||||
// CPU system-related metrics
|
||||
SysCPUAvgIdle,
|
||||
@@ -667,6 +678,13 @@ impl MetricName {
|
||||
Self::ScannerVersionsScanned => "versions_scanned".to_string(),
|
||||
Self::ScannerLastActivitySeconds => "last_activity_seconds".to_string(),
|
||||
Self::ScannerActivePaths => "active_paths".to_string(),
|
||||
Self::ScannerOldestActivePathAgeSeconds => "oldest_active_path_age_seconds".to_string(),
|
||||
Self::ScannerCurrentSetScanConcurrencyLimit => "current_set_scan_concurrency_limit".to_string(),
|
||||
Self::ScannerCurrentSetScansQueued => "current_set_scans_queued".to_string(),
|
||||
Self::ScannerCurrentSetScansActive => "current_set_scans_active".to_string(),
|
||||
Self::ScannerCurrentDiskScanConcurrencyLimit => "current_disk_scan_concurrency_limit".to_string(),
|
||||
Self::ScannerCurrentDiskBucketScansQueued => "current_disk_bucket_scans_queued".to_string(),
|
||||
Self::ScannerCurrentDiskBucketScansActive => "current_disk_bucket_scans_active".to_string(),
|
||||
Self::ScannerBucketScansFailed => "bucket_scans_failed".to_string(),
|
||||
Self::ScannerThrottleIdleModeEnabled => "throttle_idle_mode_enabled".to_string(),
|
||||
Self::ScannerThrottleSleepFactor => "throttle_sleep_factor".to_string(),
|
||||
@@ -674,6 +692,8 @@ impl MetricName {
|
||||
Self::ScannerYieldEveryNObjects => "yield_every_n_objects".to_string(),
|
||||
Self::ScannerCycleIntervalSeconds => "cycle_interval_seconds".to_string(),
|
||||
Self::ScannerCycleMaxDurationSeconds => "cycle_max_duration_seconds".to_string(),
|
||||
Self::ScannerCycleMaxObjects => "cycle_max_objects".to_string(),
|
||||
Self::ScannerCycleMaxDirectories => "cycle_max_directories".to_string(),
|
||||
Self::ScannerBitrotCycleEnabled => "bitrot_cycle_enabled".to_string(),
|
||||
Self::ScannerBitrotCycleSeconds => "bitrot_cycle_seconds".to_string(),
|
||||
Self::ScannerCurrentCycle => "current_cycle".to_string(),
|
||||
@@ -696,6 +716,7 @@ impl MetricName {
|
||||
Self::ScannerCurrentCycleUsageSaves => "current_cycle_usage_saves".to_string(),
|
||||
Self::ScannerCurrentScanMode => "current_scan_mode".to_string(),
|
||||
Self::ScannerLastCycleResult => "last_cycle_result".to_string(),
|
||||
Self::ScannerLastCyclePartialReason => "last_cycle_partial_reason".to_string(),
|
||||
Self::ScannerLastCycleDurationSeconds => "last_cycle_duration_seconds".to_string(),
|
||||
Self::ScannerLastCycleObjectsScanned => "last_cycle_objects_scanned".to_string(),
|
||||
Self::ScannerLastCycleDirectoriesScanned => "last_cycle_directories_scanned".to_string(),
|
||||
@@ -714,6 +735,7 @@ impl MetricName {
|
||||
Self::ScannerLastCycleUsageSaves => "last_cycle_usage_saves".to_string(),
|
||||
Self::ScannerFailedCycles => "failed_cycles".to_string(),
|
||||
Self::ScannerPartialCycles => "partial_cycles".to_string(),
|
||||
Self::ScannerPartialCyclesByReason => "partial_cycles_by_reason".to_string(),
|
||||
|
||||
// CPU system-related metrics
|
||||
Self::SysCPUAvgIdle => "avg_idle".to_string(),
|
||||
|
||||
@@ -89,6 +89,69 @@ pub static SCANNER_ACTIVE_PATHS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerOldestActivePathAgeSeconds,
|
||||
"Time elapsed (in seconds) since the oldest active scanner path was last updated, or zero when idle.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentSetScanConcurrencyLimit,
|
||||
"Current aggregate scanner set scan concurrency limit across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_SET_SCANS_QUEUED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentSetScansQueued,
|
||||
"Current number of queued scanner set scans across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_SET_SCANS_ACTIVE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentSetScansActive,
|
||||
"Current number of active scanner set scans across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentDiskScanConcurrencyLimit,
|
||||
"Current aggregate scanner disk-bucket scan concurrency limit across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentDiskBucketScansQueued,
|
||||
"Current number of queued scanner disk-bucket scans across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCurrentDiskBucketScansActive,
|
||||
"Current number of active scanner disk-bucket scans across active scanner work.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerThrottleIdleModeEnabled,
|
||||
@@ -143,6 +206,24 @@ pub static SCANNER_CYCLE_MAX_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = L
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_MAX_OBJECTS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCycleMaxObjects,
|
||||
"Effective maximum objects processed by one scanner cycle; zero means unlimited.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_MAX_DIRECTORIES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerCycleMaxDirectories,
|
||||
"Effective maximum directories entered by one scanner cycle; zero means unlimited.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BITROT_CYCLE_ENABLED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerBitrotCycleEnabled,
|
||||
@@ -341,6 +422,15 @@ pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_PARTIAL_REASON_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCyclePartialReason,
|
||||
"Last scanner partial cycle reason: 0 unknown, 1 runtime budget, 2 object budget, 3 directory budget.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_LAST_CYCLE_DURATION_SECONDS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleDurationSeconds,
|
||||
@@ -497,8 +587,17 @@ pub static SCANNER_FAILED_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(
|
||||
pub static SCANNER_PARTIAL_CYCLES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerPartialCycles,
|
||||
"Total number of scanner cycles stopped before completion by runtime budget.",
|
||||
"Total number of scanner cycles stopped before completion by cycle budget.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_PARTIAL_CYCLES_BY_REASON_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerPartialCyclesByReason,
|
||||
"Total number of scanner cycles stopped before completion by cycle budget reason.",
|
||||
&["reason"],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -934,12 +934,21 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
versions_scanned,
|
||||
last_activity_seconds,
|
||||
active_paths,
|
||||
oldest_active_path_age_seconds: metrics.oldest_active_path_age_seconds,
|
||||
current_set_scan_concurrency_limit: metrics.current_set_scan_concurrency_limit,
|
||||
current_set_scans_queued: metrics.current_set_scans_queued,
|
||||
current_set_scans_active: metrics.current_set_scans_active,
|
||||
current_disk_scan_concurrency_limit: metrics.current_disk_scan_concurrency_limit,
|
||||
current_disk_bucket_scans_queued: metrics.current_disk_bucket_scans_queued,
|
||||
current_disk_bucket_scans_active: metrics.current_disk_bucket_scans_active,
|
||||
throttle_idle_mode_enabled: metrics.throttle_idle_mode_enabled,
|
||||
throttle_sleep_factor: metrics.throttle_sleep_factor,
|
||||
throttle_max_sleep_seconds: metrics.throttle_max_sleep_seconds,
|
||||
yield_every_n_objects: metrics.yield_every_n_objects,
|
||||
cycle_interval_seconds: metrics.cycle_interval_seconds,
|
||||
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
|
||||
cycle_max_objects: metrics.cycle_max_objects,
|
||||
cycle_max_directories: metrics.cycle_max_directories,
|
||||
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
|
||||
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||
current_cycle: metrics.current_cycle,
|
||||
@@ -968,6 +977,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
current_cycle_usage_saves: metrics.current_cycle_usage_saves,
|
||||
current_scan_mode,
|
||||
last_cycle_result: metrics.last_cycle_result_code,
|
||||
last_cycle_partial_reason: metrics.last_cycle_partial_reason_code,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
@@ -992,6 +1002,10 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
|
||||
last_cycle_usage_saves: metrics.last_cycle_usage_saves,
|
||||
failed_cycles: metrics.failed_cycles,
|
||||
partial_cycles: metrics.partial_cycles,
|
||||
partial_cycles_unknown: metrics.partial_cycles_unknown,
|
||||
partial_cycles_runtime: metrics.partial_cycles_runtime,
|
||||
partial_cycles_objects: metrics.partial_cycles_objects,
|
||||
partial_cycles_directories: metrics.partial_cycles_directories,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user