feat(scanner): expose usage freshness status (#3577)

* feat(scanner): expose usage freshness status

* fix(scanner): reset dirty usage cycle state

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-18 21:14:55 +08:00
committed by GitHub
parent 5d9fee5c0c
commit 80144ec886
6 changed files with 291 additions and 24 deletions
+151
View File
@@ -753,6 +753,13 @@ pub struct Metrics {
scanner_checkpoint_cleared: AtomicU64,
scanner_checkpoint_ignored: AtomicU64,
scanner_checkpoint_stale: AtomicU64,
scanner_dirty_usage_pending_buckets: AtomicU64,
scanner_dirty_usage_last_mark_unix_secs: AtomicU64,
scanner_dirty_usage_last_clear_unix_secs: AtomicU64,
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
scanner_usage_last_save_unix_secs: AtomicU64,
scanner_usage_last_save_result: AtomicU8,
scanner_source_work: Vec<ScannerSourceWorkCounters>,
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
@@ -800,6 +807,38 @@ impl ScanCyclePartialReason {
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum ScannerUsageSaveResult {
#[default]
Unknown = 0,
Success = 1,
Failed = 2,
SkippedStale = 3,
EncodeFailed = 4,
}
impl ScannerUsageSaveResult {
pub fn as_str(self) -> &'static str {
match self {
Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
Self::Success => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
Self::Failed => "failed",
Self::SkippedStale => "skipped_stale",
Self::EncodeFailed => "encode_failed",
}
}
fn from_code(code: u8) -> Self {
match code {
1 => Self::Success,
2 => Self::Failed,
3 => Self::SkippedStale,
4 => Self::EncodeFailed,
_ => Self::Unknown,
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct CurrentCycle {
pub current: u64,
@@ -977,6 +1016,18 @@ pub struct ScannerLifecycleTransitionSnapshot {
pub failed: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerUsageFreshnessSnapshot {
pub dirty_pending_buckets: u64,
pub last_dirty_mark_unix_secs: u64,
pub last_dirty_clear_unix_secs: u64,
pub last_cycle_dirty_buckets: u64,
pub last_cycle_cleared_dirty_buckets: u64,
pub last_usage_save_unix_secs: u64,
pub last_usage_save_result: String,
pub last_usage_save_result_code: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerLastMinute {
pub actions: HashMap<String, ScannerTimedAction>,
@@ -1096,6 +1147,8 @@ pub struct ScannerMetricsReport {
#[serde(default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub usage_freshness: ScannerUsageFreshnessSnapshot,
#[serde(default)]
pub maintenance_control: ScannerMaintenanceControlSnapshot,
#[serde(default)]
pub throttle_idle_mode_enabled: bool,
@@ -1192,6 +1245,13 @@ fn scanner_ratio(numerator: f64, denominator: f64) -> f64 {
}
}
fn unix_now_secs() -> u64 {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn scanner_last_cycle_budget_limited(result_code: u64, partial_reason: &str) -> bool {
result_code == u64::from(SCAN_CYCLE_RESULT_PARTIAL) && matches!(partial_reason, "runtime" | "objects" | "directories")
}
@@ -1628,6 +1688,13 @@ impl Metrics {
scanner_checkpoint_cleared: AtomicU64::new(0),
scanner_checkpoint_ignored: AtomicU64::new(0),
scanner_checkpoint_stale: AtomicU64::new(0),
scanner_dirty_usage_pending_buckets: AtomicU64::new(0),
scanner_dirty_usage_last_mark_unix_secs: AtomicU64::new(0),
scanner_dirty_usage_last_clear_unix_secs: AtomicU64::new(0),
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
scanner_usage_last_save_unix_secs: AtomicU64::new(0),
scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
scanner_source_work: ScannerWorkSource::all()
.iter()
.map(|_| ScannerSourceWorkCounters::default())
@@ -1909,6 +1976,46 @@ impl Metrics {
self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_STALE);
}
pub fn record_scanner_dirty_usage_pending(&self, pending_buckets: u64) {
self.scanner_dirty_usage_pending_buckets
.store(pending_buckets, Ordering::Relaxed);
if pending_buckets > 0 {
self.scanner_dirty_usage_last_mark_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
}
}
pub fn record_scanner_dirty_usage_clear(&self, pending_buckets: u64) {
self.scanner_dirty_usage_pending_buckets
.store(pending_buckets, Ordering::Relaxed);
self.scanner_dirty_usage_last_clear_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
}
pub fn record_scanner_dirty_usage_cycle_snapshot(&self, dirty_buckets: u64) {
self.scanner_dirty_usage_last_cycle_dirty_buckets
.store(dirty_buckets, Ordering::Relaxed);
self.scanner_dirty_usage_last_cycle_cleared_buckets
.store(0, Ordering::Relaxed);
}
pub fn record_scanner_dirty_usage_cycle_clear(&self, cleared_buckets: u64, pending_buckets: u64) {
self.scanner_dirty_usage_last_cycle_cleared_buckets
.store(cleared_buckets, Ordering::Relaxed);
self.scanner_dirty_usage_pending_buckets
.store(pending_buckets, Ordering::Relaxed);
if cleared_buckets > 0 {
self.scanner_dirty_usage_last_clear_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
}
}
pub fn record_scanner_usage_save_result(&self, result: ScannerUsageSaveResult) {
self.scanner_usage_last_save_result.store(result as u8, Ordering::Relaxed);
self.scanner_usage_last_save_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
}
pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
if let Some(counters) = self.scanner_source_work.get(source.index()) {
counters.add(work);
@@ -2607,6 +2714,17 @@ impl Metrics {
completed: self.scanner_transition_completed.load(Ordering::Relaxed),
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
m.usage_freshness = ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
last_dirty_mark_unix_secs: self.scanner_dirty_usage_last_mark_unix_secs.load(Ordering::Relaxed),
last_dirty_clear_unix_secs: self.scanner_dirty_usage_last_clear_unix_secs.load(Ordering::Relaxed),
last_cycle_dirty_buckets: self.scanner_dirty_usage_last_cycle_dirty_buckets.load(Ordering::Relaxed),
last_cycle_cleared_dirty_buckets: self.scanner_dirty_usage_last_cycle_cleared_buckets.load(Ordering::Relaxed),
last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
last_usage_save_result: usage_save_result.as_str().to_string(),
last_usage_save_result_code: usage_save_result as u8 as u64,
};
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
@@ -3538,6 +3656,39 @@ mod tests {
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
}
#[tokio::test]
async fn report_includes_usage_freshness_status() {
let metrics = Metrics::new();
metrics.record_scanner_dirty_usage_pending(2);
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
let report = metrics.report().await;
assert_eq!(report.usage_freshness.dirty_pending_buckets, 1);
assert!(report.usage_freshness.last_dirty_mark_unix_secs > 0);
assert!(report.usage_freshness.last_dirty_clear_unix_secs > 0);
assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
assert_eq!(report.usage_freshness.last_usage_save_result, "success");
assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
}
#[tokio::test]
async fn dirty_usage_cycle_snapshot_resets_stale_cleared_count() {
let metrics = Metrics::new();
metrics.record_scanner_dirty_usage_cycle_snapshot(2);
metrics.record_scanner_dirty_usage_cycle_clear(2, 0);
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
let report = metrics.report().await;
assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 0);
}
#[tokio::test]
async fn report_includes_active_scan_cycle_work() {
let metrics = Metrics::new();
+37 -1
View File
@@ -26,7 +26,7 @@ use rustfs_madmin::metrics::{
ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
ScannerReplicationRepairSnapshot as MadminScannerReplicationRepairSnapshot,
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
TimedAction as MadminTimedAction,
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
};
use rustfs_storage_api::StorageAdminApi;
use rustfs_utils::os::get_drive_stats;
@@ -191,6 +191,16 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
scanner_queued: metrics.lifecycle_expiry.scanner_queued,
scanner_missed: metrics.lifecycle_expiry.scanner_missed,
},
usage_freshness: MadminScannerUsageFreshnessSnapshot {
dirty_pending_buckets: metrics.usage_freshness.dirty_pending_buckets,
last_dirty_mark_unix_secs: metrics.usage_freshness.last_dirty_mark_unix_secs,
last_dirty_clear_unix_secs: metrics.usage_freshness.last_dirty_clear_unix_secs,
last_cycle_dirty_buckets: metrics.usage_freshness.last_cycle_dirty_buckets,
last_cycle_cleared_dirty_buckets: metrics.usage_freshness.last_cycle_cleared_dirty_buckets,
last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
},
maintenance_control: MadminScannerMaintenanceControlSnapshot {
primary_control: metrics.maintenance_control.primary_control,
sources: metrics
@@ -710,6 +720,32 @@ mod test {
assert_eq!(lifecycle.partial_cycles, 5);
}
#[test]
fn scanner_metrics_mapping_preserves_usage_freshness_status() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
usage_freshness: rustfs_common::metrics::ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: 3,
last_dirty_mark_unix_secs: 10,
last_dirty_clear_unix_secs: 11,
last_cycle_dirty_buckets: 2,
last_cycle_cleared_dirty_buckets: 1,
last_usage_save_unix_secs: 12,
last_usage_save_result: "success".to_string(),
last_usage_save_result_code: 1,
},
..Default::default()
});
assert_eq!(scanner.usage_freshness.dirty_pending_buckets, 3);
assert_eq!(scanner.usage_freshness.last_dirty_mark_unix_secs, 10);
assert_eq!(scanner.usage_freshness.last_dirty_clear_unix_secs, 11);
assert_eq!(scanner.usage_freshness.last_cycle_dirty_buckets, 2);
assert_eq!(scanner.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
}
#[test]
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
+40
View File
@@ -280,6 +280,43 @@ impl ScannerMaintenanceControlSnapshot {
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerUsageFreshnessSnapshot {
#[serde(rename = "dirty_pending_buckets", default)]
pub dirty_pending_buckets: u64,
#[serde(rename = "last_dirty_mark_unix_secs", default)]
pub last_dirty_mark_unix_secs: u64,
#[serde(rename = "last_dirty_clear_unix_secs", default)]
pub last_dirty_clear_unix_secs: u64,
#[serde(rename = "last_cycle_dirty_buckets", default)]
pub last_cycle_dirty_buckets: u64,
#[serde(rename = "last_cycle_cleared_dirty_buckets", default)]
pub last_cycle_cleared_dirty_buckets: u64,
#[serde(rename = "last_usage_save_unix_secs", default)]
pub last_usage_save_unix_secs: u64,
#[serde(rename = "last_usage_save_result", default)]
pub last_usage_save_result: String,
#[serde(rename = "last_usage_save_result_code", default)]
pub last_usage_save_result_code: u64,
}
impl ScannerUsageFreshnessSnapshot {
fn merge(&mut self, other: &Self) {
self.dirty_pending_buckets = self.dirty_pending_buckets.saturating_add(other.dirty_pending_buckets);
self.last_dirty_mark_unix_secs = self.last_dirty_mark_unix_secs.max(other.last_dirty_mark_unix_secs);
self.last_dirty_clear_unix_secs = self.last_dirty_clear_unix_secs.max(other.last_dirty_clear_unix_secs);
self.last_cycle_dirty_buckets = self.last_cycle_dirty_buckets.saturating_add(other.last_cycle_dirty_buckets);
self.last_cycle_cleared_dirty_buckets = self
.last_cycle_cleared_dirty_buckets
.saturating_add(other.last_cycle_cleared_dirty_buckets);
if other.last_usage_save_unix_secs > self.last_usage_save_unix_secs {
self.last_usage_save_unix_secs = other.last_usage_save_unix_secs;
self.last_usage_save_result = other.last_usage_save_result.clone();
self.last_usage_save_result_code = other.last_usage_save_result_code;
}
}
}
impl ScannerSourceWorkSnapshot {
fn merge(&mut self, other: &Self) {
self.checked = self.checked.saturating_add(other.checked);
@@ -609,6 +646,8 @@ pub struct ScannerMetrics {
pub lifecycle_expiry: ScannerLifecycleExpirySnapshot,
#[serde(rename = "lifecycle_transition", default)]
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(rename = "usage_freshness", default)]
pub usage_freshness: ScannerUsageFreshnessSnapshot,
#[serde(rename = "maintenance_control", default)]
pub maintenance_control: ScannerMaintenanceControlSnapshot,
#[serde(rename = "throttle_idle_mode_enabled", default)]
@@ -690,6 +729,7 @@ impl ScannerMetrics {
self.pacing_pressure.merge(&other.pacing_pressure);
self.lifecycle_expiry.merge(&other.lifecycle_expiry);
self.lifecycle_transition.merge(&other.lifecycle_transition);
self.usage_freshness.merge(&other.usage_freshness);
self.maintenance_control.merge(&other.maintenance_control);
self.current_set_scan_concurrency_limit = self
.current_set_scan_concurrency_limit
+5 -1
View File
@@ -29,7 +29,7 @@ use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerWorkSource, emit_scan_cycle_complete,
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
emit_scan_cycle_partial_with_source, global_metrics,
};
use rustfs_config::ScannerSpeed;
@@ -1001,6 +1001,7 @@ pub async fn store_data_usage_in_backend(
state = "skip_stale_update",
"Scanner stale data usage update skipped"
);
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
continue;
}
@@ -1018,6 +1019,7 @@ pub async fn store_data_usage_in_backend(
error = %e,
"Scanner data usage encode failed"
);
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed);
continue;
}
};
@@ -1055,8 +1057,10 @@ pub async fn store_data_usage_in_backend(
error = %e,
"Scanner data usage save failed"
);
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed);
} else {
replace_bucket_usage_memory_from_info(&data_usage_info).await;
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
}
done_save();
+41 -20
View File
@@ -91,13 +91,22 @@ fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn usize_to_u64_saturated(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
pub fn record_dirty_usage_bucket(bucket: &str) {
if bucket.is_empty() {
return;
}
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
dirty_usage_buckets().insert(bucket.to_string(), generation);
let pending_buckets = {
let mut dirty_buckets = dirty_usage_buckets();
dirty_buckets.insert(bucket.to_string(), generation);
dirty_buckets.len()
};
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
}
pub fn clear_dirty_usage_bucket(bucket: &str) {
@@ -105,32 +114,44 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
return;
}
dirty_usage_buckets().remove(bucket);
let pending_buckets = {
let mut dirty_buckets = dirty_usage_buckets();
dirty_buckets.remove(bucket);
dirty_buckets.len()
};
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
}
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo]) -> DirtyUsageBuckets {
let dirty_buckets = dirty_usage_buckets();
buckets
.iter()
.filter_map(|bucket| {
dirty_buckets
.get(&bucket.name)
.map(|generation| (bucket.name.clone(), *generation))
})
.collect()
let snapshot = {
let dirty_buckets = dirty_usage_buckets();
buckets
.iter()
.filter_map(|bucket| {
dirty_buckets
.get(&bucket.name)
.map(|generation| (bucket.name.clone(), *generation))
})
.collect::<DirtyUsageBuckets>()
};
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
snapshot
}
fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
if snapshot.is_empty() {
return;
}
let mut dirty_buckets = dirty_usage_buckets();
for (bucket, generation) in snapshot {
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
dirty_buckets.remove(bucket);
let (cleared_buckets, pending_buckets) = {
let mut dirty_buckets = dirty_usage_buckets();
let mut cleared_buckets = 0usize;
for (bucket, generation) in snapshot {
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
dirty_buckets.remove(bucket);
cleared_buckets += 1;
}
}
}
(cleared_buckets, dirty_buckets.len())
};
global_metrics()
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
}
#[cfg(test)]
+17 -2
View File
@@ -70,8 +70,23 @@ fn materialize_object(base: String, extra: &str, flags: &[String]) -> String {
fn has_dot_segments(path: &str) -> bool {
path.split(['/', '\\']).any(|segment| {
let trimmed = segment.trim();
trimmed == "." || trimmed == ".."
let mut bytes = segment.as_bytes();
// Match ecstore's path-component check. `str::trim()` also trims
// Unicode whitespace such as vertical tab, which would reject object
// keys that the production validator accepts as ordinary path bytes.
while let Some((first, rest)) = bytes.split_first()
&& first.is_ascii_whitespace()
{
bytes = rest;
}
while let Some((last, rest)) = bytes.split_last()
&& last.is_ascii_whitespace()
{
bytes = rest;
}
bytes == b"." || bytes == b".."
})
}