mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
feat(scanner): expose usage freshness status (#3577)
* feat(scanner): expose usage freshness status * fix(scanner): reset dirty usage cycle state --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -753,6 +753,13 @@ pub struct Metrics {
|
||||
scanner_checkpoint_cleared: AtomicU64,
|
||||
scanner_checkpoint_ignored: AtomicU64,
|
||||
scanner_checkpoint_stale: AtomicU64,
|
||||
scanner_dirty_usage_pending_buckets: AtomicU64,
|
||||
scanner_dirty_usage_last_mark_unix_secs: AtomicU64,
|
||||
scanner_dirty_usage_last_clear_unix_secs: AtomicU64,
|
||||
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64,
|
||||
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64,
|
||||
scanner_usage_last_save_unix_secs: AtomicU64,
|
||||
scanner_usage_last_save_result: AtomicU8,
|
||||
scanner_source_work: Vec<ScannerSourceWorkCounters>,
|
||||
current_scan_cycle_source_work_start: Vec<ScannerSourceWorkCounters>,
|
||||
last_scan_cycle_source_work: Vec<ScannerSourceWorkCounters>,
|
||||
@@ -800,6 +807,38 @@ impl ScanCyclePartialReason {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum ScannerUsageSaveResult {
|
||||
#[default]
|
||||
Unknown = 0,
|
||||
Success = 1,
|
||||
Failed = 2,
|
||||
SkippedStale = 3,
|
||||
EncodeFailed = 4,
|
||||
}
|
||||
|
||||
impl ScannerUsageSaveResult {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
||||
Self::Success => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
|
||||
Self::Failed => "failed",
|
||||
Self::SkippedStale => "skipped_stale",
|
||||
Self::EncodeFailed => "encode_failed",
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Self {
|
||||
match code {
|
||||
1 => Self::Success,
|
||||
2 => Self::Failed,
|
||||
3 => Self::SkippedStale,
|
||||
4 => Self::EncodeFailed,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CurrentCycle {
|
||||
pub current: u64,
|
||||
@@ -977,6 +1016,18 @@ pub struct ScannerLifecycleTransitionSnapshot {
|
||||
pub failed: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerUsageFreshnessSnapshot {
|
||||
pub dirty_pending_buckets: u64,
|
||||
pub last_dirty_mark_unix_secs: u64,
|
||||
pub last_dirty_clear_unix_secs: u64,
|
||||
pub last_cycle_dirty_buckets: u64,
|
||||
pub last_cycle_cleared_dirty_buckets: u64,
|
||||
pub last_usage_save_unix_secs: u64,
|
||||
pub last_usage_save_result: String,
|
||||
pub last_usage_save_result_code: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerLastMinute {
|
||||
pub actions: HashMap<String, ScannerTimedAction>,
|
||||
@@ -1096,6 +1147,8 @@ pub struct ScannerMetricsReport {
|
||||
#[serde(default)]
|
||||
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
|
||||
#[serde(default)]
|
||||
pub usage_freshness: ScannerUsageFreshnessSnapshot,
|
||||
#[serde(default)]
|
||||
pub maintenance_control: ScannerMaintenanceControlSnapshot,
|
||||
#[serde(default)]
|
||||
pub throttle_idle_mode_enabled: bool,
|
||||
@@ -1192,6 +1245,13 @@ fn scanner_ratio(numerator: f64, denominator: f64) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_now_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
fn scanner_last_cycle_budget_limited(result_code: u64, partial_reason: &str) -> bool {
|
||||
result_code == u64::from(SCAN_CYCLE_RESULT_PARTIAL) && matches!(partial_reason, "runtime" | "objects" | "directories")
|
||||
}
|
||||
@@ -1628,6 +1688,13 @@ impl Metrics {
|
||||
scanner_checkpoint_cleared: AtomicU64::new(0),
|
||||
scanner_checkpoint_ignored: AtomicU64::new(0),
|
||||
scanner_checkpoint_stale: AtomicU64::new(0),
|
||||
scanner_dirty_usage_pending_buckets: AtomicU64::new(0),
|
||||
scanner_dirty_usage_last_mark_unix_secs: AtomicU64::new(0),
|
||||
scanner_dirty_usage_last_clear_unix_secs: AtomicU64::new(0),
|
||||
scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0),
|
||||
scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0),
|
||||
scanner_usage_last_save_unix_secs: AtomicU64::new(0),
|
||||
scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8),
|
||||
scanner_source_work: ScannerWorkSource::all()
|
||||
.iter()
|
||||
.map(|_| ScannerSourceWorkCounters::default())
|
||||
@@ -1909,6 +1976,46 @@ impl Metrics {
|
||||
self.update_scanner_checkpoint_event(SCANNER_CHECKPOINT_EVENT_STALE);
|
||||
}
|
||||
|
||||
pub fn record_scanner_dirty_usage_pending(&self, pending_buckets: u64) {
|
||||
self.scanner_dirty_usage_pending_buckets
|
||||
.store(pending_buckets, Ordering::Relaxed);
|
||||
if pending_buckets > 0 {
|
||||
self.scanner_dirty_usage_last_mark_unix_secs
|
||||
.store(unix_now_secs(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scanner_dirty_usage_clear(&self, pending_buckets: u64) {
|
||||
self.scanner_dirty_usage_pending_buckets
|
||||
.store(pending_buckets, Ordering::Relaxed);
|
||||
self.scanner_dirty_usage_last_clear_unix_secs
|
||||
.store(unix_now_secs(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_dirty_usage_cycle_snapshot(&self, dirty_buckets: u64) {
|
||||
self.scanner_dirty_usage_last_cycle_dirty_buckets
|
||||
.store(dirty_buckets, Ordering::Relaxed);
|
||||
self.scanner_dirty_usage_last_cycle_cleared_buckets
|
||||
.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_dirty_usage_cycle_clear(&self, cleared_buckets: u64, pending_buckets: u64) {
|
||||
self.scanner_dirty_usage_last_cycle_cleared_buckets
|
||||
.store(cleared_buckets, Ordering::Relaxed);
|
||||
self.scanner_dirty_usage_pending_buckets
|
||||
.store(pending_buckets, Ordering::Relaxed);
|
||||
if cleared_buckets > 0 {
|
||||
self.scanner_dirty_usage_last_clear_unix_secs
|
||||
.store(unix_now_secs(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scanner_usage_save_result(&self, result: ScannerUsageSaveResult) {
|
||||
self.scanner_usage_last_save_result.store(result as u8, Ordering::Relaxed);
|
||||
self.scanner_usage_last_save_unix_secs
|
||||
.store(unix_now_secs(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) {
|
||||
if let Some(counters) = self.scanner_source_work.get(source.index()) {
|
||||
counters.add(work);
|
||||
@@ -2607,6 +2714,17 @@ impl Metrics {
|
||||
completed: self.scanner_transition_completed.load(Ordering::Relaxed),
|
||||
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
|
||||
};
|
||||
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
|
||||
m.usage_freshness = ScannerUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
|
||||
last_dirty_mark_unix_secs: self.scanner_dirty_usage_last_mark_unix_secs.load(Ordering::Relaxed),
|
||||
last_dirty_clear_unix_secs: self.scanner_dirty_usage_last_clear_unix_secs.load(Ordering::Relaxed),
|
||||
last_cycle_dirty_buckets: self.scanner_dirty_usage_last_cycle_dirty_buckets.load(Ordering::Relaxed),
|
||||
last_cycle_cleared_dirty_buckets: self.scanner_dirty_usage_last_cycle_cleared_buckets.load(Ordering::Relaxed),
|
||||
last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed),
|
||||
last_usage_save_result: usage_save_result.as_str().to_string(),
|
||||
last_usage_save_result_code: usage_save_result as u8 as u64,
|
||||
};
|
||||
m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed);
|
||||
m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0;
|
||||
m.throttle_max_sleep_seconds = self.scanner_throttle_max_sleep_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
@@ -3538,6 +3656,39 @@ mod tests {
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_usage_freshness_status() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_dirty_usage_pending(2);
|
||||
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
|
||||
metrics.record_scanner_dirty_usage_cycle_clear(1, 1);
|
||||
metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.usage_freshness.dirty_pending_buckets, 1);
|
||||
assert!(report.usage_freshness.last_dirty_mark_unix_secs > 0);
|
||||
assert!(report.usage_freshness.last_dirty_clear_unix_secs > 0);
|
||||
assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
|
||||
assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
|
||||
assert!(report.usage_freshness.last_usage_save_unix_secs > 0);
|
||||
assert_eq!(report.usage_freshness.last_usage_save_result, "success");
|
||||
assert_eq!(report.usage_freshness.last_usage_save_result_code, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dirty_usage_cycle_snapshot_resets_stale_cleared_count() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_dirty_usage_cycle_snapshot(2);
|
||||
metrics.record_scanner_dirty_usage_cycle_clear(2, 0);
|
||||
metrics.record_scanner_dirty_usage_cycle_snapshot(1);
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.usage_freshness.last_cycle_dirty_buckets, 1);
|
||||
assert_eq!(report.usage_freshness.last_cycle_cleared_dirty_buckets, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_active_scan_cycle_work() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -26,7 +26,7 @@ use rustfs_madmin::metrics::{
|
||||
ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
|
||||
ScannerReplicationRepairSnapshot as MadminScannerReplicationRepairSnapshot,
|
||||
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
|
||||
TimedAction as MadminTimedAction,
|
||||
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
|
||||
};
|
||||
use rustfs_storage_api::StorageAdminApi;
|
||||
use rustfs_utils::os::get_drive_stats;
|
||||
@@ -191,6 +191,16 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
scanner_queued: metrics.lifecycle_expiry.scanner_queued,
|
||||
scanner_missed: metrics.lifecycle_expiry.scanner_missed,
|
||||
},
|
||||
usage_freshness: MadminScannerUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: metrics.usage_freshness.dirty_pending_buckets,
|
||||
last_dirty_mark_unix_secs: metrics.usage_freshness.last_dirty_mark_unix_secs,
|
||||
last_dirty_clear_unix_secs: metrics.usage_freshness.last_dirty_clear_unix_secs,
|
||||
last_cycle_dirty_buckets: metrics.usage_freshness.last_cycle_dirty_buckets,
|
||||
last_cycle_cleared_dirty_buckets: metrics.usage_freshness.last_cycle_cleared_dirty_buckets,
|
||||
last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs,
|
||||
last_usage_save_result: metrics.usage_freshness.last_usage_save_result,
|
||||
last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code,
|
||||
},
|
||||
maintenance_control: MadminScannerMaintenanceControlSnapshot {
|
||||
primary_control: metrics.maintenance_control.primary_control,
|
||||
sources: metrics
|
||||
@@ -710,6 +720,32 @@ mod test {
|
||||
assert_eq!(lifecycle.partial_cycles, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_usage_freshness_status() {
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
usage_freshness: rustfs_common::metrics::ScannerUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: 3,
|
||||
last_dirty_mark_unix_secs: 10,
|
||||
last_dirty_clear_unix_secs: 11,
|
||||
last_cycle_dirty_buckets: 2,
|
||||
last_cycle_cleared_dirty_buckets: 1,
|
||||
last_usage_save_unix_secs: 12,
|
||||
last_usage_save_result: "success".to_string(),
|
||||
last_usage_save_result_code: 1,
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(scanner.usage_freshness.dirty_pending_buckets, 3);
|
||||
assert_eq!(scanner.usage_freshness.last_dirty_mark_unix_secs, 10);
|
||||
assert_eq!(scanner.usage_freshness.last_dirty_clear_unix_secs, 11);
|
||||
assert_eq!(scanner.usage_freshness.last_cycle_dirty_buckets, 2);
|
||||
assert_eq!(scanner.usage_freshness.last_cycle_cleared_dirty_buckets, 1);
|
||||
assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12);
|
||||
assert_eq!(scanner.usage_freshness.last_usage_save_result, "success");
|
||||
assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
|
||||
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
|
||||
|
||||
@@ -280,6 +280,43 @@ impl ScannerMaintenanceControlSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerUsageFreshnessSnapshot {
|
||||
#[serde(rename = "dirty_pending_buckets", default)]
|
||||
pub dirty_pending_buckets: u64,
|
||||
#[serde(rename = "last_dirty_mark_unix_secs", default)]
|
||||
pub last_dirty_mark_unix_secs: u64,
|
||||
#[serde(rename = "last_dirty_clear_unix_secs", default)]
|
||||
pub last_dirty_clear_unix_secs: u64,
|
||||
#[serde(rename = "last_cycle_dirty_buckets", default)]
|
||||
pub last_cycle_dirty_buckets: u64,
|
||||
#[serde(rename = "last_cycle_cleared_dirty_buckets", default)]
|
||||
pub last_cycle_cleared_dirty_buckets: u64,
|
||||
#[serde(rename = "last_usage_save_unix_secs", default)]
|
||||
pub last_usage_save_unix_secs: u64,
|
||||
#[serde(rename = "last_usage_save_result", default)]
|
||||
pub last_usage_save_result: String,
|
||||
#[serde(rename = "last_usage_save_result_code", default)]
|
||||
pub last_usage_save_result_code: u64,
|
||||
}
|
||||
|
||||
impl ScannerUsageFreshnessSnapshot {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
self.dirty_pending_buckets = self.dirty_pending_buckets.saturating_add(other.dirty_pending_buckets);
|
||||
self.last_dirty_mark_unix_secs = self.last_dirty_mark_unix_secs.max(other.last_dirty_mark_unix_secs);
|
||||
self.last_dirty_clear_unix_secs = self.last_dirty_clear_unix_secs.max(other.last_dirty_clear_unix_secs);
|
||||
self.last_cycle_dirty_buckets = self.last_cycle_dirty_buckets.saturating_add(other.last_cycle_dirty_buckets);
|
||||
self.last_cycle_cleared_dirty_buckets = self
|
||||
.last_cycle_cleared_dirty_buckets
|
||||
.saturating_add(other.last_cycle_cleared_dirty_buckets);
|
||||
if other.last_usage_save_unix_secs > self.last_usage_save_unix_secs {
|
||||
self.last_usage_save_unix_secs = other.last_usage_save_unix_secs;
|
||||
self.last_usage_save_result = other.last_usage_save_result.clone();
|
||||
self.last_usage_save_result_code = other.last_usage_save_result_code;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ScannerSourceWorkSnapshot {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
self.checked = self.checked.saturating_add(other.checked);
|
||||
@@ -609,6 +646,8 @@ pub struct ScannerMetrics {
|
||||
pub lifecycle_expiry: ScannerLifecycleExpirySnapshot,
|
||||
#[serde(rename = "lifecycle_transition", default)]
|
||||
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
|
||||
#[serde(rename = "usage_freshness", default)]
|
||||
pub usage_freshness: ScannerUsageFreshnessSnapshot,
|
||||
#[serde(rename = "maintenance_control", default)]
|
||||
pub maintenance_control: ScannerMaintenanceControlSnapshot,
|
||||
#[serde(rename = "throttle_idle_mode_enabled", default)]
|
||||
@@ -690,6 +729,7 @@ impl ScannerMetrics {
|
||||
self.pacing_pressure.merge(&other.pacing_pressure);
|
||||
self.lifecycle_expiry.merge(&other.lifecycle_expiry);
|
||||
self.lifecycle_transition.merge(&other.lifecycle_transition);
|
||||
self.usage_freshness.merge(&other.usage_freshness);
|
||||
self.maintenance_control.merge(&other.maintenance_control);
|
||||
self.current_set_scan_concurrency_limit = self
|
||||
.current_set_scan_concurrency_limit
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
emit_scan_cycle_partial_with_source, global_metrics,
|
||||
};
|
||||
use rustfs_config::ScannerSpeed;
|
||||
@@ -1001,6 +1001,7 @@ pub async fn store_data_usage_in_backend(
|
||||
state = "skip_stale_update",
|
||||
"Scanner stale data usage update skipped"
|
||||
);
|
||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::SkippedStale);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1018,6 +1019,7 @@ pub async fn store_data_usage_in_backend(
|
||||
error = %e,
|
||||
"Scanner data usage encode failed"
|
||||
);
|
||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@@ -1055,8 +1057,10 @@ pub async fn store_data_usage_in_backend(
|
||||
error = %e,
|
||||
"Scanner data usage save failed"
|
||||
);
|
||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed);
|
||||
} else {
|
||||
replace_bucket_usage_memory_from_info(&data_usage_info).await;
|
||||
global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success);
|
||||
}
|
||||
done_save();
|
||||
|
||||
|
||||
@@ -91,13 +91,22 @@ fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
DIRTY_USAGE_BUCKETS.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let generation = DIRTY_USAGE_BUCKET_GENERATION.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
dirty_usage_buckets().insert(bucket.to_string(), generation);
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.insert(bucket.to_string(), generation);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_pending(usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
@@ -105,32 +114,44 @@ pub fn clear_dirty_usage_bucket(bucket: &str) {
|
||||
return;
|
||||
}
|
||||
|
||||
dirty_usage_buckets().remove(bucket);
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
dirty_buckets.remove(bucket);
|
||||
dirty_buckets.len()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_clear(usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
fn snapshot_dirty_usage_buckets(buckets: &[BucketInfo]) -> DirtyUsageBuckets {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| {
|
||||
dirty_buckets
|
||||
.get(&bucket.name)
|
||||
.map(|generation| (bucket.name.clone(), *generation))
|
||||
})
|
||||
.collect()
|
||||
let snapshot = {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
buckets
|
||||
.iter()
|
||||
.filter_map(|bucket| {
|
||||
dirty_buckets
|
||||
.get(&bucket.name)
|
||||
.map(|generation| (bucket.name.clone(), *generation))
|
||||
})
|
||||
.collect::<DirtyUsageBuckets>()
|
||||
};
|
||||
global_metrics().record_scanner_dirty_usage_cycle_snapshot(usize_to_u64_saturated(snapshot.len()));
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn clear_dirty_usage_buckets(snapshot: &DirtyUsageBuckets) {
|
||||
if snapshot.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
for (bucket, generation) in snapshot {
|
||||
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
|
||||
dirty_buckets.remove(bucket);
|
||||
let (cleared_buckets, pending_buckets) = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut cleared_buckets = 0usize;
|
||||
for (bucket, generation) in snapshot {
|
||||
if dirty_buckets.get(bucket).is_some_and(|current| current == generation) {
|
||||
dirty_buckets.remove(bucket);
|
||||
cleared_buckets += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
(cleared_buckets, dirty_buckets.len())
|
||||
};
|
||||
global_metrics()
|
||||
.record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared_buckets), usize_to_u64_saturated(pending_buckets));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -70,8 +70,23 @@ fn materialize_object(base: String, extra: &str, flags: &[String]) -> String {
|
||||
|
||||
fn has_dot_segments(path: &str) -> bool {
|
||||
path.split(['/', '\\']).any(|segment| {
|
||||
let trimmed = segment.trim();
|
||||
trimmed == "." || trimmed == ".."
|
||||
let mut bytes = segment.as_bytes();
|
||||
|
||||
// Match ecstore's path-component check. `str::trim()` also trims
|
||||
// Unicode whitespace such as vertical tab, which would reject object
|
||||
// keys that the production validator accepts as ordinary path bytes.
|
||||
while let Some((first, rest)) = bytes.split_first()
|
||||
&& first.is_ascii_whitespace()
|
||||
{
|
||||
bytes = rest;
|
||||
}
|
||||
while let Some((last, rest)) = bytes.split_last()
|
||||
&& last.is_ascii_whitespace()
|
||||
{
|
||||
bytes = rest;
|
||||
}
|
||||
|
||||
bytes == b"." || bytes == b".."
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user