gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 02:33:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(lifecycle): harden scanner ILM expiry accounting (#3257)

Browse Source 
* fix(lifecycle): harden scanner ILM expiry accounting

* fix(scanner): gate ILM action accounting on enqueue

* fix(metrics): avoid scanner source work argument list

* fix(scanner): gate local ILM accounting on enqueue

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-06-08 00:18:34 +08:00
committed by GitHub
parent cebea9a238
commit f49df41db9
3 changed files with 400 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder.rs
+142 -28
View File
@@ -147,6 +147,28 @@ fn should_yield_after_object(object_count: u64, yield_every: u64) -> bool {
yield_every > 0 && object_count.is_multiple_of(yield_every)
}
fn record_scanner_ilm_action_if_queued(metrics: &Metrics, count: u64, queued: bool) -> bool {
if queued {
metrics.record_scanner_ilm_action(count);
}
queued
}
#[derive(Clone, Copy)]
struct PendingScannerAccounting<'a> {
object: &'a ObjectInfo,
retained_size: i64,
expired_size: i64,
}
impl PendingScannerAccounting<'_> {
fn apply(self, size_summary: &mut SizeSummary, cumulative_size: &mut i64, queued: bool) {
let size = if queued { self.expired_size } else { self.retained_size };
size_summary.actions_accounting(self.object, size, self.retained_size);
*cumulative_size += size;
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum FolderResumeMatch {
Exact,
@@ -559,6 +581,7 @@ impl ScannerItem {
};
let mut to_delete_objs: Vec<ObjectToDelete> = Vec::new();
let mut noncurrent_events: Vec<Event> = Vec::new();
let mut noncurrent_accounting: Vec<PendingScannerAccounting<'_>> = Vec::new();
let mut cumulative_size = 0;
let mut remaining_versions = object_infos.len();
'eventLoop: {
@@ -573,48 +596,77 @@ impl ScannerItem {
};
let mut size = actual_size;
let mut account_now = true;
let done_ilm = Metrics::time_ilm(event.action);
match event.action {
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => {
remaining_versions = 0;
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) {
done_ilm(1)();
remaining_versions = 0;
} else {
PendingScannerAccounting {
object: oi,
retained_size: actual_size,
expired_size: 0,
}
.apply(size_summary, &mut cumulative_size, false);
for retained in object_infos.iter().skip(i + 1) {
let retained_size = match retained.get_actual_size() {
Ok(size) => size,
Err(_) => {
warn!("apply_actions: Failed to get actual size for object {}", retained.name);
0
}
};
PendingScannerAccounting {
object: retained,
retained_size,
expired_size: 0,
}
.apply(size_summary, &mut cumulative_size, false);
}
}
break 'eventLoop;
}
IlmAction::DeleteAction | IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
remaining_versions -= 1;
size = 0;
}
debug!("apply_actions: applying expiry rule for object: {} {}", oi.name, event.action);
apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) {
done_ilm(1)();
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
remaining_versions -= 1;
size = 0;
}
}
}
IlmAction::DeleteVersionAction => {
remaining_versions -= 1;
size = 0;
if let Some(opt) = object_opts.get(i) {
to_delete_objs.push(ObjectToDelete {
object_name: opt.name.clone(),
version_id: opt.version_id,
..Default::default()
});
noncurrent_accounting.push(PendingScannerAccounting {
object: oi,
retained_size: actual_size,
expired_size: 0,
});
account_now = false;
}
noncurrent_events.push(event.clone());
done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
}
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
debug!("apply_actions: applying transition rule for object: {} {}", oi.name, event.action);
apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
done_ilm(1)();
global_metrics().record_scanner_ilm_action(1);
let done_ilm = Metrics::time_ilm(event.action);
let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
if record_scanner_ilm_action_if_queued(global_metrics(), 1, queued) {
done_ilm(1)();
}
}
IlmAction::NoneAction | IlmAction::ActionCount => {
@@ -622,20 +674,31 @@ impl ScannerItem {
}
}
size_summary.actions_accounting(oi, size, actual_size);
cumulative_size += size;
if account_now {
size_summary.actions_accounting(oi, size, actual_size);
cumulative_size += size;
}
}
}
if !to_delete_objs.is_empty()
&& let Some(event) = noncurrent_events.first().cloned()
{
GLOBAL_ExpiryState
let action = event.action;
let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX);
let done_ilm = Metrics::time_ilm(action);
let queued = GLOBAL_ExpiryState
.write()
.await
.enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event)
.enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner)
.await;
if record_scanner_ilm_action_if_queued(global_metrics(), count, queued) {
done_ilm(count)();
remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len());
}
for pending in noncurrent_accounting {
pending.apply(size_summary, &mut cumulative_size, queued);
}
}
self.alert_excessive_versions(remaining_versions, cumulative_size);
}
@@ -756,7 +819,7 @@ impl ScannerItem {
ensure_scanner_alert_metrics_registered();
let (too_many_versions, too_large_versions) = should_alert_excessive_versions(remaining_versions, cumulative_size);
if too_many_versions {
global_metrics().record_scanner_source_work(ScannerWorkSource::Alerts, 0, 0, 1, 0, 0);
global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1);
counter!(
METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL,
"bucket" => self.bucket.clone()
@@ -771,7 +834,7 @@ impl ScannerItem {
);
}
if too_large_versions {
global_metrics().record_scanner_source_work(ScannerWorkSource::Alerts, 0, 0, 1, 0, 0);
global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1);
counter!(
METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL,
"bucket" => self.bucket.clone()
@@ -911,7 +974,7 @@ impl FolderScanner {
}
ensure_scanner_alert_metrics_registered();
global_metrics().record_scanner_source_work(ScannerWorkSource::Alerts, 0, 0, 1, 0, 0);
global_metrics().record_scanner_source_executed(ScannerWorkSource::Alerts, 1);
counter!(
METRIC_SCANNER_EXCESS_FOLDERS_TOTAL,
"root" => self.root.clone()
@@ -1987,6 +2050,57 @@ mod tests {
assert!(!ScannerItem::should_account_replication_stats(&purge_version));
}
#[tokio::test]
async fn test_scanner_ilm_action_accounting_requires_enqueue_success() {
let metrics = Metrics::new();
record_scanner_ilm_action_if_queued(&metrics, 2, false);
let report = metrics.report().await;
let lifecycle = report
.source_work
.iter()
.find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source work should be visible");
assert_eq!(lifecycle.executed, 0);
record_scanner_ilm_action_if_queued(&metrics, 3, true);
let report = metrics.report().await;
let lifecycle = report
.source_work
.iter()
.find(|work| work.source == ScannerWorkSource::Lifecycle.as_str())
.expect("lifecycle source work should be visible");
assert_eq!(lifecycle.executed, 3);
}
#[test]
fn test_pending_scanner_accounting_requires_enqueue_success() {
let object = ObjectInfo {
size: 10,
version_id: Some(uuid::Uuid::new_v4()),
..Default::default()
};
let pending = PendingScannerAccounting {
object: &object,
retained_size: 10,
expired_size: 0,
};
let mut failed_summary = SizeSummary::default();
let mut failed_cumulative_size = 0;
pending.apply(&mut failed_summary, &mut failed_cumulative_size, false);
assert_eq!(failed_summary.versions, 1);
assert_eq!(failed_summary.total_size, 10);
assert_eq!(failed_cumulative_size, 10);
let mut queued_summary = SizeSummary::default();
let mut queued_cumulative_size = 0;
pending.apply(&mut queued_summary, &mut queued_cumulative_size, true);
assert_eq!(queued_summary.versions, 0);
assert_eq!(queued_summary.total_size, 0);
assert_eq!(queued_cumulative_size, 0);
}
#[test]
#[serial]
fn test_excessive_version_alert_thresholds_use_env() {