From 05a5be51ce1e36b02d8e85c569584c0964e92e07 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 8 Aug 2026 00:09:26 +0800 Subject: [PATCH] fix(scanner): retry a superseded usage snapshot in seconds, not a full cycle (#5814) A superseded cycle is the expected outcome of the dirty-usage fast path, not a signal of pathological load: a write burst marks buckets dirty, the scanner wakes within milliseconds, and the still-landing writes then supersede the snapshot it just took. Charging that first race SUPERSEDED_RETRY_BASE_INTERVAL = 60s meant the burst surfaced in usage and quota accounting roughly two cycles late. Measured on an idle single-node instance (fresh data dir, 10 PUTs, polling /rustfs/admin/v3/datausageinfo every 5s): the dirty-usage wake fires 0.3s after the PUTs, its cycle is superseded 0.2s later, and the retry was then scheduled 55.6s out; usage first became visible at t+120s. With the base at 5s the retry is scheduled 4.7s out and usage becomes visible at t+70s. The exponential growth in retry_interval is what protects against a persistently hot bucket driving an unbroken full-scan loop, so the base does not need to be a whole cycle: 5s, 10s, 20s, 40s ... still reaches minute-scale backoff within a handful of consecutive supersedes and keeps the SUPERSEDED_RETRY_MAX_INTERVAL cap. A configured cycle shorter than the base still wins, since retrying faster than the operator's own cadence buys nothing. Verification: cargo test -p rustfs-scanner --lib (444 passed) with the three superseded-backoff tests updated to the new schedule; make pre-commit green; end-to-end probe above. --- crates/scanner/src/scanner.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index f8229b57b..162f2ff32 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -84,7 +84,23 @@ const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60); const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; -const SUPERSEDED_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(60); +/// First-retry delay after a usage snapshot is superseded by concurrent writes. +/// +/// A superseded cycle is the *expected* outcome of the dirty-usage fast path: +/// a write burst marks buckets dirty, the scanner wakes within milliseconds, +/// and the still-landing writes then supersede the snapshot it took. Charging +/// that first race a full cycle interval means a burst of writes surfaces in +/// usage/quota accounting roughly two cycles late (measured: ~120 s on an +/// otherwise idle instance whose clean-idle backoff had doubled a 60 s +/// interval), which defeats the fast path it is meant to protect. +/// +/// The exponential growth in [`ScannerSupersededBackoff::retry_interval`] is +/// what protects against a persistently hot bucket driving an unbroken +/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by +/// [`SUPERSEDED_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a +/// genuinely hot bucket still reaches minute-scale backoff within a handful of +/// cycles. +const SUPERSEDED_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5); const SUPERSEDED_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60); const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1); #[cfg(not(test))] @@ -6925,7 +6941,7 @@ mod tests { let mut backoff = ScannerSupersededBackoff::default(); assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None); - for expected in [60, 120, 240, 480, 960, 1_920, 3_840] { + for expected in [5, 10, 20, 40, 80, 160, 320] { backoff.record_cycle(ScannerCycleOutcome::Superseded); assert_eq!( backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), @@ -6949,15 +6965,19 @@ mod tests { let mut backoff = ScannerSupersededBackoff::default(); backoff.record_cycle(ScannerCycleOutcome::Superseded); - assert_eq!(backoff.retry_interval(Duration::from_secs(15)), Some(Duration::from_secs(15))); + // A configured cycle shorter than the base still wins: retrying sooner + // than the operator's own cadence buys nothing. + assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3))); backoff.record_cycle(ScannerCycleOutcome::Superseded); - assert_eq!(backoff.retry_interval(Duration::from_secs(15)), Some(Duration::from_secs(30))); + assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6))); } #[test] fn superseded_retry_backoff_grows_from_the_default_cycle() { let mut backoff = ScannerSupersededBackoff::default(); - for expected in [60, 120, 240, 480] { + // The first race after a write burst retries in seconds, not a whole + // cycle, while repeated supersedes still climb toward the cap. + for expected in [5, 10, 20, 40] { backoff.record_cycle(ScannerCycleOutcome::Superseded); assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected))); }