diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 645485e30..58202a53d 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -3181,7 +3181,8 @@ where )); let mut wait_plan = scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); - let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config); + let superseded_retry_interval = + scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features); let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval); let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); if let Some(retry_interval) = convergence_retry_interval { diff --git a/crates/scanner/src/scanner/activity.rs b/crates/scanner/src/scanner/activity.rs index 2b63d115a..efa930a6e 100644 --- a/crates/scanner/src/scanner/activity.rs +++ b/crates/scanner/src/scanner/activity.rs @@ -152,17 +152,19 @@ impl ScannerRetryBackoff { pub(super) fn scanner_superseded_retry_interval( backoff: ScannerRetryBackoff, runtime_config: &ScannerRuntimeConfig, + maintenance_features: ScannerMaintenanceFeatures, ) -> Option { - let retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; - if runtime_config.cycle_interval_source == ScannerRuntimeConfigSource::Default { - return Some(retry_interval); + let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?; + // Preserve an operator-selected duty-cycle floor during convergence. + if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default { + retry_interval = retry_interval.max(runtime_config.cycle_interval); } - - // An explicit cycle is an operator-selected duty-cycle floor. A - // superseded snapshot keeps its dirty work pending, but retrying that work - // sooner than the configured cadence would turn continuous writes into a - // repeated full-walk loop despite the override. - Some(retry_interval.max(runtime_config.cycle_interval)) + // A complete but non-authoritative walk must not postpone lifecycle or + // replication evaluation beyond its ordinary maintenance cadence. + if maintenance_features.needs_regular_cycle() { + retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1))); + } + Some(retry_interval) } const SCANNER_PUBLICATION_PROOF_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index b43fc1eb2..8a4f5a7e0 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -8489,7 +8489,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; assert_eq!( - scanner_superseded_retry_interval(backoff, &default_config), + scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()), Some(SCANNER_RETRY_BASE_INTERVAL), "the default adaptive cadence must retain fast convergence" ); @@ -8505,14 +8505,124 @@ fn superseded_retry_preserves_explicit_cycle_cadence() { ..Default::default() }; + for features in [ + ScannerMaintenanceFeatures::default(), + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ] { + assert_eq!( + scanner_superseded_retry_interval(backoff, &runtime_config, features), + Some(runtime_config.cycle_interval), + "{source:?} cycle cadence must not be shortened by convergence retries" + ); + } + } +} + +#[test] +fn superseded_retry_does_not_delay_required_maintenance() { + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + assert!(features.needs_regular_cycle()); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(2), + cycle_interval_source: ScannerRuntimeConfigSource::Env, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for consecutive_cycles in 1..=12 { + backoff.record_retryable_cycle(true); assert_eq!( - scanner_superseded_retry_interval(backoff, &runtime_config), - Some(runtime_config.cycle_interval), - "{source:?} cycle cadence must not be shortened by convergence retries" + scanner_superseded_retry_interval(backoff, &config, features), + Some(config.cycle_interval), + "superseded cycle {consecutive_cycles} must preserve regular maintenance" ); } } +#[test] +fn superseded_retry_only_caps_regular_maintenance() { + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..6 { + backoff.record_retryable_cycle(true); + } + assert_eq!( + scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()), + Some(Duration::from_secs(160)), + ); + for features in [ + ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + replication: true, + ..Default::default() + }, + ScannerMaintenanceFeatures { + inspection_failed: true, + ..Default::default() + }, + ] { + assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval)); + assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None); + } +} + +#[tokio::test(start_paused = true)] +#[serial] +async fn superseded_maintenance_wait_keeps_convergence_fencing() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + let config = ScannerRuntimeConfig { + cycle_interval: Duration::from_secs(60), + ..Default::default() + }; + let features = ScannerMaintenanceFeatures { + lifecycle: true, + ..Default::default() + }; + let mut backoff = ScannerRetryBackoff::default(); + for _ in 0..12 { + backoff.record_retryable_cycle(true); + } + let interval = scanner_superseded_retry_interval(backoff, &config, features); + assert_eq!(interval, Some(config.cycle_interval)); + let generations = ScannerCycleObservedGenerations::for_wait( + &config, + interval, + crate::scanner_io::dirty_usage_generation(), + crate::runtime_config::scanner_runtime_config_generation(), + crate::scanner_io::scanner_maintenance_generation(), + ); + assert!(generations.dirty_usage.is_none()); + assert!(generations.defer_cluster_activity); + let ctx = CancellationToken::new(); + let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])); + let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]); + let started = tokio::time::Instant::now(); + let reason = wait_for_next_scanner_cycle_with_activity( + &ctx, + interval.expect("maintenance must retain its convergence interval"), + Some(Duration::from_secs(1)), + &mut seen, + generations, + || false, + || std::future::ready(Ok(changed.clone())), + ) + .await; + assert_eq!(reason, ScannerCycleWakeReason::Timer); + assert_eq!(started.elapsed(), config.cycle_interval); + assert_eq!(seen, Some(changed)); +} + #[test] fn publication_proof_retry_backoff_reaches_its_short_cap() { for (failures, expected) in [(1, 5), (2, 10), (3, 20), (4, 30), (20, 30)] { diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 8ca95fafb..faf999409 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -98,6 +98,8 @@ An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle c Under continuous writes, a structurally complete walk can finish as `superseded`. Its durable observation remains separate from quota authority and leaves dirty work pending. Superseded retries may reuse that complete inventory to refresh dirty buckets, even while authority bootstrap is pending; missing or invalid coverage still forces a full walk. Ordinary maintenance timers, Deep scans, active lifecycle/replication and changed maintenance configuration retain full scans. Server info and datausageinfo can display complete observations, while Last Completed Scan advances only after an authoritative completed cycle. A write-free window is therefore not required to display observed counts, but arbitrary continuous writes do not guarantee authoritative completion. Historically dirty buckets remain eligible for rewalk, so narrower retries do not guarantee shorter cycles. +Superseded convergence retries retain exponential backoff for usage-only work. Active lifecycle or replication rules, and uncertain maintenance inspection, cap that retry interval at the ordinary configured cadence so complete observations cannot defer regular maintenance. An explicit cycle remains the duty-cycle floor. Convergence mode still suppresses dirty/activity shortcuts and does not grant quota authority; deferred backend/proof retries and pause catch-up limits retain their existing scheduling. Cycle runtime, pacing, and disk admission can still delay completed maintenance. Sustained supersedes with active maintenance may therefore run more full scans than usage-only convergence; use the existing pacing controls to manage that background I/O. + ## Single-disk clean-idle scheduling An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier.