gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): restore prompt bucket usage scans (#3516)

Browse Source 
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
Henry Guo
2026-06-17 21:49:56 +08:00
committed by GitHub
parent 8d24d9133b
commit a9aba323c6
1 changed files with 32 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner.rs
+32 -34
View File
@@ -53,7 +53,6 @@ use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn}; use tracing::{debug, error, info, instrument, warn};
const SINGLE_DISK_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
const LOG_COMPONENT_SCANNER: &str = "scanner"; const LOG_COMPONENT_SCANNER: &str = "scanner";
const LOG_SUBSYSTEM_RUNTIME: &str = "runtime"; const LOG_SUBSYSTEM_RUNTIME: &str = "runtime";
const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal"; const LOG_SUBSYSTEM_BACKGROUND_HEAL: &str = "background_heal";
@@ -147,12 +146,12 @@ fn initial_scanner_delay_for_startup(
has_active_replication: bool, has_active_replication: bool,
) -> Duration { ) -> Duration {
// Skip the startup delay when the cache is cold (first ever scan) OR when active replication // Skip the startup delay when the cache is cold (first ever scan) OR when active replication
// rules exist. In single-disk/Slowest mode the normal inter-cycle delay is 27-33 minutes; if // rules exist. A cold usage cache also covers startup-before-bucket-creation: running the
// the node was SIGKILL'd with FAILED-status objects queued, waiting that long leaves them // first cycle promptly keeps later bucket metrics bounded by the normal scanner cycle instead
// permanently unhealed until the next full cycle. Replication config is live-read at startup // of an extra startup delay. Replication config is live-read at startup by
// by configure_scanner_defaults, so this signal is always current regardless of when the // configure_scanner_defaults, so this signal is always current regardless of when the persisted
// persisted DataUsageInfo was last written. // DataUsageInfo was last written.
if (usage_cache_is_cold || has_active_replication) && has_buckets { if usage_cache_is_cold || (has_active_replication && has_buckets) {
Duration::ZERO Duration::ZERO
} else { } else {
initial_scanner_delay_for(start_delay_secs) initial_scanner_delay_for(start_delay_secs)
@@ -239,11 +238,7 @@ async fn initial_scanner_startup_usage_state(storeapi: &Arc<ECStore>) -> (bool,
} }
}; };
if !has_buckets { (persisted_usage_cache_is_cold_for_startup(storeapi).await, has_buckets)
return (false, false);
}
(persisted_usage_cache_is_cold_for_startup(storeapi).await, true)
} }
pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) { pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
@@ -331,12 +326,12 @@ impl ScannerMaintenanceFeatures {
} }
} }
fn single_disk_default_cycle_secs(features: ScannerMaintenanceFeatures) -> Option<u64> { fn single_disk_default_cycle_secs(_features: ScannerMaintenanceFeatures) -> Option<u64> {
if features.needs_regular_cycle() { None
None }
} else {
Some(SINGLE_DISK_SCANNER_CYCLE_SECS) fn single_disk_default_speed() -> ScannerSpeed {
} ScannerSpeed::Default
} }
async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> ScannerMaintenanceFeatures { async fn detect_scanner_maintenance_features(storeapi: &Arc<ECStore>) -> ScannerMaintenanceFeatures {
@@ -421,7 +416,7 @@ async fn configure_scanner_defaults(storeapi: &Arc<ECStore>) -> ScannerMaintenan
if is_erasure_sd().await { if is_erasure_sd().await {
let features = detect_scanner_maintenance_features(storeapi).await; let features = detect_scanner_maintenance_features(storeapi).await;
let default_cycle_secs = single_disk_default_cycle_secs(features); let default_cycle_secs = single_disk_default_cycle_secs(features);
set_scanner_default_speed(ScannerSpeed::Slowest); set_scanner_default_speed(single_disk_default_speed());
set_scanner_default_cycle_secs(default_cycle_secs); set_scanner_default_cycle_secs(default_cycle_secs);
info!( info!(
target: "rustfs::scanner", target: "rustfs::scanner",
@@ -1082,6 +1077,8 @@ mod tests {
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
use tokio::sync::Mutex; use tokio::sync::Mutex;
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
struct ScannerDefaultSpeedGuard; struct ScannerDefaultSpeedGuard;
impl ScannerDefaultSpeedGuard { impl ScannerDefaultSpeedGuard {
@@ -1219,10 +1216,9 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_initial_scanner_delay_keeps_configured_delay_without_buckets() { fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() {
let delay = initial_scanner_delay_for_startup(Some(120), true, false, false); let delay = initial_scanner_delay_for_startup(Some(120), true, false, false);
assert!(delay >= Duration::from_secs(108)); assert_eq!(delay, Duration::ZERO);
assert!(delay <= Duration::from_secs(132));
} }
#[test] #[test]
@@ -1425,7 +1421,7 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() { fn test_cycle_interval_prefers_explicit_cycle_over_default_cycle() {
let _guard = ScannerDefaultCycleGuard::set(SINGLE_DISK_SCANNER_CYCLE_SECS); let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
with_var(ENV_SCANNER_CYCLE, Some("42"), || { with_var(ENV_SCANNER_CYCLE, Some("42"), || {
assert_eq!(cycle_interval(), Duration::from_secs(42)); assert_eq!(cycle_interval(), Duration::from_secs(42));
@@ -1463,19 +1459,21 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() { fn test_cycle_interval_uses_default_cycle_override_when_unconfigured() {
let _guard = ScannerDefaultCycleGuard::set(SINGLE_DISK_SCANNER_CYCLE_SECS); let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
with_unset_scanner_timing_env(|| { with_unset_scanner_timing_env(|| {
assert_eq!(cycle_interval(), Duration::from_secs(SINGLE_DISK_SCANNER_CYCLE_SECS)); assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS));
}); });
} }
#[test] #[test]
fn test_single_disk_default_cycle_uses_long_interval_without_maintenance_features() { fn test_single_disk_default_cycle_uses_speed_based_interval_without_maintenance_features() {
assert_eq!( assert_eq!(single_disk_default_cycle_secs(ScannerMaintenanceFeatures::default()), None);
single_disk_default_cycle_secs(ScannerMaintenanceFeatures::default()), }
Some(SINGLE_DISK_SCANNER_CYCLE_SECS)
); #[test]
fn test_single_disk_default_speed_uses_regular_scanner_default() {
assert_eq!(single_disk_default_speed(), ScannerSpeed::Default);
} }
#[test] #[test]
@@ -1513,15 +1511,15 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_cycle_interval_keeps_single_disk_cycle_with_explicit_speed() { fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() {
let _guard = ScannerDefaultCycleGuard::set(SINGLE_DISK_SCANNER_CYCLE_SECS); let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
with_var_unset(ENV_SCANNER_CYCLE, || { with_var_unset(ENV_SCANNER_CYCLE, || {
with_var_unset("MINIO_SCANNER_CYCLE", || { with_var_unset("MINIO_SCANNER_CYCLE", || {
with_var_unset(ENV_SCANNER_START_DELAY_SECS, || { with_var_unset(ENV_SCANNER_START_DELAY_SECS, || {
with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || { with_var_unset(ENV_SCANNER_START_DELAY_SECS_DEPRECATED, || {
with_var(ENV_SCANNER_SPEED, Some("slowest"), || { with_var(ENV_SCANNER_SPEED, Some("slowest"), || {
assert_eq!(cycle_interval(), Duration::from_secs(SINGLE_DISK_SCANNER_CYCLE_SECS)); assert_eq!(cycle_interval(), Duration::from_secs(TEST_DEFAULT_SCANNER_CYCLE_SECS));
}); });
}); });
}); });
@@ -1532,7 +1530,7 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() { fn test_cycle_interval_prefers_explicit_start_delay_over_default_cycle() {
let _guard = ScannerDefaultCycleGuard::set(SINGLE_DISK_SCANNER_CYCLE_SECS); let _guard = ScannerDefaultCycleGuard::set(TEST_DEFAULT_SCANNER_CYCLE_SECS);
with_var_unset(ENV_SCANNER_CYCLE, || { with_var_unset(ENV_SCANNER_CYCLE, || {
with_var_unset("MINIO_SCANNER_CYCLE", || { with_var_unset("MINIO_SCANNER_CYCLE", || {