From 281e40f1cceae4f669ff454e89e4f121b845645d Mon Sep 17 00:00:00 2001 From: houseme Date: Mon, 31 Aug 2026 22:58:29 +0800 Subject: [PATCH] fix(scanner): fit usage persistence within publication lease (#6967) Lower the default scanner cache save timeout so the derived usage persistence budget stays inside the effective distributed publication lease window. Add focused regressions for the default publication budget and bootstrap-pending observational baselines, and update operator docs with the new default. Co-authored-by: heihutu --- crates/config/src/constants/scanner.rs | 4 +- crates/scanner/src/data_usage_define/tests.rs | 1 + crates/scanner/src/scanner/tests.rs | 56 +++++++++++++++++++ crates/scanner/src/storage_api.rs | 2 + ...minio-comprehensive-analysis-2026-08-16.md | 2 +- ...io-comprehensive-analysis-2026-08-16_zh.md | 2 +- docs/operations/scanner-runtime-controls.md | 2 +- 7 files changed, 64 insertions(+), 5 deletions(-) diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index 953ef789b..8c5b9231b 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE"; /// Environment variable that controls scanner cache save timeout in seconds. /// The scanner enforces a minimum value of `1`. /// - Unit: seconds (u64). -/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30` +/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14` pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS"; /// Default scanner cache save timeout in seconds. -pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30; +pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14; /// Environment variable that caps concurrent scanner set tasks. /// A value of `0` keeps the existing topology-based concurrency. diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index cc1b91251..624ba13a3 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -2241,6 +2241,7 @@ fn test_cache_save_timeout_uses_default_when_env_missing() { DataUsageCache::cache_save_timeout(), Duration::from_secs(rustfs_config::DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS) ); + assert_eq!(DataUsageCache::persistence_timeout(), Duration::from_millis(52_350)); }); crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 033032f4d..fcf815395 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -4507,6 +4507,47 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(backup.snapshot_identity())); } +#[tokio::test] +#[serial] +async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() { + let store = Arc::new(MemoryConfigStore::default()); + let primary = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + scanner_epoch: Some(7), + usage_snapshot_converged: Some(false), + usage_snapshot_bootstrap_pending: true, + ..Default::default() + }; + assert!(data_usage_info_is_bootstrap_pending(&primary)); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&primary).expect("bootstrap primary should encode"), + ); + + let (sender, receiver) = mpsc::channel(1); + let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1); + observation.usage_snapshot_converged = Some(false); + sender.send(observation).await.expect("observation should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + CancellationToken::new(), + store.clone(), + receiver, + None, + None, + || async { false }, + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Saved); + let observed = read_config(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()) + .await + .expect("observational snapshot should be persisted"); + let observed = serde_json::from_slice::(&observed).expect("observational snapshot should decode"); + assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(primary.snapshot_identity())); +} + #[tokio::test] async fn usage_baseline_does_not_fall_back_to_older_legacy_snapshot() { let store = Arc::new(MemoryConfigStore::default()); @@ -5924,6 +5965,21 @@ fn data_usage_persist_wait_covers_cache_retries_and_backup() { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } +#[test] +#[serial] +fn default_data_usage_persist_wait_fits_publication_lease_window() { + with_var_unset(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + let effective_publication_lease_window = + Duration::from_millis(crate::storage_api::ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS) + .saturating_sub(Duration::from_secs(5)); + + assert_eq!(data_usage_persist_timeout(), Duration::from_millis(52_350)); + assert!(data_usage_persist_timeout() < effective_publication_lease_window); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + #[tokio::test] async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { let ctx = CancellationToken::new(); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 2ac236f96..7df444cbd 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -111,6 +111,8 @@ pub(crate) use rustfs_ecstore::api::runtime::{ pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks; pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; #[cfg(test)] +pub(crate) use rustfs_ecstore::api::storage::SCANNER_PUBLICATION_LEASE_TTL_MS as ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS; +#[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx; use rustfs_storage_api as storage_contracts; diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md index da7572639..9ad160a7d 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -503,7 +503,7 @@ scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + ` | cycle_max_duration/objects/directories | …_MAX_* | 0 (unlimited) | | bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | | idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | | max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | | yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | | alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md index 9d5cb9461..f3efcc2aa 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md @@ -503,7 +503,7 @@ scanner（admin 子系统 `scanner`，`crates/config/src/constants/scanner.rs` + | cycle_max_duration/objects/directories | …_MAX_* | 0（不限） | | bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000（30d；0/on=每轮，off=禁用） | | idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | | max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | | yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | | alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index b480921ca..5242db647 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -57,7 +57,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. | | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | | `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` | Enables scanner sleeps and cooperative throttling. | -| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `30` | Timeout for saving scanner cache; runtime enforces a minimum of `1`. | +| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `14` | Timeout for saving scanner cache; runtime enforces a minimum of `1` and keeps the default persistence budget within the distributed publication lease. | | `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | | `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | | `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` | Controls how often object loops yield to the async runtime. `0` disables this extra yield. |