diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index 953ef789b..8c5b9231b 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE"; /// Environment variable that controls scanner cache save timeout in seconds. /// The scanner enforces a minimum value of `1`. /// - Unit: seconds (u64). -/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30` +/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14` pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS"; /// Default scanner cache save timeout in seconds. -pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30; +pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14; /// Environment variable that caps concurrent scanner set tasks. /// A value of `0` keeps the existing topology-based concurrency. diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index cc1b91251..624ba13a3 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -2241,6 +2241,7 @@ fn test_cache_save_timeout_uses_default_when_env_missing() { DataUsageCache::cache_save_timeout(), Duration::from_secs(rustfs_config::DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS) ); + assert_eq!(DataUsageCache::persistence_timeout(), Duration::from_millis(52_350)); }); crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 033032f4d..fcf815395 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -4507,6 +4507,47 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(backup.snapshot_identity())); } +#[tokio::test] +#[serial] +async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() { + let store = Arc::new(MemoryConfigStore::default()); + let primary = DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + scanner_epoch: Some(7), + usage_snapshot_converged: Some(false), + usage_snapshot_bootstrap_pending: true, + ..Default::default() + }; + assert!(data_usage_info_is_bootstrap_pending(&primary)); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&primary).expect("bootstrap primary should encode"), + ); + + let (sender, receiver) = mpsc::channel(1); + let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1); + observation.usage_snapshot_converged = Some(false); + sender.send(observation).await.expect("observation should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + CancellationToken::new(), + store.clone(), + receiver, + None, + None, + || async { false }, + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Saved); + let observed = read_config(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()) + .await + .expect("observational snapshot should be persisted"); + let observed = serde_json::from_slice::(&observed).expect("observational snapshot should decode"); + assert_eq!(observed.usage_snapshot_authoritative_baseline, Some(primary.snapshot_identity())); +} + #[tokio::test] async fn usage_baseline_does_not_fall_back_to_older_legacy_snapshot() { let store = Arc::new(MemoryConfigStore::default()); @@ -5924,6 +5965,21 @@ fn data_usage_persist_wait_covers_cache_retries_and_backup() { crate::runtime_config::refresh_scanner_runtime_config_for_tests(); } +#[test] +#[serial] +fn default_data_usage_persist_wait_fits_publication_lease_window() { + with_var_unset(rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, || { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + let effective_publication_lease_window = + Duration::from_millis(crate::storage_api::ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS) + .saturating_sub(Duration::from_secs(5)); + + assert_eq!(data_usage_persist_timeout(), Duration::from_millis(52_350)); + assert!(data_usage_persist_timeout() < effective_publication_lease_window); + }); + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + #[tokio::test] async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() { let ctx = CancellationToken::new(); diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs index 2ac236f96..7df444cbd 100644 --- a/crates/scanner/src/storage_api.rs +++ b/crates/scanner/src/storage_api.rs @@ -111,6 +111,8 @@ pub(crate) use rustfs_ecstore::api::runtime::{ pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks; pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; #[cfg(test)] +pub(crate) use rustfs_ecstore::api::storage::SCANNER_PUBLICATION_LEASE_TTL_MS as ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS; +#[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx; use rustfs_storage_api as storage_contracts; diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md index da7572639..9ad160a7d 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -503,7 +503,7 @@ scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + ` | cycle_max_duration/objects/directories | …_MAX_* | 0 (unlimited) | | bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | | idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | | max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | | yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | | alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md index 9d5cb9461..f3efcc2aa 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md @@ -503,7 +503,7 @@ scanner(admin 子系统 `scanner`,`crates/config/src/constants/scanner.rs` + | cycle_max_duration/objects/directories | …_MAX_* | 0(不限) | | bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000(30d;0/on=每轮,off=禁用) | | idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | | max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | | yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | | alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index b480921ca..5242db647 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -57,7 +57,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. | | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | | `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` | Enables scanner sleeps and cooperative throttling. | -| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `30` | Timeout for saving scanner cache; runtime enforces a minimum of `1`. | +| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `14` | Timeout for saving scanner cache; runtime enforces a minimum of `1` and keeps the default persistence budget within the distributed publication lease. | | `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | | `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | | `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` | Controls how often object loops yield to the async runtime. `0` disables this extra yield. |