diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 81c8ed194..12cf3117b 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -960,7 +960,10 @@ async fn load_observed_data_usage_snapshot(store: Arc) -> Option Some(info), + && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) => + { + Some(info) + } Ok(_) => { error!( event = "data_usage_snapshot_load_failed", @@ -4749,12 +4752,7 @@ mod tests { async fn negative_delta_waits_for_set_reconciliation() { clear_usage_memory_cache_for_test().await; - let baseline = data_usage_info_for_test( - "bucket-a", - 1, - 100, - SystemTime::UNIX_EPOCH + Duration::from_secs(100), - ); + let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100)); replace_bucket_usage_memory_from_info(&baseline).await; record_bucket_object_delete_memory("bucket-a", 25, true).await; diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 6341cb535..295a1da85 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,8 +28,8 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, - PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, DataUsageSnapshotSetState, hash_path, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME, + PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 83cdaf14f..030668f26 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -18,9 +18,8 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder}; use crate::sleeper::SCANNER_SLEEPER; use crate::{ DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome, - DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageSnapshotSetState, DataUsageScanPlanDigest, - ScannerError, SizeSummary, - TierStats, + DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState, + ScannerError, SizeSummary, TierStats, }; use futures::future::join_all; use metrics::counter; diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index dd6154e0d..cf877362e 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -353,7 +353,10 @@ pub(super) fn observational_data_usage_info( objects_total_size: u64::try_from(total.size).ok()?, tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()), buckets_count: u64::try_from(buckets_usage.len()).ok()?, - bucket_sizes: buckets_usage.iter().map(|(bucket, usage)| (bucket.clone(), usage.size)).collect(), + bucket_sizes: buckets_usage + .iter() + .map(|(bucket, usage)| (bucket.clone(), usage.size)) + .collect(), buckets_usage, usage_snapshot_complete: false, usage_snapshot_partial: true, diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 09e5b4f9d..db52cb559 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -112,9 +112,7 @@ impl ScannerIOCache for SetDisks { incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch); incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest; } - let _ = updates - .send(incomplete_scope) - .await; + let _ = updates.send(incomplete_scope).await; return Ok(()); } // Preserve the original set topology across capability filtering. During @@ -214,9 +212,7 @@ impl ScannerIOCache for SetDisks { incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch); incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest; } - let _ = updates - .send(incomplete_scope) - .await; + let _ = updates.send(incomplete_scope).await; return Ok(()); } let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index a12919e5e..07b1f3f84 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -386,16 +386,19 @@ impl ScannerIOCycle for ECStore { budget_elapsed, ctx.is_cancelled(), ); - let observational_usage = completed_usage.is_none().then(|| { - observational_data_usage_info( - &results, - &expected_sources, - &all_buckets, - scan_plan_digest, - want_cycle, - leader_epoch, - ) - }).flatten(); + let observational_usage = completed_usage + .is_none() + .then(|| { + observational_data_usage_info( + &results, + &expected_sources, + &all_buckets, + scan_plan_digest, + want_cycle, + leader_epoch, + ) + }) + .flatten(); let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some(); let cycle_status = classify_nsscanner_cycle( structurally_complete_snapshot, diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index 7715442d5..8d44ff945 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -129,16 +129,11 @@ fn partial_usage_is_observational_not_authoritative_for_quota() { let stalled = lkg_root_cache("bucket", 1, stalled_source); let expected = HashSet::from([current_source, stalled_source]); - assert!(completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none()); - let (observed, _) = observational_data_usage_info( - &[current, stalled], - &expected, - &all_buckets, - TEST_PLAN_DIGEST, - 8, - 3, - ) - .expect("a completed set should produce an observational view"); + assert!( + completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none() + ); + let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + .expect("a completed set should produce an observational view"); assert!(observed.usage_snapshot_partial); assert!(!observed.usage_snapshot_complete); assert_eq!(observed.usage_snapshot_converged, Some(false));