From eb57789f0fdbdd1ae420057661dfdef23dfe7aee Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 18 Nov 2026 11:10:55 +0800 Subject: [PATCH] fix(scanner): require a measured baseline for admin usage overlays --- crates/ecstore/src/data_usage/mod.rs | 11 ++--- .../data_usage/tests/companion_projection.rs | 48 +++++++++++++++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index d3034940cc..89adde364e 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -1473,7 +1473,7 @@ async fn project_admin_data_usage_for_current_buckets( ) -> DataUsageInfo { use futures::StreamExt as _; let snapshot_measured = response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot(); - let overlay_authoritative = with_memory_overlay && !runtime_sources::setup_is_dist_erasure().await; + let overlay_authoritative = with_memory_overlay && snapshot_measured && !runtime_sources::setup_is_dist_erasure().await; let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT; let mut incarnations = HashMap::new(); { @@ -1514,10 +1514,7 @@ async fn project_admin_data_usage_for_current_buckets( } } response.project_bucket_observations_for_admin(&incarnations); - if with_memory_overlay - && apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await - && snapshot_measured - { + if apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await { response.usage_snapshot_complete = false; response.usage_snapshot_partial = true; response.usage_snapshot_converged = Some(false); @@ -1784,8 +1781,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc) -> R /// Load the admin view with the existing authoritative single-process overlay. /// -/// The overlay is applied before current-identity projection so it cannot -/// reintroduce statistics from a retired or unverified bucket generation. +/// Only a complete or valid partial scanner snapshot can receive memory values. +/// The final identity check excludes statistics from retired or unverified buckets. /// Distributed nodes continue to use scanner statistics without local totals. /// Cancellation, unknown values, and per-bucket read semantics are the same as /// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted. diff --git a/crates/ecstore/src/data_usage/tests/companion_projection.rs b/crates/ecstore/src/data_usage/tests/companion_projection.rs index fcada2cfe2..38a167658b 100644 --- a/crates/ecstore/src/data_usage/tests/companion_projection.rs +++ b/crates/ecstore/src/data_usage/tests/companion_projection.rs @@ -402,6 +402,54 @@ async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_inv invalidate_data_usage_snapshot_cache().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn admin_memory_overlay_requires_a_measured_scanner_baseline() { + use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks}; + use crate::config::com::save_config; + use crate::storage_api_contracts::bucket::MakeBucketOptions; + + let (_dirs, store) = isolated_store_over_temp_disks().await; + init_bucket_metadata_sys(store.clone(), Vec::new()).await; + assert!(!runtime_sources::setup_is_dist_erasure().await); + let bucket = "usage-unmeasured-overlay"; + store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap(); + let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap(); + let mut baseline = data_usage_info_for_test(bucket, 1, 100, SystemTime::UNIX_EPOCH); + let usage = baseline.buckets_usage.get_mut(bucket).unwrap(); + usage.bucket_incarnation = Some(incarnation.to_string()); + let memory_usage = BucketUsageInfo { + size: 150, + ..usage.clone() + }; + clear_usage_memory_cache_for_test().await; + replace_bucket_usage_memory_from_authoritative(bucket, memory_usage, SystemTime::now()).await; + + // A missing snapshot and a mismatched bucket count both leave usage unknown. + for (buckets_count, expected_size) in [(None, None), (Some(2), None), (Some(1), Some(150))] { + if let Some(count) = buckets_count { + baseline.buckets_count = count; + save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&baseline).unwrap()) + .await + .unwrap(); + } + invalidate_data_usage_snapshot_cache().await; + let response = load_admin_data_usage_with_memory_overlay(store.clone()).await.unwrap(); + assert_eq!( + response.buckets_usage.get(bucket).map(|usage| usage.size), + expected_size, + "baseline count {buckets_count:?}" + ); + assert_eq!(response.bucket_sizes.get(bucket).copied(), expected_size); + assert_eq!(response.objects_total_size, expected_size.unwrap_or_default()); + assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some()); + assert!(!response.is_complete_bucket_usage_snapshot()); + assert!(!response.is_valid_partial_snapshot()); + } + clear_usage_memory_cache_for_test().await; + invalidate_data_usage_snapshot_cache().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[serial] async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {