fix(scanner): surface per-tier usage in the data-usage snapshot (#5623)

SizeSummary::tier_stats was populated for every scanned object but
apply_scanner_size_summary dropped it, so per-tier usage never reached
DataUsageInfo. Wire it through the same merge chain repl_target_stats
already uses, up to DataUsageInfo::tier_stats.

DataUsageEntry used the derived MessagePack encoding, which serialises
structs as arrays: appending a field turns the whole cache into a decode
error for older readers, so mixed-version nodes would invalidate each
other's cache every scan cycle. Give it the same hand-written
map-encoded Serialize DataUsageCacheInfo already carries, and record the
invariant in AGENTS.md.

Widen TierStats counters from i32 to u64 so a tier past 2^31 versions
cannot make checked_merge reject an entire usage snapshot, and drop the
duplicate TierStats/AllTierStats definitions in the scanner crate in
favour of the data-usage ones.
This commit is contained in:
Zhengchao An
2026-08-02 18:46:36 +08:00
committed by GitHub
parent f440a14e53
commit 40cd10c1d0
5 changed files with 421 additions and 96 deletions
+52
View File
@@ -1146,6 +1146,7 @@ fn completed_data_usage_info(
versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
objects_total_size: u64::try_from(total.size).ok()?,
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
buckets_count: u64::try_from(all_buckets.len()).ok()?,
bucket_sizes,
buckets_usage,
@@ -1250,6 +1251,57 @@ mod publish_gate_tests {
completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled)
}
#[test]
fn completed_data_usage_info_publishes_tier_stats_across_sets() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
let warm = |total_size, num_versions, num_objects| {
HashMap::from([(
"WARM".to_string(),
TierStats {
total_size,
num_versions,
num_objects,
},
)])
};
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
let mut tiered = DataUsageEntry::default();
tiered.add_tier_sizes(&warm(100, 2, 1));
first_set.replace("bucket-b", DATA_USAGE_ROOT, tiered);
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
let mut tiered = DataUsageEntry::default();
tiered.add_tier_sizes(&warm(50, 1, 1));
second_set.replace("bucket-a", DATA_USAGE_ROOT, tiered);
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
.expect("completed sets should publish a snapshot");
assert_eq!(
data_usage_info
.tier_stats
.expect("tier usage should reach the snapshot")
.tiers["WARM"],
TierStats {
total_size: 150,
num_versions: 3,
num_objects: 2,
}
);
}
#[test]
fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() {
let all_buckets = vec!["bucket-a".to_string()];
let set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
let (data_usage_info, _) = completed_data_usage_info_for_test(&[set], &all_buckets, false, false)
.expect("completed set should publish a snapshot");
assert!(data_usage_info.tier_stats.is_none());
}
#[test]
fn completed_data_usage_info_requires_every_set_before_publish() {
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];