fix(scanner): surface per-tier usage in the data-usage snapshot (#5623)

SizeSummary::tier_stats was populated for every scanned object but
apply_scanner_size_summary dropped it, so per-tier usage never reached
DataUsageInfo. Wire it through the same merge chain repl_target_stats
already uses, up to DataUsageInfo::tier_stats.

DataUsageEntry used the derived MessagePack encoding, which serialises
structs as arrays: appending a field turns the whole cache into a decode
error for older readers, so mixed-version nodes would invalidate each
other's cache every scan cycle. Give it the same hand-written
map-encoded Serialize DataUsageCacheInfo already carries, and record the
invariant in AGENTS.md.

Widen TierStats counters from i32 to u64 so a tier past 2^31 versions
cannot make checked_merge reject an entire usage snapshot, and drop the
duplicate TierStats/AllTierStats definitions in the scanner crate in
favour of the data-usage ones.
This commit is contained in:
Zhengchao An
2026-08-02 18:46:36 +08:00
committed by GitHub
parent f440a14e53
commit 40cd10c1d0
5 changed files with 421 additions and 96 deletions
+53 -1
View File
@@ -461,6 +461,8 @@ fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary)
.pending_count
.saturating_add(u64::try_from(st.pending_count).unwrap_or(u64::MAX));
}
into.add_tier_sizes(&summary.tier_stats);
}
/// Cached folder information for scanning
@@ -3049,7 +3051,7 @@ mod tests {
use super::*;
use crate::storage_api::VersionPurgeStatusType;
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
use rustfs_filemeta::{FileInfo, FileMeta};
use serial_test::serial;
#[cfg(unix)]
@@ -3128,6 +3130,56 @@ mod tests {
assert_eq!(target_stats.pending_count, u64::MAX);
}
#[test]
fn scanner_size_summary_application_accumulates_tier_stats() {
let mut entry = DataUsageEntry::default();
let mut summary = SizeSummary::default();
summary.tier_stats.insert(
"WARM".to_string(),
TierStats {
total_size: 100,
num_versions: 2,
num_objects: 1,
},
);
// Scanners seed a zeroed entry for every configured tier; those must not
// reach the cache as empty keys.
summary
.tier_stats
.insert(storageclass::STANDARD.to_string(), TierStats::default());
apply_scanner_size_summary(&mut entry, &summary);
apply_scanner_size_summary(&mut entry, &summary);
let tiers = entry.all_tier_stats.as_ref().expect("tier stats should be recorded");
assert_eq!(
tiers.tiers.get("WARM"),
Some(&TierStats {
total_size: 200,
num_versions: 4,
num_objects: 2,
})
);
assert!(!tiers.tiers.contains_key(storageclass::STANDARD));
}
#[test]
fn scanner_size_summary_application_skips_untiered_summaries() {
let mut entry = DataUsageEntry::default();
let mut summary = SizeSummary {
total_size: 10,
..Default::default()
};
summary
.tier_stats
.insert(storageclass::STANDARD.to_string(), TierStats::default());
summary.tier_stats.insert(storageclass::RRS.to_string(), TierStats::default());
apply_scanner_size_summary(&mut entry, &summary);
assert!(entry.all_tier_stats.is_none(), "zero-only tier maps must not allocate cache state");
}
async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-test-{}", Uuid::new_v4()));
tokio::fs::create_dir_all(&temp_dir)