mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-02 11:29:17 +00:00
fix(scanner): surface per-tier usage in the data-usage snapshot (#5623)
SizeSummary::tier_stats was populated for every scanned object but apply_scanner_size_summary dropped it, so per-tier usage never reached DataUsageInfo. Wire it through the same merge chain repl_target_stats already uses, up to DataUsageInfo::tier_stats. DataUsageEntry used the derived MessagePack encoding, which serialises structs as arrays: appending a field turns the whole cache into a decode error for older readers, so mixed-version nodes would invalidate each other's cache every scan cycle. Give it the same hand-written map-encoded Serialize DataUsageCacheInfo already carries, and record the invariant in AGENTS.md. Widen TierStats counters from i32 to u64 so a tier past 2^31 versions cannot make checked_merge reject an entire usage snapshot, and drop the duplicate TierStats/AllTierStats definitions in the scanner crate in favour of the data-usage ones.
This commit is contained in:
@@ -347,6 +347,11 @@ cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
|
||||
absent, empty, and nil all mean "no value", never `Uuid::nil()`.
|
||||
- A remote-tier version of `None`/`""` means the tier bucket is unversioned:
|
||||
send **no** `versionId` on tier GET/DELETE.
|
||||
- Structs persisted in the scanner data-usage cache (`DataUsageCacheInfo`,
|
||||
`DataUsageEntry`) carry a hand-written map-encoded `Serialize`. MessagePack
|
||||
encodes derived structs as arrays, where an appended field makes the whole
|
||||
cache a decode error for older readers — keep new fields `#[serde(default)]`
|
||||
and keep the map encoding rather than reverting to `derive(Serialize)`.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Serialize, ser::SerializeMap as _};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
@@ -51,24 +51,36 @@ pub fn usage_last_update_is_untrusted_future(existing_last_update: SystemTime, n
|
||||
existing_last_update > now + USAGE_LAST_UPDATE_FUTURE_TOLERANCE
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)]
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct TierStats {
|
||||
pub total_size: u64,
|
||||
pub num_versions: i32,
|
||||
pub num_objects: i32,
|
||||
pub num_versions: u64,
|
||||
pub num_objects: u64,
|
||||
}
|
||||
|
||||
impl TierStats {
|
||||
pub fn add(&self, u: &TierStats) -> TierStats {
|
||||
TierStats {
|
||||
total_size: self.total_size + u.total_size,
|
||||
num_versions: self.num_versions + u.num_versions,
|
||||
num_objects: self.num_objects + u.num_objects,
|
||||
total_size: self.total_size.saturating_add(u.total_size),
|
||||
num_versions: self.num_versions.saturating_add(u.num_versions),
|
||||
num_objects: self.num_objects.saturating_add(u.num_objects),
|
||||
}
|
||||
}
|
||||
|
||||
/// True when [`TierStats::add`] would report the exact sum instead of saturating.
|
||||
pub fn fits_add(&self, u: &TierStats) -> bool {
|
||||
self.total_size.checked_add(u.total_size).is_some()
|
||||
&& self.num_versions.checked_add(u.num_versions).is_some()
|
||||
&& self.num_objects.checked_add(u.num_objects).is_some()
|
||||
}
|
||||
|
||||
/// True when this tier contributed nothing, i.e. merging it is a no-op.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.total_size == 0 && self.num_versions == 0 && self.num_objects == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct AllTierStats {
|
||||
pub tiers: HashMap<String, TierStats>,
|
||||
}
|
||||
@@ -78,31 +90,35 @@ impl AllTierStats {
|
||||
Self { tiers: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.tiers.is_empty()
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map in.
|
||||
///
|
||||
/// Scanners seed the map with a zeroed entry for every configured tier, so
|
||||
/// empty contributions are skipped to keep the persisted cache from growing
|
||||
/// one key per tier on every folder that never held tiered data.
|
||||
pub fn add_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
for (tier, st) in tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
*entry = entry.add(st);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: AllTierStats) {
|
||||
for (tier, st) in other.tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
pub fn merge(&mut self, other: &AllTierStats) {
|
||||
self.add_sizes(&other.tiers);
|
||||
}
|
||||
|
||||
pub fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
|
||||
for (tier, st) in &self.tiers {
|
||||
stats.insert(
|
||||
tier.clone(),
|
||||
TierStats {
|
||||
total_size: st.total_size,
|
||||
num_versions: st.num_versions,
|
||||
num_objects: st.num_objects,
|
||||
},
|
||||
);
|
||||
}
|
||||
/// True when [`AllTierStats::merge`] would report exact sums for every tier.
|
||||
pub fn fits_merge(&self, other: &AllTierStats) -> bool {
|
||||
other
|
||||
.tiers
|
||||
.iter()
|
||||
.all(|(tier, right)| self.tiers.get(tier).is_none_or(|left| left.fits_add(right)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,6 +199,14 @@ pub struct DataUsageInfo {
|
||||
pub objects_total_size: u64,
|
||||
/// Replication info across all buckets
|
||||
pub replication_info: HashMap<String, BucketTargetUsageInfo>,
|
||||
/// Usage per storage class and remote tier across all buckets.
|
||||
///
|
||||
/// Absent on snapshots written before per-tier accounting was published,
|
||||
/// and on clusters with no remote tier configured: the scanner classifies
|
||||
/// objects by tier (including `STANDARD`/`REDUCED_REDUNDANCY`) only once a
|
||||
/// tier exists, so an absent value means "not accounted", never "zero".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tier_stats: Option<AllTierStats>,
|
||||
|
||||
/// Total number of buckets in this cluster
|
||||
pub buckets_count: u64,
|
||||
@@ -562,7 +586,7 @@ impl ReplicationAllStats {
|
||||
}
|
||||
|
||||
/// Data usage cache entry
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageEntry {
|
||||
pub children: DataUsageHashMap,
|
||||
// These fields do not include any children.
|
||||
@@ -577,6 +601,34 @@ pub struct DataUsageEntry {
|
||||
/// Number of objects that failed to scan (e.g., IO errors)
|
||||
#[serde(default)]
|
||||
pub failed_objects: usize,
|
||||
/// Per-tier usage contributed by this entry, present only once a scan
|
||||
/// observed tier-classified objects.
|
||||
#[serde(default)]
|
||||
pub all_tier_stats: Option<AllTierStats>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageEntry {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(11))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
@@ -635,10 +687,22 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(o_tiers) = other.all_tier_stats.as_ref().filter(|tiers| !tiers.is_empty()) {
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).merge(o_tiers);
|
||||
}
|
||||
|
||||
self.obj_sizes.merge_from(&other.obj_sizes);
|
||||
self.obj_versions.merge_from(&other.obj_versions);
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map into this entry.
|
||||
pub fn add_tier_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
if tiers.values().all(TierStats::is_empty) {
|
||||
return;
|
||||
}
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).add_sizes(tiers);
|
||||
}
|
||||
|
||||
pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
|
||||
let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
|
||||
&& self.versions.checked_add(other.versions).is_some()
|
||||
@@ -698,7 +762,12 @@ impl DataUsageEntry {
|
||||
}
|
||||
};
|
||||
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits {
|
||||
let tier_stats_fit = match (&self.all_tier_stats, &other.all_tier_stats) {
|
||||
(_, None) | (None, Some(_)) => true,
|
||||
(Some(left), Some(right)) => left.fits_merge(right),
|
||||
};
|
||||
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit {
|
||||
return false;
|
||||
}
|
||||
self.merge(other);
|
||||
@@ -1038,6 +1107,7 @@ impl DataUsageCache {
|
||||
versions_total_count: flat.versions as u64,
|
||||
delete_markers_total_count: flat.delete_markers as u64,
|
||||
objects_total_size: flat.size as u64,
|
||||
tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: self.info.snapshot_complete,
|
||||
@@ -1525,6 +1595,172 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
fn tier_entry(tier: &str, stats: TierStats) -> DataUsageEntry {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
entry.add_tier_sizes(&HashMap::from([(tier.to_string(), stats)]));
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 5,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
right.add_tier_sizes(&HashMap::from([(
|
||||
"COLD".to_string(),
|
||||
TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
},
|
||||
)]));
|
||||
|
||||
assert!(left.checked_merge(&right), "merging exact tier totals must be accepted");
|
||||
|
||||
let tiers = &left.all_tier_stats.expect("merged entry keeps tier stats").tiers;
|
||||
assert_eq!(
|
||||
tiers.get("WARM"),
|
||||
Some(&TierStats {
|
||||
total_size: 15,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
tiers.get("COLD"),
|
||||
Some(&TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_merge_into_an_untiered_entry() {
|
||||
let mut left = DataUsageEntry::default();
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(left.checked_merge(&right));
|
||||
|
||||
assert_eq!(
|
||||
left.all_tier_stats.expect("tier stats adopted from the merged entry").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_overflowing_tier_totals() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: u64::MAX,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 1,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!left.checked_merge(&right), "saturating tier totals must not be published");
|
||||
assert_eq!(left.all_tier_stats.expect("left is untouched").tiers["WARM"].total_size, u64::MAX);
|
||||
}
|
||||
|
||||
/// Entry shape released before per-tier accounting, using the derived
|
||||
/// (array) encoding those writers produced.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct LegacyEntry {
|
||||
children: DataUsageHashMap,
|
||||
size: usize,
|
||||
objects: usize,
|
||||
versions: usize,
|
||||
delete_markers: usize,
|
||||
obj_sizes: SizeHistogram,
|
||||
obj_versions: VersionsHistogram,
|
||||
replication_stats: Option<ReplicationAllStats>,
|
||||
compacted: bool,
|
||||
#[serde(default)]
|
||||
failed_objects: usize,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entries_are_map_encoded_so_appended_fields_stay_readable() {
|
||||
// A derived (array) encoding turns every appended field into a decode
|
||||
// error for readers built before it existed, which would cost a mixed
|
||||
// -version cluster its whole scan cache. Entries must stay map-encoded.
|
||||
let current = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 3,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut encoded = Vec::new();
|
||||
current
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode current entry");
|
||||
|
||||
let legacy: LegacyEntry = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore the appended field");
|
||||
assert_eq!(legacy.objects, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_array_encoded_entries_still_load() {
|
||||
let legacy = LegacyEntry {
|
||||
children: DataUsageHashMap::default(),
|
||||
size: 12,
|
||||
objects: 3,
|
||||
versions: 4,
|
||||
delete_markers: 1,
|
||||
obj_sizes: SizeHistogram::default(),
|
||||
obj_versions: VersionsHistogram::default(),
|
||||
replication_stats: None,
|
||||
compacted: false,
|
||||
failed_objects: 2,
|
||||
};
|
||||
let mut encoded = Vec::new();
|
||||
legacy
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode legacy entry");
|
||||
|
||||
let decoded: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("current reader should default the missing field");
|
||||
|
||||
assert_eq!(decoded.size, 12);
|
||||
assert_eq!(decoded.failed_objects, 2);
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
@@ -1901,6 +2137,44 @@ mod tests {
|
||||
assert_eq!(info.buckets_count, 2);
|
||||
assert!(info.buckets_usage.is_empty());
|
||||
assert_eq!(info.objects_total_count, 3);
|
||||
assert!(info.tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dui_reports_tier_usage_from_the_flattened_tree() {
|
||||
let root_hash = hash_path("root");
|
||||
let bucket_hash = hash_path("bucket-a");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "root".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(&root_hash, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&bucket_hash,
|
||||
&Some(root_hash),
|
||||
&tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
let info = cache.dui("root", &["bucket-a".to_string()]);
|
||||
|
||||
assert_eq!(
|
||||
info.tier_stats.expect("child tier usage should roll up to the root").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -27,8 +27,8 @@ use rustfs_common::heal_channel::HealScanMode;
|
||||
#[cfg(test)]
|
||||
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||
pub use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageEntry, DataUsageHash, DataUsageHashMap,
|
||||
DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, hash_path,
|
||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageEntry, DataUsageHash,
|
||||
DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
@@ -184,69 +184,6 @@ pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
|
||||
|
||||
const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024;
|
||||
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)]
|
||||
pub struct TierStats {
|
||||
pub total_size: u64,
|
||||
pub num_versions: i32,
|
||||
pub num_objects: i32,
|
||||
}
|
||||
|
||||
impl TierStats {
|
||||
pub fn add(&self, u: &TierStats) -> TierStats {
|
||||
TierStats {
|
||||
total_size: self.total_size + u.total_size,
|
||||
num_versions: self.num_versions + u.num_versions,
|
||||
num_objects: self.num_objects + u.num_objects,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_object_info(oi: &ObjectInfo) -> Self {
|
||||
TierStats {
|
||||
total_size: oi.size as u64,
|
||||
num_versions: 1,
|
||||
num_objects: if oi.is_latest { 1 } else { 0 },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct AllTierStats {
|
||||
pub tiers: HashMap<String, TierStats>,
|
||||
}
|
||||
|
||||
impl AllTierStats {
|
||||
pub fn new() -> Self {
|
||||
Self { tiers: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
|
||||
for (tier, st) in tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: AllTierStats) {
|
||||
for (tier, st) in other.tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
|
||||
for (tier, st) in &self.tiers {
|
||||
stats.insert(
|
||||
tier.clone(),
|
||||
TierStats {
|
||||
total_size: st.total_size,
|
||||
num_versions: st.num_versions,
|
||||
num_objects: st.num_objects,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Size summary for a single object or group of objects
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct SizeSummary {
|
||||
@@ -301,7 +238,11 @@ impl SizeSummary {
|
||||
}
|
||||
|
||||
if let Some(tier_stats) = self.tier_stats.get_mut(&tier) {
|
||||
*tier_stats = tier_stats.add(&TierStats::from_object_info(oi));
|
||||
*tier_stats = tier_stats.add(&TierStats {
|
||||
total_size: u64::try_from(oi.size).unwrap_or(0),
|
||||
num_versions: 1,
|
||||
num_objects: u64::from(oi.is_latest),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -963,6 +904,7 @@ impl DataUsageCache {
|
||||
versions_total_count: flat.versions as u64,
|
||||
delete_markers_total_count: flat.delete_markers as u64,
|
||||
objects_total_size: flat.size as u64,
|
||||
tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
||||
buckets_usage,
|
||||
..Default::default()
|
||||
|
||||
@@ -461,6 +461,8 @@ fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary)
|
||||
.pending_count
|
||||
.saturating_add(u64::try_from(st.pending_count).unwrap_or(u64::MAX));
|
||||
}
|
||||
|
||||
into.add_tier_sizes(&summary.tier_stats);
|
||||
}
|
||||
|
||||
/// Cached folder information for scanning
|
||||
@@ -3049,7 +3051,7 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::VersionPurgeStatusType;
|
||||
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass};
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use serial_test::serial;
|
||||
#[cfg(unix)]
|
||||
@@ -3128,6 +3130,56 @@ mod tests {
|
||||
assert_eq!(target_stats.pending_count, u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_size_summary_application_accumulates_tier_stats() {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
let mut summary = SizeSummary::default();
|
||||
summary.tier_stats.insert(
|
||||
"WARM".to_string(),
|
||||
TierStats {
|
||||
total_size: 100,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
// Scanners seed a zeroed entry for every configured tier; those must not
|
||||
// reach the cache as empty keys.
|
||||
summary
|
||||
.tier_stats
|
||||
.insert(storageclass::STANDARD.to_string(), TierStats::default());
|
||||
|
||||
apply_scanner_size_summary(&mut entry, &summary);
|
||||
apply_scanner_size_summary(&mut entry, &summary);
|
||||
|
||||
let tiers = entry.all_tier_stats.as_ref().expect("tier stats should be recorded");
|
||||
assert_eq!(
|
||||
tiers.tiers.get("WARM"),
|
||||
Some(&TierStats {
|
||||
total_size: 200,
|
||||
num_versions: 4,
|
||||
num_objects: 2,
|
||||
})
|
||||
);
|
||||
assert!(!tiers.tiers.contains_key(storageclass::STANDARD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_size_summary_application_skips_untiered_summaries() {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
let mut summary = SizeSummary {
|
||||
total_size: 10,
|
||||
..Default::default()
|
||||
};
|
||||
summary
|
||||
.tier_stats
|
||||
.insert(storageclass::STANDARD.to_string(), TierStats::default());
|
||||
summary.tier_stats.insert(storageclass::RRS.to_string(), TierStats::default());
|
||||
|
||||
apply_scanner_size_summary(&mut entry, &summary);
|
||||
|
||||
assert!(entry.all_tier_stats.is_none(), "zero-only tier maps must not allocate cache state");
|
||||
}
|
||||
|
||||
async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-test-{}", Uuid::new_v4()));
|
||||
tokio::fs::create_dir_all(&temp_dir)
|
||||
|
||||
@@ -1146,6 +1146,7 @@ fn completed_data_usage_info(
|
||||
versions_total_count: u64::try_from(total.versions).ok()?,
|
||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||
objects_total_size: u64::try_from(total.size).ok()?,
|
||||
tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(all_buckets.len()).ok()?,
|
||||
bucket_sizes,
|
||||
buckets_usage,
|
||||
@@ -1250,6 +1251,57 @@ mod publish_gate_tests {
|
||||
completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_publishes_tier_stats_across_sets() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()];
|
||||
let warm = |total_size, num_versions, num_objects| {
|
||||
HashMap::from([(
|
||||
"WARM".to_string(),
|
||||
TierStats {
|
||||
total_size,
|
||||
num_versions,
|
||||
num_objects,
|
||||
},
|
||||
)])
|
||||
};
|
||||
|
||||
let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
let mut tiered = DataUsageEntry::default();
|
||||
tiered.add_tier_sizes(&warm(100, 2, 1));
|
||||
first_set.replace("bucket-b", DATA_USAGE_ROOT, tiered);
|
||||
|
||||
let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0));
|
||||
let mut tiered = DataUsageEntry::default();
|
||||
tiered.add_tier_sizes(&warm(50, 1, 1));
|
||||
second_set.replace("bucket-a", DATA_USAGE_ROOT, tiered);
|
||||
|
||||
let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false)
|
||||
.expect("completed sets should publish a snapshot");
|
||||
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.tier_stats
|
||||
.expect("tier usage should reach the snapshot")
|
||||
.tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 150,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() {
|
||||
let all_buckets = vec!["bucket-a".to_string()];
|
||||
let set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0));
|
||||
|
||||
let (data_usage_info, _) = completed_data_usage_info_for_test(&[set], &all_buckets, false, false)
|
||||
.expect("completed set should publish a snapshot");
|
||||
|
||||
assert!(data_usage_info.tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_data_usage_info_requires_every_set_before_publish() {
|
||||
let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()];
|
||||
|
||||
Reference in New Issue
Block a user