refactor(data-usage): own SizeSummary once, with the scanner's semantics (#6237)

`SizeSummary` and `ReplTargetSizeSummary` existed in both `rustfs-data-usage` and `rustfs-scanner`, and the two copies had drifted three ways: four size fields were `usize` in one and `i64` in the other, only the scanner's carried `tier_stats`, and — the difference that matters — the scanner's `add` saturated while the data-usage copy used plain `+=`, which panics on overflow in a debug build and wraps in a release one.

The data-usage copy is now the only definition and takes the scanner's shape and semantics, since that is the side a test already pinned (`MAX + 1 == MAX`). An equivalent saturation test now guards it in its new home. The scanner re-exports both types alongside the ones it already re-exported.

`DataUsageEntry::add_sizes` and `BucketUsageInfo::add_size_summary` are removed. Both took a `SizeSummary` and had no callers anywhere — they were the duplicate fold paths, and `apply_scanner_size_summary` is now the only one.

`actions_accounting` stays in the scanner as the `ScannerSizeSummaryExt` extension trait: it needs `ObjectInfo`, which sits above `rustfs-data-usage`, and an inherent impl on a foreign type is not allowed. The three call sites are unchanged.

Refs backlog#1828
This commit is contained in:
Zhengchao An
2026-08-19 10:34:30 +08:00
committed by GitHub
parent e3d7892404
commit 09fe561443
3 changed files with 100 additions and 137 deletions
+11 -81
View File
@@ -29,7 +29,7 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
pub use rustfs_data_usage::{
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache,
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache,
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, Instant, sleep, timeout};
@@ -188,38 +188,18 @@ pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024;
/// Size summary for a single object or group of objects
#[derive(Debug, Default, Clone)]
pub struct SizeSummary {
/// Total size
pub total_size: usize,
/// Number of versions
pub versions: usize,
/// Number of delete markers
pub delete_markers: usize,
/// Replicated size
pub replicated_size: i64,
/// Replicated count
pub replicated_count: usize,
/// Pending size
pub pending_size: i64,
/// Failed size
pub failed_size: i64,
/// Replica size
pub replica_size: i64,
/// Replica count
pub replica_count: usize,
/// Pending count
pub pending_count: usize,
/// Failed count
pub failed_count: usize,
/// Replication target stats
pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
pub tier_stats: HashMap<String, TierStats>,
/// Scanner-side accounting on the shared [`SizeSummary`].
///
/// The type itself lives in `rustfs-data-usage`, which sits below the storage
/// layer and cannot see `ObjectInfo`, so this stays an extension trait rather
/// than an inherent method (backlog#1828).
pub trait ScannerSizeSummaryExt {
/// Fold one object's contribution into the summary, including its tier.
fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64);
}
impl SizeSummary {
pub fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) {
impl ScannerSizeSummaryExt for SizeSummary {
fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) {
if oi.delete_marker {
self.delete_markers = self.delete_markers.saturating_add(1);
return;
@@ -251,23 +231,6 @@ impl SizeSummary {
}
}
/// Replication target size summary
#[derive(Debug, Default, Clone)]
pub struct ReplTargetSizeSummary {
/// Replicated size
pub replicated_size: i64,
/// Replicated count
pub replicated_count: usize,
/// Pending size
pub pending_size: i64,
/// Failed size
pub failed_size: i64,
/// Pending count
pub pending_count: usize,
/// Failed count
pub failed_count: usize,
}
// ===== Cache-related data structures =====
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
@@ -1544,39 +1507,6 @@ pub trait DataUsageCacheStorage {
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
}
impl SizeSummary {
/// Create a new SizeSummary
pub fn new() -> Self {
Self::default()
}
/// Add another SizeSummary to this one
pub fn add(&mut self, other: &SizeSummary) {
self.total_size = self.total_size.saturating_add(other.total_size);
self.versions = self.versions.saturating_add(other.versions);
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
self.replicated_size = self.replicated_size.saturating_add(other.replicated_size);
self.replicated_count = self.replicated_count.saturating_add(other.replicated_count);
self.pending_size = self.pending_size.saturating_add(other.pending_size);
self.failed_size = self.failed_size.saturating_add(other.failed_size);
self.replica_size = self.replica_size.saturating_add(other.replica_size);
self.replica_count = self.replica_count.saturating_add(other.replica_count);
self.pending_count = self.pending_count.saturating_add(other.pending_count);
self.failed_count = self.failed_count.saturating_add(other.failed_count);
// Merge replication target stats
for (target, stats) in &other.repl_target_stats {
let entry = self.repl_target_stats.entry(target.clone()).or_default();
entry.replicated_size = entry.replicated_size.saturating_add(stats.replicated_size);
entry.replicated_count = entry.replicated_count.saturating_add(stats.replicated_count);
entry.pending_size = entry.pending_size.saturating_add(stats.pending_size);
entry.failed_size = entry.failed_size.saturating_add(stats.failed_size);
entry.pending_count = entry.pending_count.saturating_add(stats.pending_count);
entry.failed_count = entry.failed_count.saturating_add(stats.failed_count);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
+1 -1
View File
@@ -21,7 +21,7 @@ use std::time::{Duration, Instant, SystemTime};
use crate::ReplTargetSizeSummary;
use crate::data_usage_define::{
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageScanCheckpoint,
DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, SizeSummary, hash_path,
DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeSummary, hash_path,
};
use crate::error::ScannerError;
use crate::runtime_config::{