mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-19 11:06:17 +00:00
refactor(data-usage): own SizeSummary once, with the scanner's semantics (#6237)
`SizeSummary` and `ReplTargetSizeSummary` existed in both `rustfs-data-usage` and `rustfs-scanner`, and the two copies had drifted three ways: four size fields were `usize` in one and `i64` in the other, only the scanner's carried `tier_stats`, and — the difference that matters — the scanner's `add` saturated while the data-usage copy used plain `+=`, which panics on overflow in a debug build and wraps in a release one. The data-usage copy is now the only definition and takes the scanner's shape and semantics, since that is the side a test already pinned (`MAX + 1 == MAX`). An equivalent saturation test now guards it in its new home. The scanner re-exports both types alongside the ones it already re-exported. `DataUsageEntry::add_sizes` and `BucketUsageInfo::add_size_summary` are removed. Both took a `SizeSummary` and had no callers anywhere — they were the duplicate fold paths, and `apply_scanner_size_summary` is now the only one. `actions_accounting` stays in the scanner as the `ScannerSizeSummaryExt` extension trait: it needs `ObjectInfo`, which sits above `rustfs-data-usage`, and an inherent impl on a foreign type is not allowed. The three call sites are unchanged. Refs backlog#1828
This commit is contained in:
@@ -29,7 +29,7 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||
pub use rustfs_data_usage::{
|
||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
|
||||
PrefixUsageQuery, PrefixUsageSummary, TierStats, hash_path, prefix_usage_in_cache,
|
||||
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache,
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
@@ -188,38 +188,18 @@ pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
|
||||
|
||||
const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024;
|
||||
|
||||
/// Size summary for a single object or group of objects
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct SizeSummary {
|
||||
/// Total size
|
||||
pub total_size: usize,
|
||||
/// Number of versions
|
||||
pub versions: usize,
|
||||
/// Number of delete markers
|
||||
pub delete_markers: usize,
|
||||
/// Replicated size
|
||||
pub replicated_size: i64,
|
||||
/// Replicated count
|
||||
pub replicated_count: usize,
|
||||
/// Pending size
|
||||
pub pending_size: i64,
|
||||
/// Failed size
|
||||
pub failed_size: i64,
|
||||
/// Replica size
|
||||
pub replica_size: i64,
|
||||
/// Replica count
|
||||
pub replica_count: usize,
|
||||
/// Pending count
|
||||
pub pending_count: usize,
|
||||
/// Failed count
|
||||
pub failed_count: usize,
|
||||
/// Replication target stats
|
||||
pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
|
||||
pub tier_stats: HashMap<String, TierStats>,
|
||||
/// Scanner-side accounting on the shared [`SizeSummary`].
|
||||
///
|
||||
/// The type itself lives in `rustfs-data-usage`, which sits below the storage
|
||||
/// layer and cannot see `ObjectInfo`, so this stays an extension trait rather
|
||||
/// than an inherent method (backlog#1828).
|
||||
pub trait ScannerSizeSummaryExt {
|
||||
/// Fold one object's contribution into the summary, including its tier.
|
||||
fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64);
|
||||
}
|
||||
|
||||
impl SizeSummary {
|
||||
pub fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) {
|
||||
impl ScannerSizeSummaryExt for SizeSummary {
|
||||
fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) {
|
||||
if oi.delete_marker {
|
||||
self.delete_markers = self.delete_markers.saturating_add(1);
|
||||
return;
|
||||
@@ -251,23 +231,6 @@ impl SizeSummary {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication target size summary
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ReplTargetSizeSummary {
|
||||
/// Replicated size
|
||||
pub replicated_size: i64,
|
||||
/// Replicated count
|
||||
pub replicated_count: usize,
|
||||
/// Pending size
|
||||
pub pending_size: i64,
|
||||
/// Failed size
|
||||
pub failed_size: i64,
|
||||
/// Pending count
|
||||
pub pending_count: usize,
|
||||
/// Failed count
|
||||
pub failed_count: usize,
|
||||
}
|
||||
|
||||
// ===== Cache-related data structures =====
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -1544,39 +1507,6 @@ pub trait DataUsageCacheStorage {
|
||||
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
}
|
||||
|
||||
impl SizeSummary {
|
||||
/// Create a new SizeSummary
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Add another SizeSummary to this one
|
||||
pub fn add(&mut self, other: &SizeSummary) {
|
||||
self.total_size = self.total_size.saturating_add(other.total_size);
|
||||
self.versions = self.versions.saturating_add(other.versions);
|
||||
self.delete_markers = self.delete_markers.saturating_add(other.delete_markers);
|
||||
self.replicated_size = self.replicated_size.saturating_add(other.replicated_size);
|
||||
self.replicated_count = self.replicated_count.saturating_add(other.replicated_count);
|
||||
self.pending_size = self.pending_size.saturating_add(other.pending_size);
|
||||
self.failed_size = self.failed_size.saturating_add(other.failed_size);
|
||||
self.replica_size = self.replica_size.saturating_add(other.replica_size);
|
||||
self.replica_count = self.replica_count.saturating_add(other.replica_count);
|
||||
self.pending_count = self.pending_count.saturating_add(other.pending_count);
|
||||
self.failed_count = self.failed_count.saturating_add(other.failed_count);
|
||||
|
||||
// Merge replication target stats
|
||||
for (target, stats) in &other.repl_target_stats {
|
||||
let entry = self.repl_target_stats.entry(target.clone()).or_default();
|
||||
entry.replicated_size = entry.replicated_size.saturating_add(stats.replicated_size);
|
||||
entry.replicated_count = entry.replicated_count.saturating_add(stats.replicated_count);
|
||||
entry.pending_size = entry.pending_size.saturating_add(stats.pending_size);
|
||||
entry.failed_size = entry.failed_size.saturating_add(stats.failed_size);
|
||||
entry.pending_count = entry.pending_count.saturating_add(stats.pending_count);
|
||||
entry.failed_count = entry.failed_count.saturating_add(stats.failed_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -21,7 +21,7 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
use crate::ReplTargetSizeSummary;
|
||||
use crate::data_usage_define::{
|
||||
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageScanCheckpoint,
|
||||
DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, SizeSummary, hash_path,
|
||||
DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeSummary, hash_path,
|
||||
};
|
||||
use crate::error::ScannerError;
|
||||
use crate::runtime_config::{
|
||||
|
||||
Reference in New Issue
Block a user