From 728b0488c4a1a55656ccac1b20fbfb1436d2119f Mon Sep 17 00:00:00 2001 From: overtrue Date: Fri, 14 Aug 2026 09:02:23 +0800 Subject: [PATCH] chore(ecstore): drop the data_usage dead_code blanket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removing the blanket exposes eighteen items. Six are deleted; the rest belong to one feature that was never wired up. crates/ecstore/src/data_usage/local_snapshot.rs and its aggregation entry point, aggregate_local_snapshots, form a complete per-disk usage-snapshot feature: read/write of snapshot files under the metadata bucket, cross-disk aggregation, and tests. Nothing calls it. It landed in #5307 on 2026-07-27 and git log -S over the whole history shows the entry point has never had a caller; the live data-usage path is load_data_usage_from_backend / store_data_usage_in_backend. Rather than delete a recent, tested feature on cleanup grounds, the module gets a header explaining the situation and its items carry individual allows, so the gap stays greppable without reintroducing a blanket. One commit flips this to deletion if that is preferred. Deleted: - DATA_USAGE_BLOOM_NAME together with DATA_USAGE_BLOOM_NAME_PATH. Both are a dead duplicate of the pair in crates/scanner/src/data_usage_define.rs, which has roughly fifty consumers across scanner.rs and remote_scanner.rs; the ecstore copies have none. - increment_bucket_usage_memory and decrement_bucket_usage_memory, thin wrappers over the live record_bucket_object_write_memory and record_bucket_object_delete_memory. - sync_memory_cache_with_backend, whose doc comment says it is "called by scanner" — nothing calls it. - create_cache_entry_from_summary and cache_to_data_usage_info, neither with a consumer in any lane. resolve_loaded_snapshot is kept with an allow: nine tests in the same file assert its primary/backup fallback. Verification, four lanes warning-free: default, --tests, --features rio-v2 --tests, --features test-util --tests. rustfs-scanner still compiles clean. cargo nextest run -p rustfs-ecstore 4041 passed; clippy --lib --tests -D warnings clean; make pre-commit exit 0. Ref rustfs/backlog#1823 (step 2). --- .../ecstore/src/data_usage/local_snapshot.rs | 19 +++ crates/ecstore/src/data_usage/mod.rs | 118 +++--------------- 2 files changed, 33 insertions(+), 104 deletions(-) diff --git a/crates/ecstore/src/data_usage/local_snapshot.rs b/crates/ecstore/src/data_usage/local_snapshot.rs index 0ed2b6e1b..8262bee95 100644 --- a/crates/ecstore/src/data_usage/local_snapshot.rs +++ b/crates/ecstore/src/data_usage/local_snapshot.rs @@ -12,6 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Per-disk usage snapshots persisted under the metadata bucket. +//! +//! **Nothing calls into this module.** It landed complete with tests in #5307 +//! (2026-07-27) and its aggregation entry point, +//! [`crate::data_usage::aggregate_local_snapshots`], has never had a caller in +//! the tree's history. The live data-usage path is +//! `load_data_usage_from_backend` / `store_data_usage_in_backend`. The items +//! below therefore carry individual `dead_code` allows rather than a module +//! blanket, so the gap stays greppable until it is either wired up or removed. + use crate::data_usage::BucketUsageInfo; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result}; @@ -26,10 +36,12 @@ pub const DATA_USAGE_DIR: &str = "datausage"; /// Directory used to store incremental scan state files under the metadata bucket. pub const DATA_USAGE_STATE_DIR: &str = "datausage/state"; /// Snapshot file format version, allows forward compatibility if the structure evolves. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub const LOCAL_USAGE_SNAPSHOT_VERSION: u32 = 1; /// Additional metadata describing which disk produced the snapshot. #[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub struct LocalUsageSnapshotMeta { /// Disk UUID stored as a string for simpler serialization. pub disk_id: String, @@ -43,6 +55,7 @@ pub struct LocalUsageSnapshotMeta { /// Usage snapshot produced by a single disk. #[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub struct LocalUsageSnapshot { /// Format version recorded in the snapshot. pub format_version: u32, @@ -64,6 +77,7 @@ pub struct LocalUsageSnapshot { pub objects_total_size: u64, } +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] impl LocalUsageSnapshot { /// Create an empty snapshot with the default format version filled in. pub fn new(meta: LocalUsageSnapshotMeta) -> Self { @@ -99,11 +113,13 @@ impl LocalUsageSnapshot { } /// Build the snapshot file name `.json`. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_file_name(disk_id: &str) -> String { format!("{disk_id}.json") } /// Build the object path relative to `RUSTFS_META_BUCKET`, e.g. `datausage/.json`. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_object_path(disk_id: &str) -> String { format!("{}/{}", DATA_USAGE_DIR, snapshot_file_name(disk_id)) } @@ -119,11 +135,13 @@ pub fn data_usage_state_dir(root: &Path) -> PathBuf { } /// Build the absolute path to the snapshot file for the provided disk ID. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub fn snapshot_path(root: &Path, disk_id: &str) -> PathBuf { data_usage_dir(root).join(snapshot_file_name(disk_id)) } /// Read a snapshot from disk if it exists. +#[allow(dead_code, reason = "unwired local usage-snapshot feature; see module docs (backlog#1823)")] pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result> { let path = snapshot_path(root, disk_id); match fs::read(&path).await { @@ -138,6 +156,7 @@ pub async fn read_snapshot(root: &Path, disk_id: &str) -> Result Result<()> { let dir = data_usage_dir(root); fs::create_dir_all(&dir).await.map_err(Error::other)?; diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 41495f068..31827c25c 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. // #730: scanner/data-usage state is partially migrated and still owns staged cache helpers. -#![allow(dead_code)] pub mod local_snapshot; @@ -34,8 +33,8 @@ use crate::{ pub use local_snapshot::{LocalUsageSnapshot, read_snapshot as read_local_snapshot, snapshot_path}; use rustfs_data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, CompressionTotalInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, SizeSummary, - VersionsHistogram, observed_data_usage_is_newer, + DataUsageCache, DataUsageInfo, DiskUsageStatus, LEGACY_DATA_USAGE_OBJECT_NAME, SizeHistogram, VersionsHistogram, + observed_data_usage_is_newer, }; use rustfs_io_metrics::record_system_path_failure; use rustfs_utils::path::SLASH_SEPARATOR; @@ -55,7 +54,6 @@ use tracing::{debug, error, info, instrument}; // Data usage storage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json"; -const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; const DATA_USAGE_CACHE_TTL_SECS: u64 = 30; const LIVE_BUCKET_USAGE_MAX_ENTRIES: u64 = 1024; @@ -313,11 +311,6 @@ lazy_static::lazy_static! { LEGACY_DATA_USAGE_OBJECT_NAME ); static ref LEGACY_DATA_USAGE_OBJ_BACKUP_PATH: String = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); - pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", - crate::disk::BUCKET_META_PREFIX, - SLASH_SEPARATOR, - DATA_USAGE_BLOOM_NAME - ); pub static ref DATA_COMPRESSION_TOTAL_NAME_PATH: String = format!("{}{}{}", crate::disk::BUCKET_META_PREFIX, SLASH_SEPARATOR, @@ -858,6 +851,10 @@ async fn resolve_loaded_snapshot_pair_with_source( } } +#[allow( + dead_code, + reason = "primary/backup snapshot fallback asserted by this file's tests (backlog#1823)" +)] async fn resolve_loaded_snapshot( primary: Result, Error>, backup: impl Future, Error>>, @@ -1187,6 +1184,10 @@ pub async fn invalidate_admin_data_usage_snapshot_cache() { } /// Aggregate usage information from local disk snapshots. +#[allow( + dead_code, + reason = "reached only through aggregate_local_snapshots, which has no caller (backlog#1823)" +)] fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option) { if let Some(update) = snapshot.last_update && latest_update.is_none_or(|current| update > current) @@ -1220,6 +1221,10 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh } } +#[allow( + dead_code, + reason = "entry point of the local usage-snapshot feature, which has had no caller since it landed in #5307 (backlog#1823)" +)] pub async fn aggregate_local_snapshots(store: Arc) -> Result<(Vec, DataUsageInfo), Error> { let mut aggregated = DataUsageInfo::default(); let mut latest_update: Option = None; @@ -1742,11 +1747,6 @@ pub async fn record_bucket_object_write_unknown_previous_memory(bucket: &str, ne entry.pending_scanner_position = None; } -/// Fast in-memory increment for immediate quota consistency. -pub async fn increment_bucket_usage_memory(bucket: &str, size_increment: u64) { - record_bucket_object_write_memory(bucket, None, size_increment).await; -} - /// Fast in-memory update for successful object deletes. pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { ensure_bucket_usage_cached(bucket).await; @@ -1789,11 +1789,6 @@ pub async fn record_bucket_delete_marker_memory(bucket: &str) { entry.pending_scanner_position = None; } -/// Fast in-memory decrement for immediate quota consistency -pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) { - record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await; -} - /// Get bucket usage from the authoritative cache for this topology. async fn get_persisted_bucket_usage(bucket: &str) -> Option { let store = runtime_sources::object_store_handle()?; @@ -1988,91 +1983,6 @@ pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageIn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info, authoritative).await; } -/// Sync memory cache with backend data (called by scanner) -pub async fn sync_memory_cache_with_backend() -> Result<(), Error> { - if let Some(store) = runtime_sources::object_store_handle() { - match load_data_usage_from_backend(store.clone()).await { - Ok(data_usage_info) => { - replace_bucket_usage_memory_from_info(&data_usage_info).await; - } - Err(e) => { - debug!("Failed to sync memory cache with backend: {}", e); - } - } - } - Ok(()) -} - -/// Create a data usage cache entry from size summary -pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry { - let mut entry = DataUsageEntry::default(); - entry.add_sizes(summary); - entry -} - -/// Convert data usage cache to DataUsageInfo -pub fn cache_to_data_usage_info( - cache: &DataUsageCache, - path: &str, - buckets: &[crate::storage_api_contracts::bucket::BucketInfo], -) -> DataUsageInfo { - let e = match cache.find(path) { - Some(e) => e, - None => return DataUsageInfo::default(), - }; - let flat = cache.flatten(&e); - - let mut buckets_usage = HashMap::new(); - for bucket in buckets.iter() { - let e = match cache.find(&bucket.name) { - Some(e) => e, - None => continue, - }; - let flat = cache.flatten(&e); - let mut bui = BucketUsageInfo { - size: flat.size as u64, - versions_count: flat.versions as u64, - objects_count: flat.objects as u64, - delete_markers_count: flat.delete_markers as u64, - object_size_histogram: flat.obj_sizes.to_map(), - object_versions_histogram: flat.obj_versions.to_map(), - ..Default::default() - }; - - if let Some(rs) = &flat.replication_stats { - bui.replica_size = rs.replica_size; - bui.replica_count = rs.replica_count; - - for (arn, stat) in rs.targets.iter() { - bui.replication_info.insert( - arn.clone(), - BucketTargetUsageInfo { - replication_pending_size: stat.pending_size, - replicated_size: stat.replicated_size, - replication_failed_size: stat.failed_size, - replication_pending_count: stat.pending_count, - replication_failed_count: stat.failed_count, - replicated_count: stat.replicated_count, - ..Default::default() - }, - ); - } - } - buckets_usage.insert(bucket.name.clone(), bui); - } - - DataUsageInfo { - last_update: cache.info.last_update, - objects_total_count: flat.objects as u64, - versions_total_count: flat.versions as u64, - delete_markers_total_count: flat.delete_markers as u64, - objects_total_size: flat.size as u64, - buckets_count: e.children.len() as u64, - buckets_usage, - ..Default::default() - } -} - // Helper functions for DataUsageCache operations pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result { use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};