diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e3bd98ed4..de8c4a1b1 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -69,7 +69,8 @@ pub mod config { pub mod data_usage { pub use crate::data_usage::{ DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, load_data_usage_from_backend, - record_bucket_object_delete_memory, record_bucket_object_write_memory, replace_bucket_usage_memory_from_info, + record_bucket_object_delete_memory, record_bucket_object_write_memory, remove_bucket_usage_from_backend, + replace_bucket_usage_memory_from_info, }; } diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index 915f42a67..4c1c85e23 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -101,6 +101,10 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: return Ok(()); } + save_data_usage_in_backend(data_usage_info, store).await +} + +async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc) -> Result<(), Error> { let data = serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; @@ -112,6 +116,61 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Ok(()) } +fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) { + data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX); +} + +fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &str) -> bool { + if bucket.is_empty() { + return false; + } + + let removed_usage = data_usage_info.buckets_usage.remove(bucket).is_some(); + let removed_size = data_usage_info.bucket_sizes.remove(bucket).is_some(); + + if !removed_usage && !removed_size { + return false; + } + + set_buckets_count_from_usage(data_usage_info); + data_usage_info.calculate_totals(); + true +} + +fn merge_bucket_usage_removal(candidate: DataUsageInfo, existing: Option, bucket: &str) -> Option { + let mut data_usage_info = match existing { + Some(existing) if data_usage_info_updated_at(&existing) >= data_usage_info_updated_at(&candidate) => existing, + _ => candidate, + }; + + if remove_bucket_usage_from_info(&mut data_usage_info, bucket) { + Some(data_usage_info) + } else { + None + } +} + +async fn clear_bucket_usage_memory(bucket: &str) { + if bucket.is_empty() { + return; + } + + memory_cache().write().await.remove(bucket); +} + +pub async fn remove_bucket_usage_from_backend(store: Arc, bucket: &str) -> Result<(), Error> { + clear_bucket_usage_memory(bucket).await; + + let data_usage_info = load_data_usage_from_backend(store.clone()).await?; + let existing = load_data_usage_from_backend(store.clone()).await.ok(); + + if let Some(data_usage_info) = merge_bucket_usage_removal(data_usage_info, existing, bucket) { + save_data_usage_in_backend(data_usage_info, store).await?; + } + + Ok(()) +} + /// Load data usage info from backend storage #[instrument(skip(store))] pub async fn load_data_usage_from_backend(store: Arc) -> Result { @@ -949,6 +1008,96 @@ mod tests { ); } + #[test] + fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() { + let last_update = SystemTime::now(); + let mut info = data_usage_info_for_test("bucket-a", 2, 84, last_update); + info.buckets_usage.insert( + "bucket-b".to_string(), + BucketUsageInfo { + objects_count: 3, + versions_count: 3, + size: 126, + ..Default::default() + }, + ); + info.bucket_sizes.insert("bucket-b".to_string(), 126); + info.buckets_count = 2; + info.calculate_totals(); + + assert!(remove_bucket_usage_from_info(&mut info, "bucket-a")); + + assert_eq!(info.buckets_count, 1); + assert_eq!(info.objects_total_count, 3); + assert_eq!(info.objects_total_size, 126); + assert_eq!(info.last_update, Some(last_update)); + assert!(!info.buckets_usage.contains_key("bucket-a")); + assert!(!info.bucket_sizes.contains_key("bucket-a")); + assert_eq!( + info.buckets_usage + .get("bucket-b") + .map(|usage| (usage.objects_count, usage.size)), + Some((3, 126)) + ); + } + + #[test] + fn merge_bucket_usage_removal_preserves_current_snapshot() { + let now = SystemTime::now(); + let candidate = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10)); + let mut existing = data_usage_info_for_test("bucket-a", 4, 168, now); + existing.buckets_usage.insert( + "bucket-c".to_string(), + BucketUsageInfo { + objects_count: 5, + versions_count: 5, + size: 210, + ..Default::default() + }, + ); + existing.bucket_sizes.insert("bucket-c".to_string(), 210); + existing.buckets_count = 2; + existing.calculate_totals(); + + let merged = merge_bucket_usage_removal(candidate, Some(existing), "bucket-a") + .expect("bucket-a should be removed from the current snapshot"); + + assert_eq!(merged.last_update, Some(now)); + assert_eq!(merged.buckets_count, 1); + assert_eq!(merged.objects_total_count, 5); + assert_eq!(merged.objects_total_size, 210); + assert!(!merged.buckets_usage.contains_key("bucket-a")); + assert_eq!( + merged + .buckets_usage + .get("bucket-c") + .map(|usage| (usage.objects_count, usage.size)), + Some((5, 210)) + ); + } + + #[tokio::test] + #[serial] + async fn clear_bucket_usage_memory_prevents_deleted_bucket_overlay() { + clear_usage_memory_cache_for_test().await; + + let persisted = data_usage_info_for_test("bucket-a", 1, 42, SystemTime::now() - Duration::from_secs(10)); + replace_bucket_usage_memory_from_info(&persisted).await; + record_bucket_object_delete_memory("bucket-a", 42, true).await; + clear_bucket_usage_memory("bucket-a").await; + + let mut response = DataUsageInfo { + last_update: Some(SystemTime::now()), + ..Default::default() + }; + apply_bucket_usage_memory_overlay(&mut response).await; + + assert_eq!(response.buckets_count, 0); + assert_eq!(response.objects_total_count, 0); + assert!(!response.buckets_usage.contains_key("bucket-a")); + assert!(!response.bucket_sizes.contains_key("bucket-a")); + } + #[tokio::test] #[serial] async fn memory_overlay_preserves_object_count_for_overwrite() { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index ab796969f..73c7bcb1d 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -839,6 +839,9 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; rustfs_scanner::clear_dirty_usage_bucket(&input.bucket); + if let Err(err) = super::remove_bucket_usage_from_backend(store.clone(), &input.bucket).await { + warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage"); + } if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await { warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed"); diff --git a/rustfs/src/app/mod.rs b/rustfs/src/app/mod.rs index f930e7fa3..06c8b6345 100644 --- a/rustfs/src/app/mod.rs +++ b/rustfs/src/app/mod.rs @@ -76,7 +76,7 @@ mod ecstore_config { mod ecstore_data_usage { pub(crate) use crate::storage::ecstore_data_usage::{ apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory, - record_bucket_object_write_memory, + record_bucket_object_write_memory, remove_bucket_usage_from_backend, }; } @@ -603,6 +603,10 @@ pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_cur ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await; } +pub(crate) async fn remove_bucket_usage_from_backend(store: Arc, bucket: &str) -> std::result::Result<(), Error> { + ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await +} + pub(crate) fn is_all_buckets_not_found(errs: &[Option]) -> bool { crate::storage::is_all_buckets_not_found(errs) } diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 81adfff77..d023e0ef0 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -101,7 +101,7 @@ pub(crate) mod ecstore_config { pub(crate) mod ecstore_data_usage { pub(crate) use rustfs_ecstore::api::data_usage::{ apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory, - record_bucket_object_write_memory, + record_bucket_object_write_memory, remove_bucket_usage_from_backend, }; }