fix(scanner): clear deleted bucket usage stats (#3822)

* fix(scanner): clear deleted bucket usage stats

* fix(scanner): preserve usage timestamp on bucket cleanup

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>

crates/ecstore/src/api/mod.rs

@@ -69,7 +69,8 @@ pub mod config {
 pub mod data_usage {
     pub use crate::data_usage::{
         DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, load_data_usage_from_backend,
-        record_bucket_object_delete_memory, record_bucket_object_write_memory, replace_bucket_usage_memory_from_info,
+        record_bucket_object_delete_memory, record_bucket_object_write_memory, remove_bucket_usage_from_backend,
+        replace_bucket_usage_memory_from_info,
     };
 }
 

crates/ecstore/src/data_usage.rs

@@ -101,6 +101,10 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
         return Ok(());
     }
 
+    save_data_usage_in_backend(data_usage_info, store).await
+}
+
+async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
     let data =
         serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
 
@@ -112,6 +116,61 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
     Ok(())
 }
 
+fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) {
+    data_usage_info.buckets_count = u64::try_from(data_usage_info.buckets_usage.len()).unwrap_or(u64::MAX);
+}
+
+fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &str) -> bool {
+    if bucket.is_empty() {
+        return false;
+    }
+
+    let removed_usage = data_usage_info.buckets_usage.remove(bucket).is_some();
+    let removed_size = data_usage_info.bucket_sizes.remove(bucket).is_some();
+
+    if !removed_usage && !removed_size {
+        return false;
+    }
+
+    set_buckets_count_from_usage(data_usage_info);
+    data_usage_info.calculate_totals();
+    true
+}
+
+fn merge_bucket_usage_removal(candidate: DataUsageInfo, existing: Option<DataUsageInfo>, bucket: &str) -> Option<DataUsageInfo> {
+    let mut data_usage_info = match existing {
+        Some(existing) if data_usage_info_updated_at(&existing) >= data_usage_info_updated_at(&candidate) => existing,
+        _ => candidate,
+    };
+
+    if remove_bucket_usage_from_info(&mut data_usage_info, bucket) {
+        Some(data_usage_info)
+    } else {
+        None
+    }
+}
+
+async fn clear_bucket_usage_memory(bucket: &str) {
+    if bucket.is_empty() {
+        return;
+    }
+
+    memory_cache().write().await.remove(bucket);
+}
+
+pub async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
+    clear_bucket_usage_memory(bucket).await;
+
+    let data_usage_info = load_data_usage_from_backend(store.clone()).await?;
+    let existing = load_data_usage_from_backend(store.clone()).await.ok();
+
+    if let Some(data_usage_info) = merge_bucket_usage_removal(data_usage_info, existing, bucket) {
+        save_data_usage_in_backend(data_usage_info, store).await?;
+    }
+
+    Ok(())
+}
+
 /// Load data usage info from backend storage
 #[instrument(skip(store))]
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
@@ -949,6 +1008,96 @@ mod tests {
         );
     }
 
+    #[test]
+    fn remove_bucket_usage_from_info_drops_bucket_and_recomputes_totals() {
+        let last_update = SystemTime::now();
+        let mut info = data_usage_info_for_test("bucket-a", 2, 84, last_update);
+        info.buckets_usage.insert(
+            "bucket-b".to_string(),
+            BucketUsageInfo {
+                objects_count: 3,
+                versions_count: 3,
+                size: 126,
+                ..Default::default()
+            },
+        );
+        info.bucket_sizes.insert("bucket-b".to_string(), 126);
+        info.buckets_count = 2;
+        info.calculate_totals();
+
+        assert!(remove_bucket_usage_from_info(&mut info, "bucket-a"));
+
+        assert_eq!(info.buckets_count, 1);
+        assert_eq!(info.objects_total_count, 3);
+        assert_eq!(info.objects_total_size, 126);
+        assert_eq!(info.last_update, Some(last_update));
+        assert!(!info.buckets_usage.contains_key("bucket-a"));
+        assert!(!info.bucket_sizes.contains_key("bucket-a"));
+        assert_eq!(
+            info.buckets_usage
+                .get("bucket-b")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((3, 126))
+        );
+    }
+
+    #[test]
+    fn merge_bucket_usage_removal_preserves_current_snapshot() {
+        let now = SystemTime::now();
+        let candidate = data_usage_info_for_test("bucket-a", 2, 84, now - Duration::from_secs(10));
+        let mut existing = data_usage_info_for_test("bucket-a", 4, 168, now);
+        existing.buckets_usage.insert(
+            "bucket-c".to_string(),
+            BucketUsageInfo {
+                objects_count: 5,
+                versions_count: 5,
+                size: 210,
+                ..Default::default()
+            },
+        );
+        existing.bucket_sizes.insert("bucket-c".to_string(), 210);
+        existing.buckets_count = 2;
+        existing.calculate_totals();
+
+        let merged = merge_bucket_usage_removal(candidate, Some(existing), "bucket-a")
+            .expect("bucket-a should be removed from the current snapshot");
+
+        assert_eq!(merged.last_update, Some(now));
+        assert_eq!(merged.buckets_count, 1);
+        assert_eq!(merged.objects_total_count, 5);
+        assert_eq!(merged.objects_total_size, 210);
+        assert!(!merged.buckets_usage.contains_key("bucket-a"));
+        assert_eq!(
+            merged
+                .buckets_usage
+                .get("bucket-c")
+                .map(|usage| (usage.objects_count, usage.size)),
+            Some((5, 210))
+        );
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn clear_bucket_usage_memory_prevents_deleted_bucket_overlay() {
+        clear_usage_memory_cache_for_test().await;
+
+        let persisted = data_usage_info_for_test("bucket-a", 1, 42, SystemTime::now() - Duration::from_secs(10));
+        replace_bucket_usage_memory_from_info(&persisted).await;
+        record_bucket_object_delete_memory("bucket-a", 42, true).await;
+        clear_bucket_usage_memory("bucket-a").await;
+
+        let mut response = DataUsageInfo {
+            last_update: Some(SystemTime::now()),
+            ..Default::default()
+        };
+        apply_bucket_usage_memory_overlay(&mut response).await;
+
+        assert_eq!(response.buckets_count, 0);
+        assert_eq!(response.objects_total_count, 0);
+        assert!(!response.buckets_usage.contains_key("bucket-a"));
+        assert!(!response.bucket_sizes.contains_key("bucket-a"));
+    }
+
     #[tokio::test]
     #[serial]
     async fn memory_overlay_preserves_object_count_for_overwrite() {

rustfs/src/app/bucket_usecase.rs

@@ -839,6 +839,9 @@ impl DefaultBucketUsecase {
             .await
             .map_err(ApiError::from)?;
         rustfs_scanner::clear_dirty_usage_bucket(&input.bucket);
+        if let Err(err) = super::remove_bucket_usage_from_backend(store.clone(), &input.bucket).await {
+            warn!(bucket = %input.bucket, error = ?err, "failed to remove deleted bucket from data usage");
+        }
 
         if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
             warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");

rustfs/src/app/mod.rs

@@ -76,7 +76,7 @@ mod ecstore_config {
 mod ecstore_data_usage {
     pub(crate) use crate::storage::ecstore_data_usage::{
         apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
-        record_bucket_object_write_memory,
+        record_bucket_object_write_memory, remove_bucket_usage_from_backend,
     };
 }
 
@@ -603,6 +603,10 @@ pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_cur
     ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await;
 }
 
+pub(crate) async fn remove_bucket_usage_from_backend(store: Arc<ECStore>, bucket: &str) -> std::result::Result<(), Error> {
+    ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await
+}
+
 pub(crate) fn is_all_buckets_not_found(errs: &[Option<DiskError>]) -> bool {
     crate::storage::is_all_buckets_not_found(errs)
 }

rustfs/src/storage/mod.rs

@@ -101,7 +101,7 @@ pub(crate) mod ecstore_config {
 pub(crate) mod ecstore_data_usage {
     pub(crate) use rustfs_ecstore::api::data_usage::{
         apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
-        record_bucket_object_write_memory,
+        record_bucket_object_write_memory, remove_bucket_usage_from_backend,
     };
 }