fix(admin): refresh datausage live bucket usage (#4490)

* fix(admin): refresh datausage live bucket usage

* fix(admin): reapply datausage memory overlay

* fix(ecstore): avoid runtime lookup for empty shard costs

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>

crates/ecstore/src/set_disk/core/io_primitives.rs

@@ -613,6 +613,10 @@ pub(in crate::set_disk) fn resolve_read_part_from_responses(
 }
 
 pub(in crate::set_disk) fn shard_read_costs_for_disks(disks: &[Option<DiskStore>]) -> Vec<ShardReadCost> {
+    if disks.is_empty() {
+        return Vec::new();
+    }
+
     let local_endpoint_hosts = local_endpoint_hosts_for_shard_costs();
     disks
         .iter()

rustfs/src/app/admin_usecase.rs

@@ -19,9 +19,10 @@ use super::storage_api::admin_usecase::capacity::{
     PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
 };
 use super::storage_api::admin_usecase::contract::StorageAdminApi;
+use super::storage_api::admin_usecase::contract::bucket::{BucketOperations, BucketOptions};
 use super::storage_api::admin_usecase::data_usage::{
-    apply_bucket_usage_memory_overlay, load_data_usage_from_backend, refresh_versioned_bucket_usage_from_object_layer,
-    replace_bucket_usage_memory_from_info,
+    apply_bucket_usage_memory_overlay, load_data_usage_from_backend, refresh_bucket_usage_from_object_layer,
+    refresh_versioned_bucket_usage_from_object_layer, replace_bucket_usage_memory_from_info,
 };
 use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
 use crate::app::runtime_sources::{
@@ -251,6 +252,8 @@ impl DefaultAdminUsecase {
         refresh_versioned_bucket_usage_from_object_layer(store.clone(), &mut info).await;
         replace_bucket_usage_memory_from_info(&info).await;
         apply_bucket_usage_memory_overlay(&mut info).await;
+        Self::refresh_live_bucket_usage_for_data_usage_info(store.clone(), &mut info).await;
+        apply_bucket_usage_memory_overlay(&mut info).await;
 
         let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
 
@@ -320,6 +323,32 @@ impl DefaultAdminUsecase {
         Ok(info)
     }
 
+    async fn refresh_live_bucket_usage_for_data_usage_info(store: Arc<ECStore>, data_usage_info: &mut DataUsageInfo) {
+        let buckets = match store
+            .list_bucket(&BucketOptions {
+                no_metadata: true,
+                ..Default::default()
+            })
+            .await
+        {
+            Ok(buckets) => buckets,
+            Err(err) => {
+                debug!(error = %err, "failed to list buckets while refreshing data usage info");
+                return;
+            }
+        };
+
+        for bucket in buckets {
+            if let Err(err) = refresh_bucket_usage_from_object_layer(store.clone(), data_usage_info, &bucket.name).await {
+                debug!(
+                    bucket = %bucket.name,
+                    error = %err,
+                    "failed to refresh data usage info bucket usage from object layer"
+                );
+            }
+        }
+    }
+
     pub async fn execute_list_pool_statuses(&self) -> AdminUsecaseResult<Vec<PoolStatus>> {
         let Some(store) = self.object_store() else {
             return Err(Self::app_error(S3ErrorCode::InternalError, "Not init"));

rustfs/src/app/storage_api.rs

@@ -57,6 +57,19 @@ pub(crate) mod data_usage {
         crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await
     }
 
+    pub(crate) async fn refresh_bucket_usage_from_object_layer(
+        store: Arc<crate::storage::storage_api::ECStore>,
+        data_usage_info: &mut rustfs_data_usage::DataUsageInfo,
+        bucket_name: &str,
+    ) -> Result<rustfs_data_usage::BucketUsageInfo, crate::storage::storage_api::StorageError> {
+        crate::storage::storage_api::ecstore_data_usage::refresh_bucket_usage_from_object_layer(
+            store,
+            data_usage_info,
+            bucket_name,
+        )
+        .await
+    }
+
     pub(crate) async fn refresh_versioned_bucket_usage_from_object_layer(
         store: Arc<crate::storage::storage_api::ECStore>,
         data_usage_info: &mut rustfs_data_usage::DataUsageInfo,
@@ -913,6 +926,10 @@ pub(crate) mod s3_api {
 
 pub(crate) mod admin_usecase {
     pub(crate) mod contract {
+        pub(crate) mod bucket {
+            pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions};
+        }
+
         pub(crate) use super::super::storage_contracts::StorageAdminApi;
     }