Fix: fix collect usage data (#500)

Signed-off-by: junxiang Mu <1948535941@qq.com>

crates/ahm/src/scanner/data_scanner.rs

@@ -840,6 +840,16 @@ impl Scanner {
             warn!("Failed to save checkpoint: {}", e);
         }
 
+        // Always trigger data usage collection during scan cycle
+        let config = self.config.read().await;
+        if config.enable_data_usage_stats {
+            info!("Data usage stats enabled, collecting data");
+            if let Err(e) = self.collect_and_persist_data_usage().await {
+                error!("Failed to collect data usage during scan cycle: {}", e);
+            }
+        }
+        drop(config);
+
         // Get aggregated statistics from all nodes
         debug!("About to get aggregated stats");
         match self.stats_aggregator.get_aggregated_stats().await {
@@ -920,6 +930,126 @@ impl Scanner {
         Ok(())
     }
 
+    /// Collect and persist data usage statistics
+    async fn collect_and_persist_data_usage(&self) -> Result<()> {
+        info!("Starting data usage collection and persistence");
+
+        // Get ECStore instance
+        let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() else {
+            warn!("ECStore not available for data usage collection");
+            return Ok(());
+        };
+
+        // Collect data usage from NodeScanner stats
+        let _local_stats = self.node_scanner.get_stats_summary().await;
+
+        // Build data usage from ECStore directly for now
+        let data_usage = self.build_data_usage_from_ecstore(&ecstore).await?;
+
+        // Update NodeScanner with collected data
+        self.node_scanner.update_data_usage(data_usage.clone()).await;
+
+        // Store to local cache
+        {
+            let mut data_usage_guard = self.data_usage_stats.lock().await;
+            data_usage_guard.insert("consolidated".to_string(), data_usage.clone());
+        }
+
+        // Update last collection time
+        {
+            let mut last_collection = self.last_data_usage_collection.write().await;
+            *last_collection = Some(SystemTime::now());
+        }
+
+        // Persist to backend asynchronously
+        let data_clone = data_usage.clone();
+        let store_clone = ecstore.clone();
+        tokio::spawn(async move {
+            if let Err(e) = store_data_usage_in_backend(data_clone, store_clone).await {
+                error!("Failed to persist data usage to backend: {}", e);
+            } else {
+                info!("Successfully persisted data usage to backend");
+            }
+        });
+
+        info!(
+            "Data usage collection completed: {} buckets, {} objects",
+            data_usage.buckets_count, data_usage.objects_total_count
+        );
+
+        Ok(())
+    }
+
+    /// Build data usage statistics directly from ECStore
+    async fn build_data_usage_from_ecstore(&self, ecstore: &Arc<rustfs_ecstore::store::ECStore>) -> Result<DataUsageInfo> {
+        let mut data_usage = DataUsageInfo::default();
+
+        // Get bucket list
+        match ecstore
+            .list_bucket(&rustfs_ecstore::store_api::BucketOptions::default())
+            .await
+        {
+            Ok(buckets) => {
+                data_usage.buckets_count = buckets.len() as u64;
+                data_usage.last_update = Some(SystemTime::now());
+
+                let mut total_objects = 0u64;
+                let mut total_size = 0u64;
+
+                for bucket_info in buckets {
+                    if bucket_info.name.starts_with('.') {
+                        continue; // Skip system buckets
+                    }
+
+                    // Try to get actual object count for this bucket
+                    let (object_count, bucket_size) = match ecstore
+                        .clone()
+                        .list_objects_v2(
+                            &bucket_info.name,
+                            "",    // prefix
+                            None,  // continuation_token
+                            None,  // delimiter
+                            100,   // max_keys - small limit for performance
+                            false, // fetch_owner
+                            None,  // start_after
+                        )
+                        .await
+                    {
+                        Ok(result) => {
+                            let count = result.objects.len() as u64;
+                            let size = result.objects.iter().map(|obj| obj.size as u64).sum();
+                            (count, size)
+                        }
+                        Err(_) => (0, 0),
+                    };
+
+                    total_objects += object_count;
+                    total_size += bucket_size;
+
+                    let bucket_usage = rustfs_common::data_usage::BucketUsageInfo {
+                        size: bucket_size,
+                        objects_count: object_count,
+                        versions_count: object_count, // Simplified
+                        delete_markers_count: 0,
+                        ..Default::default()
+                    };
+
+                    data_usage.buckets_usage.insert(bucket_info.name.clone(), bucket_usage);
+                    data_usage.bucket_sizes.insert(bucket_info.name, bucket_size);
+                }
+
+                data_usage.objects_total_count = total_objects;
+                data_usage.objects_total_size = total_size;
+                data_usage.versions_total_count = total_objects;
+            }
+            Err(e) => {
+                warn!("Failed to list buckets for data usage collection: {}", e);
+            }
+        }
+
+        Ok(data_usage)
+    }
+
     /// Verify object integrity and trigger healing if necessary
     #[allow(dead_code)]
     async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<()> {

crates/ahm/src/scanner/node_scanner.rs

@@ -251,6 +251,8 @@ pub struct ScanProgress {
     /// estimated completion time
     #[serde(with = "option_system_time_serde")]
     pub estimated_completion: Option<SystemTime>,
+    /// data usage statistics
+    pub data_usage: Option<DataUsageInfo>,
 }
 
 impl Default for ScanProgress {
@@ -265,6 +267,7 @@ impl Default for ScanProgress {
             last_scan_key: None,
             scan_start_time: SystemTime::now(),
             estimated_completion: None,
+            data_usage: None,
         }
     }
 }

crates/ahm/tests/heal_integration_test.rs

@@ -140,13 +140,16 @@ async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str,
     info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_heal_object_basic() {
+mod serial_tests {
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_heal_object_basic() {
         let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
 
         // Create test bucket and object
-    let bucket_name = "test-bucket";
+        let bucket_name = "test-heal-object-basic";
         let object_name = "test-object.txt";
         let test_data = b"Hello, this is test data for healing!";
 
@@ -218,15 +221,15 @@ async fn test_heal_object_basic() {
         assert!(target_part.exists());
 
         info!("Heal object basic test passed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_heal_bucket_basic() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_heal_bucket_basic() {
         let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
 
         // Create test bucket
-    let bucket_name = "test-bucket-heal";
+        let bucket_name = "test-heal-bucket-basic";
         create_test_bucket(&ecstore, bucket_name).await;
 
         // ─── 1️⃣ delete bucket dir on disk ──────────────
@@ -286,11 +289,11 @@ async fn test_heal_bucket_basic() {
         assert!(broken_bucket_path.exists(), "bucket dir does not exist on disk");
 
         info!("Heal bucket basic test passed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_heal_format_basic() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_heal_format_basic() {
         let (disk_paths, _ecstore, heal_storage) = setup_test_env().await;
 
         // ─── 1️⃣ delete format.json on one disk ──────────────
@@ -315,15 +318,15 @@ async fn test_heal_format_basic() {
         assert!(format_path.exists(), "format.json does not exist on disk after heal");
 
         info!("Heal format basic test passed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_heal_format_with_data() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_heal_format_with_data() {
         let (disk_paths, ecstore, heal_storage) = setup_test_env().await;
 
         // Create test bucket and object
-    let bucket_name = "test-bucket";
+        let bucket_name = "test-heal-format-with-data";
         let object_name = "test-object.txt";
         let test_data = b"Hello, this is test data for healing!";
 
@@ -363,11 +366,11 @@ async fn test_heal_format_with_data() {
         assert!(target_part.exists());
 
         info!("Heal format basic test passed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_heal_storage_api_direct() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_heal_storage_api_direct() {
         let (_disk_paths, ecstore, heal_storage) = setup_test_env().await;
 
         // Test direct heal storage API calls
@@ -421,4 +424,5 @@ async fn test_heal_storage_api_direct() {
         info!("Direct heal_object test passed");
 
         info!("Direct heal storage API test passed");
+    }
 }

crates/ahm/tests/lifecycle_integration_test.rs

@@ -296,13 +296,16 @@ async fn object_is_transitioned(ecstore: &Arc<ECStore>, bucket: &str, object: &s
     }
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_lifecycle_expiry_basic() {
+mod serial_tests {
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_lifecycle_expiry_basic() {
         let (_disk_paths, ecstore) = setup_test_env().await;
 
         // Create test bucket and object
-    let bucket_name = "test-lifecycle-bucket";
+        let bucket_name = "test-lifecycle-expiry-basic-bucket";
         let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
         let test_data = b"Hello, this is test data for lifecycle expiry!";
 
@@ -387,15 +390,15 @@ async fn test_lifecycle_expiry_basic() {
         println!("✅ Scanner stopped");
 
         println!("Lifecycle expiry basic test completed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_lifecycle_expiry_deletemarker() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_lifecycle_expiry_deletemarker() {
         let (_disk_paths, ecstore) = setup_test_env().await;
 
         // Create test bucket and object
-    let bucket_name = "test-lifecycle-bucket";
+        let bucket_name = "test-lifecycle-expiry-deletemarker-bucket";
         let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
         let test_data = b"Hello, this is test data for lifecycle expiry!";
 
@@ -481,17 +484,17 @@ async fn test_lifecycle_expiry_deletemarker() {
         println!("✅ Scanner stopped");
 
         println!("Lifecycle expiry basic test completed");
-}
+    }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-#[serial]
-async fn test_lifecycle_transition_basic() {
+    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+    #[serial]
+    async fn test_lifecycle_transition_basic() {
         let (_disk_paths, ecstore) = setup_test_env().await;
 
         //create_test_tier().await;
 
         // Create test bucket and object
-    let bucket_name = "test-lifecycle-bucket";
+        let bucket_name = "test-lifecycle-transition-basic-bucket";
         let object_name = "test/object.txt"; // Match the lifecycle rule prefix "test/"
         let test_data = b"Hello, this is test data for lifecycle expiry!";
 
@@ -578,4 +581,5 @@ async fn test_lifecycle_transition_basic() {
         println!("✅ Scanner stopped");
 
         println!("Lifecycle transition basic test completed");
+    }
 }

crates/ecstore/src/data_usage.rs

@@ -14,10 +14,10 @@
 
 use std::{collections::HashMap, sync::Arc};
 
-use crate::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore};
+use crate::{bucket::metadata_sys::get_replication_config, config::com::read_config, store::ECStore, store_api::StorageAPI};
 use rustfs_common::data_usage::{BucketTargetUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, SizeSummary};
 use rustfs_utils::path::SLASH_SEPARATOR;
-use tracing::{error, warn};
+use tracing::{error, info, warn};
 
 use crate::error::Error;
 
@@ -61,12 +61,13 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store:
 
 /// Load data usage info from backend storage
 pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
-    let buf: Vec<u8> = match read_config(store, &DATA_USAGE_OBJ_NAME_PATH).await {
+    let buf: Vec<u8> = match read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await {
         Ok(data) => data,
         Err(e) => {
             error!("Failed to read data usage info from backend: {}", e);
             if e == crate::error::Error::ConfigNotFound {
-                return Ok(DataUsageInfo::default());
+                warn!("Data usage config not found, building basic statistics");
+                return build_basic_data_usage_info(store).await;
             }
             return Err(Error::other(e));
         }
@@ -75,9 +76,22 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
     let mut data_usage_info: DataUsageInfo =
         serde_json::from_slice(&buf).map_err(|e| Error::other(format!("Failed to deserialize data usage info: {e}")))?;
 
-    warn!("Loaded data usage info from backend {:?}", &data_usage_info);
+    info!("Loaded data usage info from backend with {} buckets", data_usage_info.buckets_count);
 
-    // Handle backward compatibility like original code
+    // Validate data and supplement if empty
+    if data_usage_info.buckets_count == 0 || data_usage_info.buckets_usage.is_empty() {
+        warn!("Loaded data is empty, supplementing with basic statistics");
+        if let Ok(basic_info) = build_basic_data_usage_info(store.clone()).await {
+            data_usage_info.buckets_count = basic_info.buckets_count;
+            data_usage_info.buckets_usage = basic_info.buckets_usage;
+            data_usage_info.bucket_sizes = basic_info.bucket_sizes;
+            data_usage_info.objects_total_count = basic_info.objects_total_count;
+            data_usage_info.objects_total_size = basic_info.objects_total_size;
+            data_usage_info.last_update = basic_info.last_update;
+        }
+    }
+
+    // Handle backward compatibility
     if data_usage_info.buckets_usage.is_empty() {
         data_usage_info.buckets_usage = data_usage_info
             .bucket_sizes
@@ -102,6 +116,7 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
             .collect();
     }
 
+    // Handle replication info
     for (bucket, bui) in &data_usage_info.buckets_usage {
         if bui.replicated_size_v1 > 0
             || bui.replication_failed_count_v1 > 0
@@ -129,6 +144,73 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
     Ok(data_usage_info)
 }
 
+/// Build basic data usage info with real object counts
+async fn build_basic_data_usage_info(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    let mut data_usage_info = DataUsageInfo::default();
+
+    // Get bucket list
+    match store.list_bucket(&crate::store_api::BucketOptions::default()).await {
+        Ok(buckets) => {
+            data_usage_info.buckets_count = buckets.len() as u64;
+            data_usage_info.last_update = Some(std::time::SystemTime::now());
+
+            let mut total_objects = 0u64;
+            let mut total_size = 0u64;
+
+            for bucket_info in buckets {
+                if bucket_info.name.starts_with('.') {
+                    continue; // Skip system buckets
+                }
+
+                // Try to get actual object count for this bucket
+                let (object_count, bucket_size) = match store
+                    .clone()
+                    .list_objects_v2(
+                        &bucket_info.name,
+                        "",    // prefix
+                        None,  // continuation_token
+                        None,  // delimiter
+                        100,   // max_keys - small limit for performance
+                        false, // fetch_owner
+                        None,  // start_after
+                    )
+                    .await
+                {
+                    Ok(result) => {
+                        let count = result.objects.len() as u64;
+                        let size = result.objects.iter().map(|obj| obj.size as u64).sum();
+                        (count, size)
+                    }
+                    Err(_) => (0, 0),
+                };
+
+                total_objects += object_count;
+                total_size += bucket_size;
+
+                let bucket_usage = rustfs_common::data_usage::BucketUsageInfo {
+                    size: bucket_size,
+                    objects_count: object_count,
+                    versions_count: object_count, // Simplified
+                    delete_markers_count: 0,
+                    ..Default::default()
+                };
+
+                data_usage_info.buckets_usage.insert(bucket_info.name.clone(), bucket_usage);
+                data_usage_info.bucket_sizes.insert(bucket_info.name, bucket_size);
+            }
+
+            data_usage_info.objects_total_count = total_objects;
+            data_usage_info.objects_total_size = total_size;
+            data_usage_info.versions_total_count = total_objects;
+        }
+        Err(e) => {
+            warn!("Failed to list buckets for basic data usage info: {}", e);
+        }
+    }
+
+    Ok(data_usage_info)
+}
+
 /// Create a data usage cache entry from size summary
 pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry {
     let mut entry = DataUsageEntry::default();

crates/protos/src/main.rs

@@ -29,8 +29,9 @@ fn main() -> Result<(), AnyError> {
     let need_compile = match version.compare_ext(&VERSION_PROTOBUF) {
         Ok(cmp::Ordering::Greater) => true,
         Ok(_) => {
-            let version_err = Version::build_error_message(&version, &VERSION_PROTOBUF).unwrap();
+            if let Some(version_err) = Version::build_error_message(&version, &VERSION_PROTOBUF) {
                 println!("cargo:warning=Tool `protoc` {version_err}, skip compiling.");
+            }
             false
         }
         Err(version_err) => {
@@ -144,8 +145,9 @@ fn compile_flatbuffers_models<P: AsRef<Path>, S: AsRef<str>>(
     let need_compile = match version.compare_ext(&VERSION_FLATBUFFERS) {
         Ok(cmp::Ordering::Greater) => true,
         Ok(_) => {
-            let version_err = Version::build_error_message(&version, &VERSION_FLATBUFFERS).unwrap();
+            if let Some(version_err) = Version::build_error_message(&version, &VERSION_FLATBUFFERS) {
                 println!("cargo:warning=Tool `{flatc_path}` {version_err}, skip compiling.");
+            }
             false
         }
         Err(version_err) => {
@@ -253,7 +255,13 @@ impl Version {
                 } else {
                     match self.compare_major_version(expected_version) {
                         cmp::Ordering::Greater => Ok(cmp::Ordering::Greater),
-                        _ => Err(Self::build_error_message(self, expected_version).unwrap()),
+                        _ => {
+                            if let Some(error_msg) = Self::build_error_message(self, expected_version) {
+                                Err(error_msg)
+                            } else {
+                                Err("Unknown version comparison error".to_string())
+                            }
+                        }
                     }
                 }
             }

rustfs/src/admin/handlers.rs

@@ -365,6 +365,16 @@ impl Operation for DataUsageInfoHandler {
             s3_error!(InternalError, "load_data_usage_from_backend failed")
         })?;
 
+        // If no valid data exists, attempt real-time collection
+        if info.objects_total_count == 0 && info.buckets_count == 0 {
+            info!("No data usage statistics found, attempting real-time collection");
+
+            if let Err(e) = collect_realtime_data_usage(&mut info, store.clone()).await {
+                warn!("Failed to collect real-time data usage: {}", e);
+            }
+        }
+
+        // Set capacity information
         let sinfo = store.storage_info().await;
         info.total_capacity = get_total_usable_capacity(&sinfo.disks, &sinfo) as u64;
         info.total_free_capacity = get_total_usable_capacity_free(&sinfo.disks, &sinfo) as u64;
@@ -1093,6 +1103,86 @@ impl Operation for RemoveRemoteTargetHandler {
     }
 }
 
+/// Real-time data collection function
+async fn collect_realtime_data_usage(
+    info: &mut rustfs_common::data_usage::DataUsageInfo,
+    store: Arc<rustfs_ecstore::store::ECStore>,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    // Get bucket list and collect basic statistics
+    let buckets = store
+        .list_bucket(&rustfs_ecstore::store_api::BucketOptions::default())
+        .await?;
+
+    info.buckets_count = buckets.len() as u64;
+    info.last_update = Some(std::time::SystemTime::now());
+
+    let mut total_objects = 0u64;
+    let mut total_size = 0u64;
+
+    // For each bucket, try to get object count
+    for bucket_info in buckets {
+        let bucket_name = &bucket_info.name;
+
+        // Skip system buckets
+        if bucket_name.starts_with('.') {
+            continue;
+        }
+
+        // Try to count objects in this bucket
+        let (object_count, bucket_size) = count_bucket_objects(&store, bucket_name).await.unwrap_or((0, 0));
+
+        total_objects += object_count;
+        total_size += bucket_size;
+
+        let bucket_usage = rustfs_common::data_usage::BucketUsageInfo {
+            objects_count: object_count,
+            size: bucket_size,
+            versions_count: object_count, // Simplified: assume 1 version per object
+            ..Default::default()
+        };
+
+        info.buckets_usage.insert(bucket_name.clone(), bucket_usage);
+        info.bucket_sizes.insert(bucket_name.clone(), bucket_size);
+    }
+
+    info.objects_total_count = total_objects;
+    info.objects_total_size = total_size;
+    info.versions_total_count = total_objects; // Simplified
+
+    Ok(())
+}
+
+/// Helper function to count objects in a bucket
+async fn count_bucket_objects(
+    store: &Arc<rustfs_ecstore::store::ECStore>,
+    bucket_name: &str,
+) -> Result<(u64, u64), Box<dyn std::error::Error + Send + Sync>> {
+    // Use list_objects_v2 to get actual object count
+    match store
+        .clone()
+        .list_objects_v2(
+            bucket_name,
+            "",    // prefix
+            None,  // continuation_token
+            None,  // delimiter
+            1000,  // max_keys - limit for performance
+            false, // fetch_owner
+            None,  // start_after
+        )
+        .await
+    {
+        Ok(result) => {
+            let object_count = result.objects.len() as u64;
+            let total_size = result.objects.iter().map(|obj| obj.size as u64).sum();
+            Ok((object_count, total_size))
+        }
+        Err(e) => {
+            warn!("Failed to list objects in bucket {}: {}", bucket_name, e);
+            Ok((0, 0))
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;