fix(scanner): publish bounded observational usage (#5742)

* fix(scanner): publish bounded observational usage

* test(ci): serialize embedded integration ports

* test(cache): isolate generation-change timeout

* fix(scanner): address observational usage review

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: Anthony Martin <949506+anthonymartin@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/metrics/collectors/cluster_usage.rs

@@ -38,6 +38,9 @@ pub struct ClusterUsageStats {
     pub delete_markers_count: u64,
     /// Total number of buckets in the usage snapshot
     pub buckets_count: u64,
+    /// Whether the selected admin usage snapshot completed without concurrent
+    /// namespace activity.
+    pub snapshot_converged: bool,
     /// Object size distribution by range
     pub object_size_distribution: Vec<(String, u64)>,
     /// Version count distribution by range
@@ -69,7 +72,7 @@ pub struct BucketUsageStats {
 ///
 /// Returns a vector of Prometheus metrics for cluster usage.
 pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<PrometheusMetric> {
-    let mut metrics = Vec::with_capacity(6 + stats.object_size_distribution.len() + stats.versions_distribution.len());
+    let mut metrics = Vec::with_capacity(7 + stats.object_size_distribution.len() + stats.versions_distribution.len());
 
     metrics.push(PrometheusMetric::from_descriptor(
         &USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
@@ -83,6 +86,10 @@ pub fn collect_cluster_usage_metrics(stats: &ClusterUsageStats) -> Vec<Prometheu
         stats.delete_markers_count as f64,
     ));
     metrics.push(PrometheusMetric::from_descriptor(&USAGE_BUCKETS_COUNT_MD, stats.buckets_count as f64));
+    metrics.push(PrometheusMetric::from_descriptor(
+        &USAGE_SNAPSHOT_CONVERGED_MD,
+        if stats.snapshot_converged { 1.0 } else { 0.0 },
+    ));
 
     // Object size distribution
     for (range, count) in &stats.object_size_distribution {
@@ -176,6 +183,7 @@ mod tests {
             versions_count: 15000,
             delete_markers_count: 500,
             buckets_count: 8,
+            snapshot_converged: false,
             object_size_distribution: vec![
                 ("0-1KB".to_string(), 5000),
                 ("1KB-1MB".to_string(), 3000),
@@ -188,8 +196,8 @@ mod tests {
         let metrics = collect_cluster_usage_metrics(&stats);
         report_metrics(&metrics);
 
-        // 6 base metrics + 4 size distribution + 3 version distribution = 13
-        assert_eq!(metrics.len(), 13);
+        // 7 base metrics + 4 size distribution + 3 version distribution = 14
+        assert_eq!(metrics.len(), 14);
 
         let total_bytes_name = USAGE_TOTAL_BYTES_MD.get_full_metric_name();
         let total_bytes = metrics.iter().find(|m| m.name == total_bytes_name);
@@ -198,6 +206,9 @@ mod tests {
         let stale_name = USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name();
         let stale = metrics.iter().find(|m| m.name == stale_name && m.value == 45.0);
         assert!(stale.is_some());
+
+        let converged_name = USAGE_SNAPSHOT_CONVERGED_MD.get_full_metric_name();
+        assert!(metrics.iter().any(|m| m.name == converged_name && m.value == 0.0));
     }
 
     #[test]

crates/obs/src/metrics/schema/cluster_usage.rs

@@ -76,6 +76,15 @@ pub static USAGE_BUCKETS_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
     )
 });
 
+pub static USAGE_SNAPSHOT_CONVERGED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
+    new_gauge_md(
+        MetricName::UsageSnapshotConverged,
+        "Whether the selected admin usage snapshot is converged (1) or observational (0)",
+        &[],
+        subsystems::CLUSTER_USAGE_OBJECTS,
+    )
+});
+
 pub static USAGE_OBJECTS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::UsageSizeDistribution,

crates/obs/src/metrics/schema/entry/metric_name.rs

@@ -233,6 +233,7 @@ pub enum MetricName {
     UsageVersionsCount,
     UsageDeleteMarkersCount,
     UsageBucketsCount,
+    UsageSnapshotConverged,
     UsageSizeDistribution,
     UsageVersionCountDistribution,
 
@@ -642,6 +643,7 @@ impl MetricName {
             Self::UsageVersionsCount => "versions_count".to_string(),
             Self::UsageDeleteMarkersCount => "delete_markers_count".to_string(),
             Self::UsageBucketsCount => "buckets_count".to_string(),
+            Self::UsageSnapshotConverged => "snapshot_converged".to_string(),
             Self::UsageSizeDistribution => "size_distribution".to_string(),
             Self::UsageVersionCountDistribution => "version_count_distribution".to_string(),
 

crates/obs/src/metrics/stats_collector.rs

@@ -65,6 +65,7 @@ type ObsBackendInfo = <ObsStore as StorageAdminApi>::BackendInfo;
 struct ObsDataUsageInfo {
     last_update: Option<SystemTime>,
     usage_snapshot_complete: bool,
+    usage_snapshot_converged: bool,
     buckets_count: u64,
     objects_total_count: u64,
     versions_total_count: u64,
@@ -102,6 +103,7 @@ async fn load_obs_data_usage_from_backend(store: Arc<ObsStore>) -> ObsEcstoreRes
     Ok(ObsDataUsageInfo {
         last_update: data_usage.last_update,
         usage_snapshot_complete,
+        usage_snapshot_converged: data_usage.usage_snapshot_converged == Some(true),
         buckets_count: data_usage.buckets_count,
         objects_total_count: data_usage.objects_total_count,
         versions_total_count: data_usage.versions_total_count,
@@ -1204,6 +1206,7 @@ async fn collect_cluster_usage_metric_stats_from_data_usage(
             versions_count: data_usage.versions_total_count,
             delete_markers_count: data_usage.delete_markers_total_count,
             buckets_count: data_usage.buckets_count,
+            snapshot_converged: data_usage.usage_snapshot_converged,
             object_size_distribution: data_usage
                 .buckets_usage
                 .values()
@@ -1579,6 +1582,7 @@ mod tests {
         let (cluster, buckets) = collect_cluster_usage_metric_stats_from_data_usage(
             ObsDataUsageInfo {
                 usage_snapshot_complete: true,
+                usage_snapshot_converged: true,
                 ..Default::default()
             },
             &HashSet::new(),
@@ -1589,9 +1593,26 @@ mod tests {
         assert_eq!(cluster.buckets_count, 0);
         assert_eq!(cluster.objects_count, 0);
         assert_eq!(cluster.total_bytes, 0);
+        assert!(cluster.snapshot_converged);
         assert!(buckets.is_empty());
     }
 
+    #[tokio::test]
+    async fn cluster_usage_metrics_publish_unknown_convergence_as_unconverged() {
+        let (cluster, _) = collect_cluster_usage_metric_stats_from_data_usage(
+            ObsDataUsageInfo {
+                usage_snapshot_complete: true,
+                usage_snapshot_converged: false,
+                ..Default::default()
+            },
+            &HashSet::new(),
+        )
+        .await
+        .expect("complete usage with unknown convergence should remain publishable");
+
+        assert!(!cluster.snapshot_converged);
+    }
+
     #[tokio::test]
     async fn cluster_usage_metrics_skip_snapshot_for_a_different_bucket_namespace() {
         let data_usage = ObsDataUsageInfo {

crates/obs/src/metrics/storage_api.rs

@@ -27,8 +27,8 @@ pub(crate) use rustfs_ecstore::api::capacity::{
     get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
 };
 pub(crate) use rustfs_ecstore::api::compression::is_disk_compression_enabled as obs_is_disk_compression_enabled;
+pub(crate) use rustfs_ecstore::api::data_usage::load_admin_data_usage_from_backend_cached as obs_load_data_usage_from_backend;
 pub(crate) use rustfs_ecstore::api::data_usage::load_compression_total_from_memory as obs_load_compression_total_from_memory;
-pub(crate) use rustfs_ecstore::api::data_usage::load_data_usage_from_backend as obs_load_data_usage_from_backend;
 pub(crate) use rustfs_ecstore::api::error::Result as ObsEcstoreResult;
 pub(crate) use rustfs_ecstore::api::runtime::{
     bucket_monitor as obs_get_global_bucket_monitor, expiry_state_handle as obs_expiry_state_handle,