fix(obs): clarify cluster bucket usage metrics (#5081)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>

.docker/observability/grafana/dashboards/rustfs.json

@@ -1744,7 +1744,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (bucket) (rustfs_bucket_api_objects_total{job=~\"$job\", bucket=~\"$bucket\"})",
+          "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})",
           "legendFormat": "{{bucket}}",
           "range": true,
           "refId": "A"
@@ -1844,7 +1844,7 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "expr": "sum by (bucket) (rustfs_bucket_api_usage_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
+          "expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
           "legendFormat": "{{bucket}}",
           "range": true,
           "refId": "A"
@@ -11583,7 +11583,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "definition": "label_values(rustfs_bucket_api_objects_total,bucket)",
+        "definition": "label_values(rustfs_cluster_usage_buckets_objects_count,bucket)",
         "includeAll": true,
         "label": "Bucket",
         "multi": true,
@@ -11591,7 +11591,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(rustfs_bucket_api_objects_total,bucket)",
+          "query": "label_values(rustfs_cluster_usage_buckets_objects_count,bucket)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,

crates/obs/src/global.rs

@@ -258,6 +258,14 @@ mod tests {
         assert!(GRAFANA_DASHBOARD.contains("rustfs_log_cleaner_active_file_size_bytes"));
     }
 
+    #[test]
+    fn test_grafana_dashboard_deduplicates_cluster_bucket_usage() {
+        assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count"));
+        assert!(GRAFANA_DASHBOARD.contains("max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes"));
+        assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_objects_total"));
+        assert!(!GRAFANA_DASHBOARD.contains("sum by (bucket) (rustfs_bucket_api_usage_bytes"));
+    }
+
     #[test]
     fn test_readme_mentions_deployed_dashboard_path() {
         assert!(README.contains(".docker/observability/grafana/dashboards/rustfs.json"));

crates/obs/src/metrics/schema/cluster_usage.rs

@@ -97,7 +97,7 @@ pub static USAGE_VERSIONS_DISTRIBUTION_MD: LazyLock<MetricDescriptor> = LazyLock
 pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::UsageBucketTotalBytes,
-        "Total bucket size in bytes",
+        "Cluster-wide logical bucket size in bytes",
         &[BUCKET_LABEL],
         subsystems::CLUSTER_USAGE_BUCKETS,
     )
@@ -106,7 +106,7 @@ pub static USAGE_BUCKET_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::n
 pub static USAGE_BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::UsageBucketObjectsCount,
-        "Total objects count in bucket",
+        "Cluster-wide logical object count in bucket",
         &[BUCKET_LABEL],
         subsystems::CLUSTER_USAGE_BUCKETS,
     )
@@ -156,3 +156,22 @@ pub static USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD: LazyLock<MetricDes
         subsystems::CLUSTER_USAGE_BUCKETS,
     )
 });
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn bucket_usage_descriptors_identify_cluster_wide_logical_values() {
+        assert_eq!(
+            USAGE_BUCKET_TOTAL_BYTES_MD.get_full_metric_name(),
+            "rustfs_cluster_usage_buckets_total_bytes"
+        );
+        assert_eq!(
+            USAGE_BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(),
+            "rustfs_cluster_usage_buckets_objects_count"
+        );
+        assert!(USAGE_BUCKET_TOTAL_BYTES_MD.help.contains("Cluster-wide logical"));
+        assert!(USAGE_BUCKET_OBJECTS_TOTAL_MD.help.contains("Cluster-wide logical"));
+    }
+}

crates/obs/src/metrics/schema/node_bucket.rs

@@ -19,21 +19,21 @@ use std::sync::LazyLock;
 
 const BUCKET_LABEL: &str = "bucket";
 
-/// Total bytes used by the bucket
+/// Cluster-wide logical bytes used by the bucket.
 pub static BUCKET_USAGE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::Custom("usage_bytes".to_string()),
-        "Total bytes used by the bucket",
+        "Cluster-wide logical bytes used by the bucket",
         &[BUCKET_LABEL],
         MetricSubsystem::new("/bucket/usage"),
     )
 });
 
-/// Total number of objects in the bucket
+/// Cluster-wide logical object count in the bucket.
 pub static BUCKET_OBJECTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::Custom("objects_total".to_string()),
-        "Total number of objects in the bucket",
+        "Cluster-wide logical object count in the bucket",
         &[BUCKET_LABEL],
         MetricSubsystem::new("/bucket/usage"),
     )
@@ -58,5 +58,7 @@ mod tests {
         assert_eq!(BUCKET_USAGE_BYTES_MD.subsystem.path(), "/bucket/usage");
         assert_eq!(BUCKET_OBJECTS_TOTAL_MD.subsystem.path(), "/bucket/usage");
         assert_eq!(BUCKET_QUOTA_BYTES_MD.subsystem.path(), "/bucket/usage");
+        assert!(BUCKET_USAGE_BYTES_MD.help.contains("Cluster-wide logical"));
+        assert!(BUCKET_OBJECTS_TOTAL_MD.help.contains("Cluster-wide logical"));
     }
 }