fix(obs): align cluster capacity semantics (#4457)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/metrics/collectors/cluster.rs

@@ -34,9 +34,9 @@ pub struct ClusterStats {
     pub raw_capacity_bytes: u64,
     /// Usable capacity after erasure coding overhead in bytes
     pub usable_capacity_bytes: u64,
-    /// Currently used storage in bytes
+    /// Currently used usable storage in bytes
     pub used_bytes: u64,
-    /// Available free storage in bytes
+    /// Available usable free storage in bytes
     pub free_bytes: u64,
     /// Number of drives backed by stale capacity snapshots
     pub stale_capacity_drives: u64,

crates/obs/src/metrics/schema/cluster.rs

@@ -37,21 +37,21 @@ pub static CLUSTER_CAPACITY_USABLE_TOTAL_BYTES_MD: LazyLock<MetricDescriptor> =
     )
 });
 
-/// Total used storage capacity in bytes
+/// Total used usable storage capacity in bytes
 pub static CLUSTER_CAPACITY_USED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::Custom("capacity_used_bytes".to_string()),
-        "Total used storage capacity in bytes",
+        "Total used usable storage capacity in bytes (matches usable total and free capacity)",
         &[],
         subsystems::CLUSTER_BASE_PATH,
     )
 });
 
-/// Total free storage capacity in bytes
+/// Total usable free storage capacity in bytes
 pub static CLUSTER_CAPACITY_FREE_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
     new_gauge_md(
         MetricName::Custom("capacity_free_bytes".to_string()),
-        "Total free storage capacity in bytes",
+        "Total usable free storage capacity in bytes (matches usable total and used capacity)",
         &[],
         subsystems::CLUSTER_BASE_PATH,
     )

crates/obs/src/metrics/stats_collector.rs

@@ -119,6 +119,10 @@ fn obs_total_usable_capacity_free_bytes(storage_info: &ObsStorageInfo) -> u64 {
     usize_to_u64_saturating(obs_get_total_usable_capacity_free(&storage_info.disks, storage_info))
 }
 
+fn usable_capacity_used_bytes(usable_capacity: u64, free_bytes: u64) -> u64 {
+    usable_capacity.saturating_sub(free_bytes)
+}
+
 async fn obs_bucket_quota_limit_bytes(bucket: &str) -> u64 {
     obs_get_quota_config(bucket)
         .await
@@ -326,9 +330,9 @@ pub async fn collect_cluster_and_health_stats() -> (ClusterStats, ClusterHealthS
 
     let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
     let raw_capacity: u64 = storage_info.disks.iter().map(|d| d.total_space).sum();
-    let used: u64 = storage_info.disks.iter().map(|d| d.used_space).sum();
     let usable_capacity = obs_total_usable_capacity_bytes(&storage_info);
     let free = obs_total_usable_capacity_free_bytes(&storage_info);
+    let used = usable_capacity_used_bytes(usable_capacity, free);
     let stale_capacity_drives = storage_info
         .disks
         .iter()
@@ -1270,4 +1274,14 @@ mod tests {
         assert_eq!(scanner_work_rate_per_second(90, f64::INFINITY), 0.0);
         assert_eq!(scanner_work_rate_per_second(90, f64::NAN), 0.0);
     }
+
+    #[test]
+    fn usable_capacity_used_bytes_matches_usable_total_minus_free() {
+        assert_eq!(usable_capacity_used_bytes(240, 90), 150);
+    }
+
+    #[test]
+    fn usable_capacity_used_bytes_saturates_when_free_exceeds_total() {
+        assert_eq!(usable_capacity_used_bytes(90, 240), 0);
+    }
 }