fix(admin): serve data usage endpoints from scanner snapshot instead of live listing (#4980)

crates/ecstore/src/api/mod.rs

@@ -258,12 +258,14 @@ pub mod config {
 
 pub mod data_usage {
     pub use crate::data_usage::{
-        DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend,
-        load_compression_total_from_memory, load_data_usage_from_backend, record_bucket_delete_marker_memory,
+        DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
+        init_compression_total_memory_from_backend, live_bucket_usage_computations, load_compression_total_from_memory,
+        load_data_usage_from_backend, load_data_usage_from_backend_cached, record_bucket_delete_marker_memory,
         record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
         record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
         refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
         remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
+        store_data_usage_in_backend,
     };
 }
 

crates/ecstore/src/data_usage/mod.rs

@@ -41,7 +41,10 @@ use rustfs_utils::path::SLASH_SEPARATOR;
 use std::{
     collections::{HashMap, HashSet, hash_map::Entry},
     future::Future,
-    sync::{Arc, LazyLock, OnceLock},
+    sync::{
+        Arc, LazyLock, OnceLock,
+        atomic::{AtomicU64, Ordering},
+    },
     time::{Duration, SystemTime},
 };
 use tokio::fs;
@@ -76,6 +79,28 @@ static USAGE_MEMORY_CACHE: OnceLock<UsageMemoryCache> = OnceLock::new();
 static USAGE_CACHE_UPDATING: OnceLock<CacheUpdating> = OnceLock::new();
 static LIVE_BUCKET_USAGE_CACHE: OnceLock<LiveBucketUsageCache> = OnceLock::new();
 
+/// Cached copy of the last persisted data usage snapshot, served to admin
+/// endpoints for up to `DATA_USAGE_CACHE_TTL_SECS` between backend reads.
+#[derive(Debug, Clone)]
+struct CachedDataUsageSnapshot {
+    info: DataUsageInfo,
+    loaded_at: SystemTime,
+}
+
+type DataUsageSnapshotCache = Arc<RwLock<Option<CachedDataUsageSnapshot>>>;
+
+static DATA_USAGE_SNAPSHOT_CACHE: OnceLock<DataUsageSnapshotCache> = OnceLock::new();
+
+// Always-on revert detector for rustfs/backlog#1306: one relaxed increment per
+// full-bucket version listing is negligible and lets tests prove that admin
+// request paths never trigger live listings.
+static LIVE_BUCKET_USAGE_COMPUTATIONS: AtomicU64 = AtomicU64::new(0);
+
+/// Number of live full-bucket usage computations performed by this process.
+pub fn live_bucket_usage_computations() -> u64 {
+    LIVE_BUCKET_USAGE_COMPUTATIONS.load(Ordering::Relaxed)
+}
+
 /// Deferred persist thresholds for compression totals: persist after this many
 /// operations recorded, but no more often than the min interval.
 const COMPRESSION_PERSIST_BATCH_SIZE: u64 = 100;
@@ -115,6 +140,10 @@ fn cache_updating() -> &'static CacheUpdating {
     USAGE_CACHE_UPDATING.get_or_init(|| Arc::new(RwLock::new(false)))
 }
 
+fn data_usage_snapshot_cache() -> &'static DataUsageSnapshotCache {
+    DATA_USAGE_SNAPSHOT_CACHE.get_or_init(|| Arc::new(RwLock::new(None)))
+}
+
 fn live_bucket_usage_cache() -> &'static LiveBucketUsageCache {
     LIVE_BUCKET_USAGE_CACHE.get_or_init(|| {
         moka::future::Cache::builder()
@@ -193,6 +222,12 @@ async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<E
         .await
         .map_err(Error::other)?;
 
+    // Invalidate the cached snapshot so readers observe the new save on their
+    // next request instead of waiting out the remaining TTL. The next cached
+    // read reloads through `load_data_usage_from_backend`, keeping its
+    // backward-compatibility post-processing.
+    *data_usage_snapshot_cache().write().await = None;
+
     Ok(())
 }
 
@@ -346,6 +381,41 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
     Ok(data_usage_info)
 }
 
+/// Load the persisted data usage snapshot through a small in-process cache.
+///
+/// Admin read endpoints call this on every request; the cache bounds backend
+/// reads (and the associated JSON parse and INFO log) to once per
+/// `DATA_USAGE_CACHE_TTL_SECS` per process. `save_data_usage_in_backend`
+/// invalidates the cache so a fresh scanner save is visible immediately.
+pub async fn load_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
+    let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
+
+    {
+        let cache = data_usage_snapshot_cache().read().await;
+        if let Some(cached) = cache.as_ref()
+            && SystemTime::now().duration_since(cached.loaded_at).unwrap_or_default() < ttl
+        {
+            return Ok(cached.info.clone());
+        }
+    }
+
+    // Re-check under the write lock so concurrent expirations trigger a single
+    // backend read instead of a stampede.
+    let mut cache = data_usage_snapshot_cache().write().await;
+    if let Some(cached) = cache.as_ref()
+        && SystemTime::now().duration_since(cached.loaded_at).unwrap_or_default() < ttl
+    {
+        return Ok(cached.info.clone());
+    }
+
+    let info = load_data_usage_from_backend(store).await?;
+    *cache = Some(CachedDataUsageSnapshot {
+        info: info.clone(),
+        loaded_at: SystemTime::now(),
+    });
+    Ok(info)
+}
+
 /// Aggregate usage information from local disk snapshots.
 fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
     if let Some(update) = snapshot.last_update
@@ -548,6 +618,7 @@ impl BucketUsageAccumulator {
 type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
 
 pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
+    LIVE_BUCKET_USAGE_COMPUTATIONS.fetch_add(1, Ordering::Relaxed);
     let bucket = bucket_name.to_string();
     compute_bucket_usage_with_pages(bucket_name, move |marker, version_marker| {
         let store = Arc::clone(&store);

crates/madmin/src/user.rs

@@ -406,10 +406,17 @@ pub struct AccountInfo {
 #[derive(Debug, Serialize, Deserialize, Default)]
 pub struct BucketAccessInfo {
     pub name: String,
-    pub size: u64,
-    pub objects: u64,
-    pub object_sizes_histogram: HashMap<String, u64>,
-    pub object_versions_histogram: HashMap<String, u64>,
+    // Usage stats are absent (not zero) when no scanner snapshot covers the
+    // bucket yet, so clients can distinguish "unknown" from "empty"
+    // (rustfs/backlog#1306).
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub size: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub objects: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub object_sizes_histogram: Option<HashMap<String, u64>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub object_versions_histogram: Option<HashMap<String, u64>>,
     pub details: Option<BucketDetails>,
     pub prefix_usage: HashMap<String, u64>,
     #[serde(rename = "expiration", with = "time::serde::rfc3339::option")]
@@ -727,6 +734,43 @@ mod tests {
     use time::OffsetDateTime;
     use time::macros::datetime;
 
+    /// Wire pin (rustfs/backlog#1306): usage stats without a scanner snapshot
+    /// must be omitted from the JSON, not serialized as zeros.
+    #[test]
+    fn bucket_access_info_omits_absent_usage_stats() {
+        let info = BucketAccessInfo {
+            name: "no-snapshot".to_string(),
+            ..Default::default()
+        };
+
+        let value = serde_json::to_value(&info).unwrap();
+        let obj = value.as_object().unwrap();
+        assert!(!obj.contains_key("size"));
+        assert!(!obj.contains_key("objects"));
+        assert!(!obj.contains_key("object_sizes_histogram"));
+        assert!(!obj.contains_key("object_versions_histogram"));
+    }
+
+    /// Wire pin (rustfs/backlog#1306): populated usage stats keep their
+    /// existing snake_case keys and numeric values.
+    #[test]
+    fn bucket_access_info_serializes_present_usage_stats() {
+        let info = BucketAccessInfo {
+            name: "snapshot".to_string(),
+            size: Some(1024),
+            objects: Some(7),
+            object_sizes_histogram: Some(HashMap::from([("1MiB-10MiB".to_string(), 7)])),
+            object_versions_histogram: Some(HashMap::from([("SINGLE_VERSION".to_string(), 7)])),
+            ..Default::default()
+        };
+
+        let value = serde_json::to_value(&info).unwrap();
+        assert_eq!(value["size"], 1024);
+        assert_eq!(value["objects"], 7);
+        assert_eq!(value["object_sizes_histogram"]["1MiB-10MiB"], 7);
+        assert_eq!(value["object_versions_histogram"]["SINGLE_VERSION"], 7);
+    }
+
     #[test]
     fn test_account_status_try_from_invalid() {
         let result = AccountStatus::try_from("invalid");
@@ -1029,10 +1073,10 @@ mod tests {
 
         let bucket_info = BucketAccessInfo {
             name: "test-bucket".to_string(),
-            size: 6000000,
-            objects: 150,
-            object_sizes_histogram: sizes_histogram,
-            object_versions_histogram: versions_histogram,
+            size: Some(6000000),
+            objects: Some(150),
+            object_sizes_histogram: Some(sizes_histogram),
+            object_versions_histogram: Some(versions_histogram),
             details: Some(BucketDetails {
                 versioning: true,
                 versioning_suspended: false,
@@ -1049,10 +1093,10 @@ mod tests {
         };
 
         assert_eq!(bucket_info.name, "test-bucket");
-        assert_eq!(bucket_info.size, 6000000);
-        assert_eq!(bucket_info.objects, 150);
-        assert_eq!(bucket_info.object_sizes_histogram.len(), 2);
-        assert_eq!(bucket_info.object_versions_histogram.len(), 2);
+        assert_eq!(bucket_info.size, Some(6000000));
+        assert_eq!(bucket_info.objects, Some(150));
+        assert_eq!(bucket_info.object_sizes_histogram.as_ref().map(HashMap::len), Some(2));
+        assert_eq!(bucket_info.object_versions_histogram.as_ref().map(HashMap::len), Some(2));
         assert!(bucket_info.details.is_some());
         assert_eq!(bucket_info.prefix_usage.len(), 2);
         assert!(bucket_info.created.is_some());