fix(scanner): publish bounded observational usage (#5742)

* fix(scanner): publish bounded observational usage

* test(ci): serialize embedded integration ports

* test(cache): isolate generation-change timeout

* fix(scanner): address observational usage review

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: Anthony Martin <949506+anthonymartin@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/handlers/account_info.rs

@@ -18,7 +18,7 @@ use crate::admin::runtime_sources::{current_action_credentials, object_store_fro
 use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
 use crate::admin::storage_api::contract::admin::StorageAdminApi;
 use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
-use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
+use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
 use crate::admin::storage_api::metadata_sys;
 use crate::auth::get_condition_values;
 use crate::server::{ADMIN_PREFIX, RemoteAddr};
@@ -264,8 +264,9 @@ impl Operation for AccountInfoHandler {
         // process-local absolute counters to cluster-wide bucket totals.
         // This path never triggers a live full-version listing
         // (rustfs/backlog#1306); freshness is owned by the scanner.
-        let mut data_usage_info = map_data_usage_result(load_data_usage_from_backend_cached(store.clone()).await)?;
+        let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
         apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
+        account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
 
         for bucket in buckets.iter() {
             let (rd, wr) = is_allow(bucket.name.clone()).await;

rustfs/src/admin/storage_api.rs

@@ -784,10 +784,10 @@ pub(crate) mod data_usage {
         crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await
     }
 
-    pub(crate) async fn load_data_usage_from_backend_cached(
+    pub(crate) async fn load_admin_data_usage_from_backend_cached(
         store: Arc<crate::storage::storage_api::ECStore>,
     ) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
-        crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend_cached(store).await
+        crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
     }
 }
 

rustfs/src/app/admin_usecase.rs

@@ -20,7 +20,9 @@ use super::storage_api::admin_usecase::capacity::{
 };
 use super::storage_api::admin_usecase::contract::StorageAdminApi;
 use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
-use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend_cached};
+use super::storage_api::admin_usecase::data_usage::{
+    apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
+};
 use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
 use crate::app::runtime_sources::{
     AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
@@ -265,7 +267,7 @@ impl DefaultAdminUsecase {
     /// This path never triggers a live full-version listing
     /// (rustfs/backlog#1306); freshness is owned by the scanner.
     pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
-        let mut info = Self::map_data_usage_load_result(load_data_usage_from_backend_cached(store.clone()).await)?;
+        let mut info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
         apply_bucket_usage_memory_overlay(&mut info).await;
         let buckets = store
             .list_bucket(&BucketOptions {

rustfs/src/app/object_usecase.rs

@@ -3417,6 +3417,8 @@ static IO_QUEUE_CONGESTION_WARN_THROTTLE: IoQueueCongestionWarnThrottle = IoQueu
 #[derive(Clone, Default)]
 pub struct DefaultObjectUsecase {
     context: Option<Arc<AppContext>>,
+    #[cfg(test)]
+    get_object_timeout_policy: Option<GetObjectTimeoutPolicy>,
 }
 
 impl DefaultObjectUsecase {
@@ -3426,12 +3428,17 @@ impl DefaultObjectUsecase {
 
     #[cfg(test)]
     pub fn without_context() -> Self {
-        Self { context: None }
+        Self {
+            context: None,
+            get_object_timeout_policy: None,
+        }
     }
 
     pub fn from_global() -> Self {
         Self {
             context: current_app_context(),
+            #[cfg(test)]
+            get_object_timeout_policy: None,
         }
     }
 
@@ -3440,7 +3447,22 @@ impl DefaultObjectUsecase {
     /// so the use-case resolves that server's store; `None` falls back to the
     /// ambient default.
     pub fn with_context(context: Option<std::sync::Arc<crate::runtime_sources::AppContext>>) -> Self {
-        Self { context }
+        Self {
+            context,
+            #[cfg(test)]
+            get_object_timeout_policy: None,
+        }
+    }
+
+    #[cfg(test)]
+    fn with_context_and_get_object_timeout_policy(
+        context: Option<std::sync::Arc<crate::runtime_sources::AppContext>>,
+        get_object_timeout_policy: GetObjectTimeoutPolicy,
+    ) -> Self {
+        Self {
+            context,
+            get_object_timeout_policy: Some(get_object_timeout_policy),
+        }
     }
 
     fn bucket_metadata_sys(&self) -> Option<Arc<RwLock<metadata_sys::BucketMetadataSys>>> {
@@ -3581,7 +3603,13 @@ impl DefaultObjectUsecase {
         blob
     }
 
-    fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
+    fn init_get_object_bootstrap(&self, bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
+        #[cfg(test)]
+        let timeout_config = self
+            .get_object_timeout_policy
+            .clone()
+            .unwrap_or_else(GetObjectTimeoutPolicy::cached_from_env);
+        #[cfg(not(test))]
         let timeout_config = GetObjectTimeoutPolicy::cached_from_env();
         let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string());
         let request_start = std::time::Instant::now();
@@ -5763,7 +5791,7 @@ impl DefaultObjectUsecase {
                 context.start_time.elapsed().as_secs_f64(),
             );
         }
-        let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
+        let bootstrap = self.init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
         let timeout_config = bootstrap.timeout_config;
         let wrapper = bootstrap.wrapper;
         let request_start = bootstrap.request_start;
@@ -10728,7 +10756,17 @@ mod tests {
             .key(object.to_string())
             .build()
             .expect("real cold-fill GET input must build");
-        let usecase = DefaultObjectUsecase::with_context(Some(context));
+        // The request is intentionally held behind the first producer while a
+        // 1.3 MiB replacement write changes its generation. Disable dynamic
+        // sizing for this test so runner I/O load cannot consume the five-second
+        // production minimum before the behavior under test is released.
+        let usecase = DefaultObjectUsecase::with_context_and_get_object_timeout_policy(
+            Some(context),
+            GetObjectTimeoutPolicy {
+                enable_dynamic_timeout: false,
+                ..GetObjectTimeoutPolicy::default()
+            },
+        );
         let request = tokio::spawn(async move { usecase.execute_get_object(build_request(input, Method::GET)).await });
         tokio::time::timeout(Duration::from_secs(2), async {
             while coordinator.global_waiter_count_for_test() != 1 {

rustfs/src/app/storage_api.rs

@@ -51,6 +51,13 @@ pub(crate) mod data_usage {
         crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
     }
 
+    pub(crate) async fn load_admin_data_usage_from_backend_cached(
+        store: Arc<crate::storage::storage_api::ECStore>,
+    ) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
+        crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
+    }
+
+    #[cfg(test)]
     pub(crate) async fn load_data_usage_from_backend_cached(
         store: Arc<crate::storage::storage_api::ECStore>,
     ) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {

rustfs/src/storage/storage_api.rs

@@ -415,15 +415,15 @@ pub(crate) mod ecstore_config {
 
 pub(crate) mod ecstore_data_usage {
     pub(crate) use rustfs_ecstore::api::data_usage::{
-        apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_data_usage_from_backend,
-        load_data_usage_from_backend_cached, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
+        apply_bucket_usage_memory_overlay, init_compression_total_memory_from_backend, load_admin_data_usage_from_backend_cached,
+        load_data_usage_from_backend, record_bucket_delete_marker_memory, record_bucket_object_delete_memory,
         record_bucket_object_version_write_memory, record_bucket_object_write_memory,
         record_bucket_object_write_unknown_previous_memory, store_compression_total_in_backend,
     };
     // Test-only observables for the rustfs/backlog#1306 revert detector.
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::data_usage::{
-        compute_bucket_usage, live_bucket_usage_computations, store_data_usage_in_backend,
+        compute_bucket_usage, live_bucket_usage_computations, load_data_usage_from_backend_cached, store_data_usage_in_backend,
     };
 }