fix(scanner): publish bounded observational usage (#5742)

* fix(scanner): publish bounded observational usage

* test(ci): serialize embedded integration ports

* test(cache): isolate generation-change timeout

* fix(scanner): address observational usage review

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: Anthony Martin <949506+anthonymartin@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

crates/scanner/src/data_usage_define.rs

@@ -27,8 +27,8 @@ use rustfs_common::heal_channel::HealScanMode;
 #[cfg(test)]
 use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
 pub use rustfs_data_usage::{
-    AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DataUsageEntry, DataUsageHash,
-    DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
+    AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
+    DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, TierStats, hash_path,
 };
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
 use tokio::time::{Duration, Instant, sleep, timeout};
@@ -173,6 +173,9 @@ pub static DATA_USAGE_BUCKET: LazyLock<String> =
 pub static DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJECT_NAME}"));
 
+pub static DATA_USAGE_OBSERVED_OBJ_NAME_PATH: LazyLock<String> =
+    LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBSERVED_OBJECT_NAME}"));
+
 pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
     LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{LEGACY_DATA_USAGE_OBJECT_NAME}"));
 

crates/scanner/src/lib.rs

@@ -35,10 +35,11 @@ use storage_api::owner::{
     EcstoreTierConfig, EcstoreVersioningApi, HTTPPreconditions, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete,
     ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
     ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config,
-    ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_data_usage_snapshot_cache,
-    ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
-    ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
-    ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
+    ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_invalidate_admin_data_usage_snapshot_cache,
+    ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
+    ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
+    ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
+    ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
     scanner_replication_config_for_lifecycle_eval,
 };
 #[cfg(test)]
@@ -475,6 +476,10 @@ pub(crate) async fn invalidate_data_usage_snapshot_cache() {
     ecstore_invalidate_data_usage_snapshot_cache().await;
 }
 
+pub(crate) async fn invalidate_admin_data_usage_snapshot_cache() {
+    ecstore_invalidate_admin_data_usage_snapshot_cache().await;
+}
+
 pub trait ScannerObjectIO:
     ObjectIO<
         Error = EcstoreError,
@@ -501,6 +506,28 @@ impl<T> ScannerObjectIO for T where
 {
 }
 
+#[async_trait::async_trait]
+pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static {
+    async fn delete_config_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: ScannerObjectOptions,
+    ) -> EcstoreResult<ScannerObjectInfo>;
+}
+
+#[async_trait::async_trait]
+impl ScannerConfigObjectDelete for ECStore {
+    async fn delete_config_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: ScannerObjectOptions,
+    ) -> EcstoreResult<ScannerObjectInfo> {
+        ObjectOperations::delete_object(self, bucket, object, opts).await
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/scanner/src/scanner_io.rs

@@ -776,6 +776,34 @@ fn classify_nsscanner_cycle(
     }
 }
 
+fn should_publish_usage_snapshot(status: ScannerCycleStatus) -> bool {
+    matches!(status, ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded)
+}
+
+fn prepare_usage_snapshot_for_publication(
+    status: ScannerCycleStatus,
+    mut data_usage_info: DataUsageInfo,
+) -> Option<DataUsageInfo> {
+    if !should_publish_usage_snapshot(status) {
+        return None;
+    }
+
+    data_usage_info.usage_snapshot_converged = Some(status == ScannerCycleStatus::Complete);
+    Some(data_usage_info)
+}
+
+async fn publish_usage_snapshot(
+    updates: &mpsc::Sender<DataUsageInfo>,
+    status: ScannerCycleStatus,
+    data_usage_info: DataUsageInfo,
+) -> Result<bool> {
+    let Some(data_usage_info) = prepare_usage_snapshot_for_publication(status, data_usage_info) else {
+        return Ok(false);
+    };
+    send_data_usage_update(updates, data_usage_info).await?;
+    Ok(true)
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum ScannerCycleActivityStatus {
     Unchanged,
@@ -2347,22 +2375,29 @@ impl ScannerIOCycle for ECStore {
                 dirty_usage_status,
                 activity_status,
             );
-            if status != ScannerCycleStatus::Complete {
+            if !publish_usage_snapshot(
+                &updates,
+                status,
+                DataUsageInfo {
+                    last_update: Some(SystemTime::now()),
+                    scanner_cycle: Some(want_cycle),
+                    usage_snapshot_complete: true,
+                    ..Default::default()
+                },
+            )
+            .await?
+            {
                 return Ok(ScannerCycleResult::new(status, None));
             }
-            let empty_usage = DataUsageInfo {
-                last_update: Some(SystemTime::now()),
-                scanner_cycle: Some(want_cycle),
-                usage_snapshot_complete: true,
-                ..Default::default()
+            let dirty_usage_clear =
+                (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone());
+            let remote_dirty_usage_acknowledgements = if status == ScannerCycleStatus::Complete {
+                crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before)
+            } else {
+                Vec::new()
             };
-            send_data_usage_update(&updates, empty_usage).await?;
-            let dirty_usage_clear = Some(dirty_usage_snapshot.buckets.as_ref().clone());
-            return Ok(
-                ScannerCycleResult::new(status, dirty_usage_clear).with_remote_dirty_usage_acknowledgements(
-                    crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before),
-                ),
-            );
+            return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
+                .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements));
         }
 
         let total_results = expected_sources.len();
@@ -2576,10 +2611,8 @@ impl ScannerIOCycle for ECStore {
             dirty_usage_status,
             activity_status,
         );
-        if cycle_status == ScannerCycleStatus::Complete
-            && let Some((data_usage_info, _)) = completed_usage
-        {
-            send_data_usage_update(&updates, data_usage_info).await?;
+        if let Some((data_usage_info, _)) = completed_usage {
+            publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?;
         }
         let dirty_usage_clear = should_clear_dirty_usage_snapshot(
             result.is_ok(),
@@ -4597,6 +4630,44 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn structurally_complete_superseded_cycles_publish_without_claiming_convergence() {
+        let (updates, mut receiver) = mpsc::channel(2);
+
+        assert!(
+            publish_usage_snapshot(&updates, ScannerCycleStatus::Complete, DataUsageInfo::default())
+                .await
+                .expect("complete snapshot publication should succeed")
+        );
+        assert!(
+            publish_usage_snapshot(&updates, ScannerCycleStatus::Superseded, DataUsageInfo::default())
+                .await
+                .expect("superseded snapshot publication should succeed")
+        );
+        assert!(
+            !publish_usage_snapshot(&updates, ScannerCycleStatus::Incomplete, DataUsageInfo::default())
+                .await
+                .expect("incomplete snapshot suppression should succeed")
+        );
+
+        assert_eq!(
+            receiver
+                .recv()
+                .await
+                .expect("complete update should be sent")
+                .usage_snapshot_converged,
+            Some(true)
+        );
+        assert_eq!(
+            receiver
+                .recv()
+                .await
+                .expect("superseded update should be sent")
+                .usage_snapshot_converged,
+            Some(false)
+        );
+    }
+
     #[test]
     fn scanner_cycle_fails_closed_for_namespace_disappearance() {
         for activity_status in [

crates/scanner/src/storage_api.rs

@@ -58,6 +58,7 @@ pub(crate) use rustfs_ecstore::api::config::storageclass::{
     RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
 };
 pub(crate) use rustfs_ecstore::api::data_usage::{
+    invalidate_admin_data_usage_snapshot_cache as ecstore_invalidate_admin_data_usage_snapshot_cache,
     invalidate_data_usage_snapshot_cache as ecstore_invalidate_data_usage_snapshot_cache,
     replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info,
 };
@@ -111,11 +112,11 @@ pub(crate) mod owner {
         ScannerReplicationHealObject, ScannerReplicationHealResult, ScannerReplicationQueueAdmission, ecstore_apply_expiry_rule,
         ecstore_apply_transition_rule, ecstore_expiry_state_handle, ecstore_get_global_tier_config_mgr,
         ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config,
-        ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure, ecstore_is_erasure_sd,
-        ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_object_opts_from_object_info,
-        ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path, ecstore_read_config,
-        ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
-        scanner_replication_config_for_lifecycle_eval,
+        ecstore_invalidate_admin_data_usage_snapshot_cache, ecstore_invalidate_data_usage_snapshot_cache, ecstore_is_erasure,
+        ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw,
+        ecstore_object_opts_from_object_info, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
+        ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle,
+        ecstore_save_config, scanner_replication_config_for_lifecycle_eval,
     };
 
     #[cfg(test)]