fix(scanner): fence system metadata publication (#6444)

* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/store/bucket.rs

@@ -327,7 +327,7 @@ impl ECStore {
     async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> {
         run_bucket_usage_cleanup(guard, bucket, async {
             crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?;
-            crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await
+            crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await
         })
         .await
     }

crates/ecstore/src/store/init.rs

@@ -495,6 +495,11 @@ impl ECStore {
             );
         }
 
+        // Initialize the storage-owned scanner publication state only after
+        // both movement metadata sources have been loaded. SetDisks cache
+        // writers remain fail-closed until this snapshot is available.
+        let _ = self.scanner_data_usage_publication_blocked().await;
+
         let pools = installed_pool_meta.return_resumable_pools();
         let mut pool_indices = Vec::with_capacity(pools.len());
 

crates/ecstore/src/store/mod.rs

@@ -44,7 +44,7 @@ use crate::error::{
 use crate::runtime::global::DISK_RESERVE_FRACTION;
 use crate::runtime::instance::InstanceContext;
 use crate::runtime::sources as runtime_sources;
-use crate::services::rebalance::RebalanceMeta;
+use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
 use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -349,16 +349,81 @@ impl ECStore {
     /// remain suspended until an operator clears or retries them, so they are
     /// a publication barrier even after the worker has stopped.
     pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
-        if self.scanner_data_movement_active().await {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let _operation_guard = operation_gate.read_owned().await;
+        self.scanner_data_usage_publication_snapshot_blocked().await
+    }
+
+    async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
+        if self.ctx.data_movement_operation_epoch_exhausted() {
+            self.ctx.set_scanner_publication_state(true);
             return true;
         }
-
+        let decommission_cancelers = self.decommission_cancelers.read().await;
+        let decommission_active = decommission_cancelers
+            .iter()
+            .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active));
         let pool_meta = self.pool_meta.read().await;
-        pool_meta.pools.iter().any(|pool| {
+        let decommission_active = decommission_active
+            || pool_meta.pools.iter().any(|pool| {
+                pool.decommission
+                    .as_ref()
+                    .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
+            });
+        let decommission_terminal = pool_meta.pools.iter().any(|pool| {
             pool.decommission
                 .as_ref()
                 .is_some_and(|info| !info.queued && (info.failed || info.canceled))
-        })
+        });
+        drop(pool_meta);
+
+        let rebalance_active = self
+            .rebalance_meta
+            .read()
+            .await
+            .as_ref()
+            .is_some_and(is_rebalance_conflicting_with_decommission);
+
+        let blocked = decommission_active || decommission_terminal || rebalance_active;
+        self.ctx.set_scanner_publication_state(blocked);
+        blocked
+    }
+
+    /// Admit one short data-usage publication commit under the same
+    /// per-instance gate used by decommission side effects and transitions.
+    /// The epoch is sampled while the read guard is held, so a transition
+    /// cannot cross this admission without waiting for the commit to finish.
+    pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let operation_guard = operation_gate.read_owned().await;
+        let epoch = self.ctx.data_movement_operation_epoch();
+        (operation_guard, epoch)
+    }
+
+    /// Acquire the movement gate and inspect the movement owner once. The
+    /// state inspection is performed after acquiring the read guard so a
+    /// transition cannot update its durable state between the check and the
+    /// publication commit.
+    pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let operation_guard = operation_gate.read_owned().await;
+        if self.ctx.data_movement_operation_epoch_exhausted() {
+            return None;
+        }
+        if self.scanner_data_usage_publication_snapshot_blocked().await {
+            return None;
+        }
+
+        Some((operation_guard, self.ctx.data_movement_operation_epoch()))
+    }
+
+    /// Capture the current publication epoch without holding the movement
+    /// gate across backend I/O. Callers must re-admit the same epoch before a
+    /// mutation commits.
+    pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option<u64> {
+        let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?;
+        drop(operation_guard);
+        Some(epoch)
     }
 }
 
@@ -995,6 +1060,85 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let operation_gate = store.ctx.data_movement_operation_gate();
+        let movement_guard = operation_gate.write().await;
+        let pending = {
+            let store = store.clone();
+            tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await })
+        };
+        tokio::task::yield_now().await;
+        assert!(!pending.is_finished(), "publication admission must wait for a movement writer");
+        drop(movement_guard);
+
+        let (_, epoch) = pending
+            .await
+            .expect("publication admission task should not panic")
+            .expect("idle store should admit publication");
+        assert_eq!(epoch, 0);
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1);
+        let (_, next_epoch) = store
+            .scanner_data_usage_publication_admission_guard()
+            .await
+            .expect("idle store should admit the next publication");
+        assert_eq!(next_epoch, 1);
+    }
+
+    #[tokio::test]
+    async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        let epoch = store
+            .scanner_data_usage_publication_epoch()
+            .await
+            .expect("idle store should expose a publication epoch");
+        assert_eq!(epoch, 0);
+
+        let operation_gate = store.ctx.data_movement_operation_gate();
+        let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write())
+            .await
+            .expect("epoch capture must not hold the movement gate across backend I/O");
+    }
+
+    #[tokio::test]
+    async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *store.rebalance_meta.write().await = Some(RebalanceMeta {
+            pool_stats: vec![crate::services::rebalance::RebalanceStats {
+                participating: true,
+                info: crate::services::rebalance::RebalanceInfo {
+                    status: crate::services::rebalance::RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+
+        assert!(
+            store.scanner_data_usage_publication_admission_guard().await.is_none(),
+            "active rebalance must fail closed at the storage-owned admission boundary"
+        );
+    }
+
+    #[tokio::test]
+    async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() {
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1);
+
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
+        assert!(store.ctx.data_movement_operation_epoch_exhausted());
+        assert!(
+            store.scanner_data_usage_publication_admission_guard().await.is_none(),
+            "publication must fail closed at the reserved terminal epoch"
+        );
+
+        assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX);
+        assert!(store.ctx.data_movement_operation_epoch_exhausted());
+        assert!(store.scanner_data_usage_publication_blocked().await);
+    }
+
     // The object graph is the isolation carrier: two ECStore instances holding
     // distinct contexts report independent erasure state through their real
     // `&self` accessors — no cross-contamination.

crates/ecstore/src/store/rebalance.rs

@@ -694,6 +694,11 @@ impl ECStore {
     /// callers only trigger missing-worker recovery after a real state change;
     /// delayed snapshots are merged monotonically and never blind-assigned.
     pub async fn reload_pool_meta(&self) -> Result<bool> {
+        // Serialize the durable reload with local movement transitions. Loading
+        // before acquiring this gate would allow a stale disk snapshot to
+        // overwrite a newer local transition after the writer commits.
+        let movement_gate = self.ctx.data_movement_operation_gate();
+        let _movement_guard = movement_gate.write().await;
         let mut reloaded = PoolMeta::default();
         resolve_store_rebalance_pool_meta_reload_result(
             reloaded.load(self.pools[0].clone(), self.pools.clone()).await,
@@ -701,15 +706,22 @@ impl ECStore {
         )?;
 
         // Lock order: release the decommission_cancelers guard before taking
-        // the pool_meta write guard; neither is held across the disk read.
+        // the pool_meta write guard; neither is held without the movement gate.
         let active_workers = {
             let cancelers = self.decommission_cancelers.read().await;
-            cancelers.iter().map(Option::is_some).collect::<Vec<_>>()
+            cancelers
+                .iter()
+                .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
+                .collect::<Vec<_>>()
         };
 
         let incoming_has_pools = !reloaded.pools.is_empty();
         let mut pool_meta = self.pool_meta.write().await;
+        let movement_before = pool_meta.clone();
         let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
+        if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
+            self.ctx.advance_data_movement_operation_epoch();
+        }
 
         if !merged_newer && !incoming_has_pools {
             warn!(