fix(scanner): persist decommission catch-up debt (#6922)

crates/ecstore/src/api/mod.rs

@@ -489,9 +489,9 @@ pub mod storage {
     pub use crate::core::pools::HealLifecycleExpiryContext;
     pub use crate::store::HealWalkVersion;
     pub use crate::store::{
-        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
-        init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
-        prewarm_local_disk_id_map_with_instance_ctx,
+        ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
+        find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
+        prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
     };
 }
 

crates/ecstore/src/error/mod.rs

@@ -185,6 +185,12 @@ pub enum StorageError {
     DecommissionAlreadyRunning,
     #[error("Rebalance already running")]
     RebalanceAlreadyRunning,
+    #[error("{operation}: stale pool metadata update rejected for pool {pool_index}; {reason}")]
+    StalePoolMetadataUpdate {
+        operation: String,
+        pool_index: usize,
+        reason: &'static str,
+    },
     #[error("Operation canceled")]
     OperationCanceled,
     #[error("No heal required")]
@@ -564,6 +570,15 @@ impl Clone for StorageError {
             StorageError::DoneForNow => StorageError::DoneForNow,
             StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning,
             StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning,
+            StorageError::StalePoolMetadataUpdate {
+                operation,
+                pool_index,
+                reason,
+            } => StorageError::StalePoolMetadataUpdate {
+                operation: operation.clone(),
+                pool_index: *pool_index,
+                reason,
+            },
             StorageError::OperationCanceled => StorageError::OperationCanceled,
             StorageError::ErasureReadQuorum => StorageError::ErasureReadQuorum,
             StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum,
@@ -667,6 +682,7 @@ impl StorageError {
             StorageError::DoneForNow => StorageErrorCode::DoneForNow,
             StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning,
             StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning,
+            StorageError::StalePoolMetadataUpdate { .. } => StorageErrorCode::InvalidArgument,
             StorageError::OperationCanceled => StorageErrorCode::OperationCanceled,
             StorageError::ErasureReadQuorum => StorageErrorCode::ErasureReadQuorum,
             StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum,

crates/ecstore/src/runtime/instance.rs

@@ -368,6 +368,19 @@ impl InstanceContext {
         Arc::clone(&self.data_movement_generation_notify)
     }
 
+    pub(crate) fn observe_durable_data_movement_generation(&self, generation: u64) {
+        if generation == 0 || self.data_movement_generation_exhausted.load(Ordering::Acquire) {
+            return;
+        }
+        let previous = self.data_movement_generation.fetch_max(generation, Ordering::AcqRel);
+        if generation == u64::MAX {
+            self.data_movement_generation_exhausted.store(true, Ordering::Release);
+        }
+        if generation > previous {
+            self.data_movement_generation_notify.notify_waiters();
+        }
+    }
+
     pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
         !self.data_movement_operation_epoch_exhausted()
             && !self.data_movement_generation_exhausted()
@@ -386,6 +399,20 @@ impl InstanceContext {
     }
 
     pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
+        let (previous, result) = self.advance_data_movement_operation_epoch_only();
+        if result != previous {
+            let _ = self.advance_data_movement_generation();
+        }
+        result
+    }
+
+    pub(crate) fn advance_data_movement_operation_epoch_to_durable_generation(&self, generation: u64) -> u64 {
+        let (_, result) = self.advance_data_movement_operation_epoch_only();
+        self.observe_durable_data_movement_generation(generation);
+        result
+    }
+
+    fn advance_data_movement_operation_epoch_only(&self) -> (u64, u64) {
         self.scanner_publication_state
             .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
         let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
@@ -396,10 +423,7 @@ impl InstanceContext {
         if result == u64::MAX {
             self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
         }
-        if result != previous {
-            let _ = self.advance_data_movement_generation();
-        }
-        result
+        (previous, result)
     }
 
     /// Advance the movement generation after a durable movement transition.

crates/ecstore/src/services/rebalance/control.rs

@@ -845,7 +845,7 @@ impl ECStore {
 
         let mut pool_stats = Vec::with_capacity(self.pools.len());
 
-        let now = OffsetDateTime::now_utc();
+        let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
 
         for disk_stat in disk_stats.iter() {
             let mut pool_stat = RebalanceStats {
@@ -868,8 +868,10 @@ impl ECStore {
             pool_stats.push(pool_stat);
         }
 
+        let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
         let meta = RebalanceMeta {
             id: Uuid::new_v4().to_string(),
+            stopped_at: (!has_participating_pool).then_some(now),
             percent_free_goal,
             pool_stats,
             ..Default::default()
@@ -963,6 +965,18 @@ impl ECStore {
                 )));
             }
             if meta.stopped_at.is_some() {
+                if !is_rebalance_conflicting_with_decommission(meta) {
+                    debug!(
+                        event = EVENT_REBALANCE_STATE,
+                        component = LOG_COMPONENT_ECSTORE,
+                        subsystem = LOG_SUBSYSTEM_REBALANCE,
+                        state = "start_skipped",
+                        reason = "not_started_terminal",
+                        rebalance_id = %expected_id,
+                        "Skipped rebalance start because metadata is already terminal"
+                    );
+                    return Ok(());
+                }
                 return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
             }
         }
@@ -1214,11 +1228,11 @@ impl ECStore {
         };
         let movement_gate = self.ctx.data_movement_operation_gate();
         let _movement_guard = movement_gate.write().await;
+        let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
         let (previous_meta, meta_to_save) = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
             let previous_meta = rebalance_meta.clone();
-            let meta_to_save =
-                stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)?;
+            let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
             (previous_meta, meta_to_save)
         };
 
@@ -1250,14 +1264,10 @@ impl ECStore {
             .await?;
         let movement_gate = self.ctx.data_movement_operation_gate();
         let _movement_guard = movement_gate.write().await;
+        let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
-            rollback_rebalance_start_meta_snapshot_for_id(
-                rebalance_meta.as_mut(),
-                OffsetDateTime::now_utc(),
-                expected_id,
-                start_error,
-            )
+            rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
         };
 
         if let Some(meta_to_save) = meta_to_save {
@@ -1402,6 +1412,62 @@ mod tests {
         assert!(cancel.is_cancelled());
     }
 
+    #[tokio::test]
+    #[serial_test::serial]
+    async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
+        let (_temp_dirs, store, restarted) =
+            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
+        let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
+        *store.rebalance_meta.write().await = Some(RebalanceMeta {
+            id: "previous-terminal-rebalance".to_string(),
+            stopped_at: Some(movement_floor),
+            ..Default::default()
+        });
+        set_rebalance_disk_stats_override_for_test(
+            store.id,
+            vec![
+                DiskStat {
+                    total_space: 100,
+                    available_space: 50,
+                },
+                DiskStat {
+                    total_space: 100,
+                    available_space: 50,
+                },
+            ],
+        );
+
+        let rebalance_id = store
+            .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
+            .await
+            .expect("equal free ratio admin rebalance should succeed as a terminal no-op");
+        let stopped_at = {
+            let local = store.rebalance_meta.read().await;
+            let local = local.as_ref().expect("no-op rebalance metadata should remain available");
+            assert_eq!(local.id, rebalance_id);
+            assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
+            let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
+            assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
+            stopped_at
+        };
+
+        let stopped_generation =
+            u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
+        let live_status = store.scanner_data_movement_pause_status().await;
+        assert!(!live_status.paused);
+        assert_eq!(live_status.movement_generation, stopped_generation);
+
+        restarted
+            .load_rebalance_meta()
+            .await
+            .expect("restarted store should load the persisted no-op rebalance metadata");
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, stopped_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
+    }
+
     #[tokio::test]
     #[serial_test::serial]
     async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {

crates/ecstore/src/services/rebalance/runtime.rs

@@ -161,6 +161,7 @@ impl ECStore {
 
         let cancel_tx = CancellationToken::new();
         let rx = cancel_tx.clone();
+        let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
         let activation_outcome;
         let candidate;
         let expected_cancel;
@@ -185,12 +186,8 @@ impl ECStore {
                 return Ok(false);
             }
             expected_cancel = meta.cancel.clone();
-            (candidate, activation_outcome, must_persist) = stage_local_rebalance_worker_activation(
-                meta,
-                expected_id.as_ref(),
-                cancel_tx.clone(),
-                OffsetDateTime::now_utc(),
-            )?;
+            (candidate, activation_outcome, must_persist) =
+                stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
             if let Err(err) = activation_fence.ensure_held() {
                 cancel_tx.cancel();
                 return Err(err);
@@ -384,11 +381,11 @@ impl ECStore {
                 tokio::select! {
                     result = done_rx.recv() => {
                         quit = true;
-                        let now = OffsetDateTime::now_utc();
-                        let terminal_event = classify_rebalance_terminal_event(result, now);
-                        msg = terminal_event.message().to_string();
                         let movement_gate = store.ctx.data_movement_operation_gate();
                         let movement_guard = movement_gate.write().await;
+                        let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
+                        let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
+                        msg = terminal_event.message().to_string();
                         let previous_meta = store.rebalance_meta.read().await.clone();
                         let terminal_state_present = {
                             let mut rebalance_meta = store.rebalance_meta.write().await;
@@ -405,7 +402,7 @@ impl ECStore {
                                     {
                                         pool_stat.info.stopping = false;
                                         pool_stat.info.status = RebalStatus::Failed;
-                                        pool_stat.info.end_time = Some(now);
+                                        pool_stat.info.end_time = Some(terminal_at);
                                         pool_stat.info.last_error = Some(
                                             pool_stat
                                                 .cleanup_warnings
@@ -433,7 +430,7 @@ impl ECStore {
                                             &mut pool_stat.info.end_time,
                                             &mut pool_stat.info.last_error,
                                             terminal_event,
-                                            now,
+                                            terminal_at,
                                         );
                                     }
                                     true
@@ -835,6 +832,10 @@ impl ECStore {
         opt: RebalSaveOpt,
         expected_id: Option<&str>,
     ) -> Result<()> {
+        let now = match opt {
+            RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
+            RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
+        };
         let meta_to_save = {
             let mut rebalance_meta = self.rebalance_meta.write().await;
             if let Some(expected_id) = expected_id {
@@ -844,7 +845,6 @@ impl ECStore {
                 return Ok(());
             };
 
-            let now = OffsetDateTime::now_utc();
             apply_rebalance_save_option(meta, pool_idx, opt, now);
             meta.clone()
         };

crates/ecstore/src/store/mod.rs

@@ -44,7 +44,7 @@ use crate::error::{
 use crate::runtime::global::DISK_RESERVE_FRACTION;
 use crate::runtime::instance::InstanceContext;
 use crate::runtime::sources as runtime_sources;
-use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
+use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
 use crate::storage_api_contracts::{
     bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
     list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
@@ -273,6 +273,215 @@ pub struct ECStore {
     pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
 }
 
+const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
+const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
+const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
+const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ScannerDataMovementPauseReason {
+    OperationEpochExhausted,
+    MovementGenerationExhausted,
+    DecommissionActive,
+    DecommissionFailed,
+    DecommissionCanceled,
+    RebalanceActive,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
+pub struct ScannerDataMovementPauseStatus {
+    pub paused: bool,
+    pub policy: &'static str,
+    pub reasons: Vec<ScannerDataMovementPauseReason>,
+    pub started_at_unix_secs: u64,
+    pub duration_seconds: u64,
+    pub operation_epoch: u64,
+    pub movement_generation: u64,
+    pub movement_backlog_work_items: u64,
+    pub movement_backlog_estimated: bool,
+}
+
+impl Default for ScannerDataMovementPauseStatus {
+    fn default() -> Self {
+        Self {
+            paused: false,
+            policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
+            reasons: Vec::new(),
+            started_at_unix_secs: 0,
+            duration_seconds: 0,
+            operation_epoch: 0,
+            movement_generation: 0,
+            movement_backlog_work_items: 0,
+            movement_backlog_estimated: false,
+        }
+    }
+}
+
+fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
+    u64::try_from(value.unix_timestamp()).unwrap_or(0)
+}
+
+fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
+    match (current, candidate) {
+        (Some(current), Some(candidate)) => Some(current.min(candidate)),
+        (Some(current), None) => Some(current),
+        (None, candidate) => candidate,
+    }
+}
+
+fn usize_to_u64(value: usize) -> u64 {
+    u64::try_from(value).unwrap_or(u64::MAX)
+}
+
+fn metric_u64(value: u64) -> f64 {
+    f64::from(u32::try_from(value).unwrap_or(u32::MAX))
+}
+
+pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
+    let timestamp = value.unix_timestamp_nanos();
+    if timestamp <= 0 {
+        0
+    } else {
+        u64::try_from(timestamp).unwrap_or(u64::MAX)
+    }
+}
+
+fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
+    let generation = scanner_data_movement_timestamp_generation(value);
+    (generation != 0 && generation != u64::MAX).then_some(generation)
+}
+
+fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
+    let mut generation = 0;
+    for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
+        let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
+            return u64::MAX;
+        };
+        generation = generation.max(pool_generation);
+    }
+
+    for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
+        meta.stopped_at.into_iter().chain(
+            meta.pool_stats
+                .iter()
+                .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
+                .flatten(),
+        )
+    }) {
+        let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
+            return u64::MAX;
+        };
+        generation = generation.max(rebalance_generation);
+    }
+
+    if generation == 0
+        && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
+    {
+        u64::MAX
+    } else {
+        generation
+    }
+}
+
+#[derive(Clone, Copy)]
+struct ScannerDataMovementSequenceState {
+    operation_epoch: u64,
+    operation_epoch_exhausted: bool,
+    movement_generation: u64,
+    movement_generation_exhausted: bool,
+}
+
+fn resolve_scanner_data_movement_pause_status(
+    pool_meta: &PoolMeta,
+    rebalance_meta: Option<&RebalanceMeta>,
+    decommission_worker_active: bool,
+    sequence: ScannerDataMovementSequenceState,
+    now: OffsetDateTime,
+) -> ScannerDataMovementPauseStatus {
+    let mut decommission_active = decommission_worker_active;
+    let mut decommission_failed = false;
+    let mut decommission_canceled = false;
+    let mut rebalance_active = false;
+    let mut started_at = None;
+    let mut movement_backlog_work_items = 0_u64;
+
+    for pool in &pool_meta.pools {
+        let Some(info) = pool.decommission.as_ref() else {
+            continue;
+        };
+        let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
+        let failed = !info.queued && info.failed;
+        let canceled = !info.queued && info.canceled;
+        if !(active || failed || canceled) {
+            continue;
+        }
+
+        decommission_active |= active;
+        decommission_failed |= failed;
+        decommission_canceled |= canceled;
+        started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
+        let queued = usize_to_u64(info.queued_buckets.len());
+        let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
+        movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
+    }
+
+    if let Some(rebalance_meta) = rebalance_meta {
+        for pool in &rebalance_meta.pool_stats {
+            let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
+            if !active {
+                continue;
+            }
+            rebalance_active = true;
+            started_at = earliest_timestamp(started_at, pool.info.start_time);
+            movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
+        }
+    }
+
+    let mut reasons = Vec::with_capacity(6);
+    if sequence.operation_epoch_exhausted {
+        reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
+    }
+    if sequence.movement_generation_exhausted {
+        reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
+    }
+    if decommission_active {
+        reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
+    }
+    if decommission_failed {
+        reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
+    }
+    if decommission_canceled {
+        reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
+    }
+    if rebalance_active {
+        reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
+    }
+    let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
+    let duration_seconds = started_at
+        .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
+        .unwrap_or(0);
+    let paused = !reasons.is_empty();
+
+    ScannerDataMovementPauseStatus {
+        paused,
+        policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
+        reasons,
+        started_at_unix_secs,
+        duration_seconds,
+        operation_epoch: sequence.operation_epoch,
+        movement_generation: sequence.movement_generation,
+        movement_backlog_work_items,
+        movement_backlog_estimated: paused,
+    }
+}
+
+fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
+    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
+    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
+    metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
+}
+
 impl std::fmt::Debug for ECStore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
@@ -300,6 +509,28 @@ impl ECStore {
         self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
     }
 
+    /// Erasure sets that may receive scanner pause-backlog replicas.
+    ///
+    /// An actively decommissioning or already decommissioned source pool is
+    /// excluded so an operational record acknowledged during movement always
+    /// has a copy on storage that remains in the cluster. The record is kept
+    /// separate from pool and rebalance metadata.
+    pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
+        let pool_meta = self.pool_meta.read().await;
+        self.pools
+            .iter()
+            .enumerate()
+            .filter(|(pool_index, _)| {
+                !pool_meta.pools.get(*pool_index).is_some_and(|pool| {
+                    pool.decommission
+                        .as_ref()
+                        .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
+                })
+            })
+            .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
+            .collect()
+    }
+
     /// Get server configuration (delegates to global)
     pub fn get_server_config(&self) -> Option<Config> {
         runtime_sources::server_config()
@@ -454,14 +685,14 @@ impl ECStore {
         self.scanner_data_usage_publication_snapshot_blocked().await
     }
 
+    pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
+        let operation_gate = self.ctx.data_movement_operation_gate();
+        let _operation_guard = operation_gate.read_owned().await;
+        self.scanner_data_movement_pause_snapshot().await
+    }
+
     async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
-        if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
-            self.ctx.set_scanner_publication_state(true);
-            return true;
-        }
-        let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
-        self.ctx.set_scanner_publication_state(blocked);
-        blocked
+        self.scanner_data_movement_pause_snapshot().await.paused
     }
 
     async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
@@ -481,19 +712,56 @@ impl ECStore {
                 .as_ref()
                 .is_some_and(|info| !info.queued && (info.failed || info.canceled))
         });
-        drop(pool_meta);
-
-        let rebalance_active = self
-            .rebalance_meta
-            .read()
-            .await
+        let rebalance_meta = self.rebalance_meta.read().await;
+        let rebalance_active = rebalance_meta
             .as_ref()
             .is_some_and(is_rebalance_conflicting_with_decommission);
+        self.ctx
+            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
+                &pool_meta,
+                rebalance_meta.as_ref(),
+            ));
 
         let blocked = decommission_active || decommission_terminal || rebalance_active;
         (decommission_active || rebalance_active, blocked)
     }
 
+    async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
+        let decommission_active = {
+            let decommission_cancelers = self.decommission_cancelers.read().await;
+            decommission_cancelers
+                .iter()
+                .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
+        };
+        let pool_meta = self.pool_meta.read().await.clone();
+        let rebalance_meta = self.rebalance_meta.read().await.clone();
+        self.ctx
+            .observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
+                &pool_meta,
+                rebalance_meta.as_ref(),
+            ));
+        let status = resolve_scanner_data_movement_pause_status(
+            &pool_meta,
+            rebalance_meta.as_ref(),
+            decommission_active,
+            ScannerDataMovementSequenceState {
+                operation_epoch: self.ctx.data_movement_operation_epoch(),
+                operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
+                movement_generation: self.ctx.data_movement_generation(),
+                movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
+            },
+            OffsetDateTime::now_utc(),
+        );
+        self.ctx.set_scanner_publication_state(status.paused);
+        record_scanner_data_movement_pause_status(&status);
+        status
+    }
+
+    #[cfg(test)]
+    pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
+        self.scanner_data_movement_pause_snapshot().await
+    }
+
     /// Admit one short data-usage publication commit under the same
     /// per-instance gate used by decommission side effects and transitions.
     /// The epoch is sampled while the read guard is held, so a transition
@@ -1196,7 +1464,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
+    use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
     use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
     use crate::object_api::ObjectOptions;
     use crate::runtime::global::reset_local_disk_test_state;
@@ -1326,6 +1594,558 @@ mod tests {
         })
     }
 
+    fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
+        ScannerDataMovementSequenceState {
+            operation_epoch,
+            operation_epoch_exhausted: false,
+            movement_generation,
+            movement_generation_exhausted: false,
+        }
+    }
+
+    #[test]
+    fn scanner_pause_status_derives_restart_stable_decommission_fields() {
+        let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
+        let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
+        let pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: started_at,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(started_at),
+                    queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+                    bucket: "bucket-a".to_string(),
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
+
+        assert!(status.paused);
+        assert_eq!(status.policy, "global_pause");
+        assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
+        assert_eq!(status.started_at_unix_secs, 1_000);
+        assert_eq!(status.duration_seconds, 90);
+        assert_eq!(status.operation_epoch, 7);
+        assert_eq!(status.movement_generation, 11);
+        assert_eq!(status.movement_backlog_work_items, 2);
+        assert!(status.movement_backlog_estimated);
+    }
+
+    #[test]
+    fn completed_decommission_restores_durable_movement_generation() {
+        let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
+        let pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: completed_at,
+                decommission: Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
+        let ctx = InstanceContext::new();
+
+        ctx.observe_durable_data_movement_generation(durable_generation);
+
+        assert_eq!(durable_generation, 1_100_000_000_000);
+        assert_eq!(ctx.data_movement_generation(), durable_generation);
+    }
+
+    #[tokio::test]
+    async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
+        let mut pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(PoolDecommissionInfo {
+                    failed: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+        assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
+        assert!(
+            pool_meta.pools[0]
+                .decommission
+                .as_ref()
+                .is_some_and(|info| !info.has_decommission_state())
+        );
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert_ne!(durable_generation, 0);
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, durable_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
+    }
+
+    #[tokio::test]
+    async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
+        let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
+        let mut pool_meta = PoolMeta {
+            pools: vec![
+                PoolStatus {
+                    id: 0,
+                    cmd_line: "pool-0".to_string(),
+                    last_update: same_tick,
+                    decommission: Some(PoolDecommissionInfo {
+                        failed: true,
+                        ..Default::default()
+                    }),
+                },
+                PoolStatus {
+                    id: 1,
+                    cmd_line: "pool-1".to_string(),
+                    last_update: same_tick,
+                    decommission: Some(PoolDecommissionInfo {
+                        canceled: true,
+                        ..Default::default()
+                    }),
+                },
+            ],
+            ..Default::default()
+        };
+
+        assert!(
+            pool_meta
+                .clear_decommission_at_for_test(0, same_tick, None)
+                .expect("first terminal decommission should clear")
+        );
+        let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
+        assert_eq!(
+            first_generation,
+            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
+        );
+
+        assert!(
+            pool_meta
+                .clear_decommission_at_for_test(1, same_tick, None)
+                .expect("second terminal decommission should clear")
+        );
+        let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
+        assert_eq!(
+            second_generation,
+            scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
+        );
+        assert!(second_generation > first_generation);
+
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, second_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
+    }
+
+    #[tokio::test]
+    async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
+        let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
+        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
+        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
+        let rebalance = RebalanceMeta {
+            stopped_at: Some(rebalance_floor),
+            id: "completed-rebalance".to_string(),
+            ..Default::default()
+        };
+        let active_decommission = |id| PoolStatus {
+            id,
+            cmd_line: format!("pool-{id}"),
+            last_update: same_tick,
+            decommission: Some(PoolDecommissionInfo {
+                start_time: Some(same_tick),
+                ..Default::default()
+            }),
+        };
+        let mut pool_meta = PoolMeta {
+            pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
+            ..Default::default()
+        };
+
+        assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
+        assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
+
+        assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
+        assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
+
+        assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
+        assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
+        assert_eq!(
+            durable_generation,
+            scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
+        );
+
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+        *restarted.rebalance_meta.write().await = Some(rebalance);
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert_eq!(status.movement_generation, durable_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
+        assert_eq!(
+            status.reasons,
+            vec![
+                ScannerDataMovementPauseReason::DecommissionFailed,
+                ScannerDataMovementPauseReason::DecommissionCanceled
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
+        let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
+        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
+        let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
+        let rebalance = RebalanceMeta {
+            stopped_at: Some(rebalance_floor),
+            id: "completed-rebalance".to_string(),
+            ..Default::default()
+        };
+        let mut pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: same_tick,
+                decommission: Some(PoolDecommissionInfo {
+                    failed: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        assert!(
+            pool_meta
+                .clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
+                .expect("failed decommission should clear")
+        );
+        let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
+        assert_eq!(pool_meta.pools[0].last_update, cleared_at);
+
+        pool_meta
+            .decommission_at_for_test(
+                0,
+                PoolSpaceInfo {
+                    total: 200,
+                    free: 50,
+                    used: 150,
+                },
+                earlier_tick,
+                Some(&rebalance),
+            )
+            .expect("decommission restart after clear should be allowed");
+        let started_at = cleared_at + time::Duration::nanoseconds(1);
+        assert_eq!(pool_meta.pools[0].last_update, started_at);
+        assert_eq!(
+            pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
+            Some(started_at)
+        );
+
+        assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
+        let completed_at = started_at + time::Duration::nanoseconds(1);
+        assert_eq!(pool_meta.pools[0].last_update, completed_at);
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
+        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
+
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+        *restarted.rebalance_meta.write().await = Some(rebalance);
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, durable_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
+    }
+
+    #[tokio::test]
+    async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
+        let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
+        let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
+        let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
+        let rebalance = RebalanceMeta {
+            stopped_at: Some(rebalance_floor),
+            id: "completed-rebalance".to_string(),
+            ..Default::default()
+        };
+        let mut pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: terminal_at,
+                decommission: Some(PoolDecommissionInfo {
+                    start_time: Some(terminal_at),
+                    complete: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        assert!(
+            pool_meta
+                .record_decommission_terminal_reload_failure_at_for_test(
+                    0,
+                    "complete_decommission",
+                    "peer reload failed".to_string(),
+                    earlier_tick,
+                    Some(&rebalance),
+                )
+                .expect("reload failure should be recorded")
+        );
+        let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
+        assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
+        let info = pool_meta.pools[0]
+            .decommission
+            .as_ref()
+            .expect("decommission metadata should exist");
+        assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
+        assert_eq!(
+            info.terminal_reload_failures,
+            vec!["complete_decommission: peer reload failed".to_string()]
+        );
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
+        assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
+
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+        *restarted.rebalance_meta.write().await = Some(rebalance);
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, durable_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
+    }
+
+    #[tokio::test]
+    async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
+        let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
+        let earlier_tick = same_tick - time::Duration::nanoseconds(10);
+        let decommission_floor = same_tick + time::Duration::nanoseconds(5);
+        let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *store.pool_meta.write().await = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: decommission_floor,
+                decommission: Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        let started_at = store.next_scanner_data_movement_update(same_tick).await;
+        assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
+        *store.rebalance_meta.write().await = Some(RebalanceMeta {
+            id: "rebalance-generation".to_string(),
+            pool_stats: vec![crate::services::rebalance::RebalanceStats {
+                participating: true,
+                info: crate::services::rebalance::RebalanceInfo {
+                    start_time: Some(started_at),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        });
+
+        let completed_at = store.next_scanner_data_movement_update(same_tick).await;
+        assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
+        {
+            let mut rebalance_meta = store.rebalance_meta.write().await;
+            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
+            meta.pool_stats[0].info.status = RebalStatus::Completed;
+            meta.pool_stats[0].info.end_time = Some(completed_at);
+        }
+
+        let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
+        assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
+        {
+            let mut rebalance_meta = store.rebalance_meta.write().await;
+            let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
+            meta.stopped_at = Some(stopped_at);
+        }
+        let pool_meta = store.pool_meta.read().await.clone();
+        let rebalance_meta = store.rebalance_meta.read().await.clone();
+        let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
+        assert_eq!(
+            durable_generation,
+            scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
+        );
+
+        let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
+        *restarted.pool_meta.write().await = pool_meta;
+        *restarted.rebalance_meta.write().await = rebalance_meta;
+        let status = restarted.scanner_data_movement_pause_status().await;
+
+        assert!(!status.paused);
+        assert_eq!(status.movement_generation, durable_generation);
+        assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
+    }
+
+    #[test]
+    fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
+        let pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+
+        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
+        let exhausted_generation =
+            OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
+        assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
+    }
+
+    #[test]
+    fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
+        let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
+        let pool_meta = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update: OffsetDateTime::UNIX_EPOCH,
+                decommission: Some(PoolDecommissionInfo {
+                    complete: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+        let rebalance_meta = RebalanceMeta {
+            id: "completed-rebalance".to_string(),
+            stopped_at: Some(valid_rebalance_at),
+            pool_stats: vec![crate::services::rebalance::RebalanceStats {
+                participating: true,
+                info: crate::services::rebalance::RebalanceInfo {
+                    start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
+                    end_time: Some(valid_rebalance_at),
+                    status: RebalStatus::Completed,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+
+        assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
+    }
+
+    #[test]
+    fn durable_movement_generation_without_records_is_zero() {
+        assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
+        assert_eq!(
+            durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
+            0
+        );
+    }
+
+    #[test]
+    fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
+        let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
+        let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
+        let failed = PoolMeta {
+            pools: vec![PoolStatus {
+                id: 0,
+                cmd_line: "pool-0".to_string(),
+                last_update,
+                decommission: Some(PoolDecommissionInfo {
+                    failed: true,
+                    ..Default::default()
+                }),
+            }],
+            ..Default::default()
+        };
+        let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
+        assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
+        assert_eq!(failed_status.started_at_unix_secs, 2_000);
+        assert_eq!(failed_status.duration_seconds, 30);
+
+        let rebalance = RebalanceMeta {
+            pool_stats: vec![crate::services::rebalance::RebalanceStats {
+                buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
+                participating: true,
+                info: crate::services::rebalance::RebalanceInfo {
+                    start_time: Some(last_update),
+                    status: RebalStatus::Started,
+                    ..Default::default()
+                },
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        let rebalance_status = resolve_scanner_data_movement_pause_status(
+            &PoolMeta::default(),
+            Some(&rebalance),
+            false,
+            scanner_sequence_state(4, 13),
+            now,
+        );
+        assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
+        assert_eq!(rebalance_status.movement_backlog_work_items, 2);
+
+        let exhausted = resolve_scanner_data_movement_pause_status(
+            &PoolMeta::default(),
+            None,
+            false,
+            ScannerDataMovementSequenceState {
+                operation_epoch: u64::MAX,
+                operation_epoch_exhausted: true,
+                movement_generation: 14,
+                movement_generation_exhausted: false,
+            },
+            now,
+        );
+        assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
+        assert_eq!(exhausted.started_at_unix_secs, 0);
+
+        let generation_exhausted = resolve_scanner_data_movement_pause_status(
+            &PoolMeta::default(),
+            None,
+            false,
+            ScannerDataMovementSequenceState {
+                operation_epoch: 5,
+                operation_epoch_exhausted: false,
+                movement_generation: u64::MAX,
+                movement_generation_exhausted: true,
+            },
+            now,
+        );
+        assert_eq!(
+            generation_exhausted.reasons,
+            vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
+        );
+
+        let idle =
+            resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
+        assert!(!idle.paused);
+        assert!(idle.reasons.is_empty());
+        assert!(!idle.movement_backlog_estimated);
+    }
+
     #[tokio::test]
     async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
         let store = build_store_with_ctx(Arc::new(InstanceContext::new()));

crates/scanner/Cargo.toml

@@ -106,7 +106,7 @@ hex-simd.workspace = true
 [dev-dependencies]
 tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
 serial_test = { workspace = true }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tempfile = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }

crates/scanner/src/lib.rs

@@ -82,8 +82,10 @@ pub use remote_scanner::{
 pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
 pub use rustfs_scanner_contracts::last_minute;
 pub use scanner::{
-    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
-    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
+    ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, init_data_scanner,
+    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status,
+    scanner_topology_digest,
 };
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,

crates/scanner/src/scanner.rs

@@ -90,6 +90,144 @@ const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state
 const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total";
 const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
 const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60);
+
+#[cfg(test)]
+static SCANNER_STARTUP_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerStartupObservedProbeState>>>> =
+    LazyLock::new(|| StdMutex::new(None));
+
+#[cfg(test)]
+struct ScannerStartupObservedProbeState {
+    observed: Notify,
+    resume: Notify,
+}
+
+#[cfg(test)]
+struct ScannerObservedProbeState {
+    store_key: usize,
+    paused: bool,
+    notify: Notify,
+}
+
+#[cfg(test)]
+pub(super) struct ScannerStartupObservedProbe {
+    state: Arc<ScannerStartupObservedProbeState>,
+}
+
+#[cfg(test)]
+static SCANNER_RUNTIME_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerObservedProbeState>>>> =
+    LazyLock::new(|| StdMutex::new(None));
+
+#[cfg(test)]
+pub(super) struct ScannerRuntimeObservedProbe {
+    state: Arc<ScannerObservedProbeState>,
+}
+
+#[cfg(test)]
+impl ScannerStartupObservedProbe {
+    pub(super) fn install() -> Self {
+        let state = Arc::new(ScannerStartupObservedProbeState {
+            observed: Notify::new(),
+            resume: Notify::new(),
+        });
+        let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned");
+        assert!(probe.is_none(), "scanner startup observed probe must be unique");
+        *probe = Some(state.clone());
+        Self { state }
+    }
+
+    pub(super) async fn wait(&self) {
+        tokio::time::timeout(Duration::from_secs(5), self.state.observed.notified())
+            .await
+            .expect("scanner should complete startup pause-backlog observation");
+    }
+
+    pub(super) fn resume(&self) {
+        self.state.resume.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl ScannerRuntimeObservedProbe {
+    pub(super) fn install(storeapi: &Arc<ECStore>, paused: bool) -> Self {
+        let state = Arc::new(ScannerObservedProbeState {
+            store_key: scanner_observed_probe_store_key(storeapi),
+            paused,
+            notify: Notify::new(),
+        });
+        let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
+            .lock()
+            .expect("scanner runtime observed probe should not be poisoned");
+        assert!(probe.is_none(), "scanner runtime observed probe must be unique");
+        *probe = Some(state.clone());
+        Self { state }
+    }
+
+    pub(super) async fn wait(&self) {
+        tokio::time::timeout(Duration::from_secs(10), self.state.notify.notified())
+            .await
+            .expect("scanner should complete runtime pause-backlog observation");
+    }
+}
+
+#[cfg(test)]
+impl Drop for ScannerStartupObservedProbe {
+    fn drop(&mut self) {
+        let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned");
+        if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *probe = None;
+        }
+        self.state.resume.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for ScannerRuntimeObservedProbe {
+    fn drop(&mut self) {
+        let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
+            .lock()
+            .expect("scanner runtime observed probe should not be poisoned");
+        if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *probe = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn notify_scanner_startup_observed_for_test() {
+    let probe = {
+        SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned")
+            .clone()
+    };
+    if let Some(probe) = probe {
+        probe.observed.notify_one();
+        probe.resume.notified().await;
+    }
+}
+
+#[cfg(test)]
+fn scanner_observed_probe_store_key(storeapi: &Arc<ECStore>) -> usize {
+    Arc::as_ptr(storeapi).cast::<()>() as usize
+}
+
+#[cfg(test)]
+fn notify_scanner_runtime_observed_for_test(storeapi: &Arc<ECStore>, observation: ScannerPauseBacklogObservation) {
+    if let Some(probe) = SCANNER_RUNTIME_OBSERVED_PROBE
+        .lock()
+        .expect("scanner runtime observed probe should not be poisoned")
+        .clone()
+        && probe.store_key == scanner_observed_probe_store_key(storeapi)
+        && probe.paused == observation.paused
+    {
+        probe.notify.notify_one();
+    }
+}
+
 const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
 /// First-retry delay after a scanner cycle cannot publish authoritative usage.
 ///
@@ -2190,6 +2328,74 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
     run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await
 }
 
+async fn current_scanner_pause_backlog_observation(storeapi: &Arc<ECStore>) -> ScannerPauseBacklogObservation {
+    let now_unix_secs = scanner_pause_backlog_now();
+    let pause = storeapi.scanner_data_movement_pause_status().await;
+    let metrics = global_metrics().report().await;
+    ScannerPauseBacklogObservation {
+        now_unix_secs,
+        paused: pause.paused,
+        movement_generation: pause.movement_generation,
+        movement_work_items: pause.movement_backlog_work_items,
+        pause_started_at_unix_secs: pause.started_at_unix_secs,
+        dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets,
+        discovered_expiry_items: metrics
+            .lifecycle_expiry
+            .current_queued
+            .saturating_add(metrics.lifecycle_expiry.current_active),
+        discovered_transition_items: metrics
+            .lifecycle_transition
+            .current_queued
+            .saturating_add(metrics.lifecycle_transition.current_active)
+            .saturating_add(metrics.lifecycle_transition.compensation_pending)
+            .saturating_add(metrics.lifecycle_transition.compensation_running),
+    }
+}
+
+async fn wait_for_scanner_data_movement_resume(
+    ctx: &CancellationToken,
+    storeapi: &Arc<ECStore>,
+    guard: &NamespaceLockGuard,
+    pause_backlog: &mut ScannerPauseBacklogController,
+) -> bool {
+    loop {
+        let observation = current_scanner_pause_backlog_observation(storeapi).await;
+        pause_backlog.observe(observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(storeapi, observation);
+        if !observation.paused {
+            return !ctx.is_cancelled() && !guard.is_lock_lost();
+        }
+
+        let movement_changed = storeapi.scanner_data_movement_changed();
+        if storeapi.scanner_data_movement_generation() != observation.movement_generation {
+            continue;
+        }
+        tokio::select! {
+            _ = ctx.cancelled() => return false,
+            _ = guard.lock_lost_notified() => return false,
+            _ = movement_changed.notified() => {},
+            _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
+        }
+    }
+}
+
+async fn finish_scanner_pause_backlog_cycle(
+    pause_backlog: &mut ScannerPauseBacklogController,
+    storeapi: &Arc<ECStore>,
+    attempt: ScannerPauseBacklogAttemptDecision,
+    outcome: ScannerCycleOutcome,
+) {
+    let observation = current_scanner_pause_backlog_observation(storeapi).await;
+    if let ScannerPauseBacklogAttemptDecision::Tracked(serial) = attempt {
+        pause_backlog.finish_attempt(serial, outcome, observation).await;
+    } else {
+        pause_backlog.observe_cycle_outcome(outcome, observation).await;
+    }
+    #[cfg(test)]
+    notify_scanner_runtime_observed_for_test(storeapi, observation);
+}
+
 async fn run_data_scanner_with_maintenance_state(
     ctx: CancellationToken,
     storeapi: Arc<ECStore>,
@@ -2269,6 +2475,28 @@ async fn run_data_scanner_with_maintenance_state(
             return Ok(());
         }
     };
+    let pause_backlog_now = scanner_pause_backlog_now();
+    let mut pause_backlog = match ScannerPauseBacklogController::claim(storeapi.clone(), pause_backlog_now).await {
+        Ok(controller) => controller,
+        Err(err) => {
+            error!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                state = "pause_backlog_claim_failed",
+                error = %err,
+                "Scanner pause backlog persistence is unavailable"
+            );
+            ScannerPauseBacklogController::unavailable(storeapi.clone(), err, pause_backlog_now)
+        }
+    };
+    if !wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
+        global_metrics().set_cycle(None).await;
+        return Ok(());
+    }
+    #[cfg(test)]
+    notify_scanner_startup_observed_for_test().await;
     let single_disk = storeapi.setup_is_erasure_sd().await;
     let erasure = storeapi.setup_is_erasure().await;
     let distributed = storeapi.setup_is_dist_erasure().await;
@@ -2416,6 +2644,19 @@ async fn run_data_scanner_with_maintenance_state(
         return Ok(());
     }
     if !leadership_claimed {
+        let observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, observation);
+        if observation.paused {
+            if wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
+                return Err(ScannerError::Other(
+                    "scanner startup was fenced by data movement; retrying from durable state".to_string(),
+                ));
+            }
+            global_metrics().set_cycle(None).await;
+            return Ok(());
+        }
         error!(
             target: "rustfs::scanner",
             event = EVENT_SCANNER_LOCK_STATE,
@@ -2448,7 +2689,13 @@ async fn run_data_scanner_with_maintenance_state(
         return Ok(());
     }
 
-    if !ctx.is_cancelled() {
+    let initial_pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
+    if !ctx.is_cancelled()
+        && matches!(
+            initial_pause_backlog_attempt,
+            ScannerPauseBacklogAttemptDecision::Untracked | ScannerPauseBacklogAttemptDecision::Tracked(_)
+        )
+    {
         // Preserve previous behavior: run one cycle immediately after lock acquisition.
         let dirty_generation_before_cycle = dirty_usage_generation();
         let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending();
@@ -2506,6 +2753,7 @@ async fn run_data_scanner_with_maintenance_state(
                 return Ok(());
             }
         };
+        finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, initial_pause_backlog_attempt, initial_outcome).await;
         superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -2558,6 +2806,10 @@ async fn run_data_scanner_with_maintenance_state(
             break;
         }
 
+        let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(pause_backlog_observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
         let runtime_config = resolve_scanner_runtime_config();
         if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) {
             let current_generation = scanner_maintenance_generation();
@@ -2594,11 +2846,16 @@ async fn run_data_scanner_with_maintenance_state(
             scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
         let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval);
         let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
-        let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
+        let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
         if let Some(retry_interval) = convergence_retry_interval {
             wait_plan.effective_interval = retry_interval;
             wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval);
         }
+        if let Some(pause_backlog_delay) = pause_backlog.scheduling_delay(scanner_pause_backlog_now()) {
+            wait_plan.effective_interval = pause_backlog_delay.max(Duration::from_secs(1));
+            wait_plan.delay = pause_backlog_delay;
+            convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1)));
+        }
         let dirty_generation_before_wait = dirty_usage_generation();
         let dirty_usage_pending_before_wait = dirty_usage_buckets_pending();
         let maintenance_generation_before_wait = scanner_maintenance_generation();
@@ -2723,6 +2980,20 @@ async fn run_data_scanner_with_maintenance_state(
             record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await;
             break;
         }
+        let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(pause_backlog_observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
+        if pause_backlog_observation.paused {
+            continue;
+        }
+        let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
+        if matches!(
+            pause_backlog_attempt,
+            ScannerPauseBacklogAttemptDecision::RateLimited | ScannerPauseBacklogAttemptDecision::PersistenceUnavailable
+        ) {
+            continue;
+        }
         let dirty_generation_before_cycle = dirty_usage_generation();
         let cycle_ctx = ctx.child_token();
         let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
@@ -2771,6 +3042,7 @@ async fn run_data_scanner_with_maintenance_state(
                 return Ok(());
             }
         };
+        finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, pause_backlog_attempt, outcome).await;
         superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -3088,12 +3360,14 @@ fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Op
 
 /// Store data usage info in backend. Will store all objects sent on the receiver until closed.
 mod activity;
+mod backlog;
 mod cycle_state;
 mod heal_info;
 mod leadership;
 mod usage_store;
 
 use activity::*;
+use backlog::*;
 use cycle_state::*;
 use leadership::*;
 use usage_store::*;
@@ -3104,6 +3378,10 @@ pub(crate) use activity::{
     scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
 };
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
+pub use backlog::{
+    ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds,
+    scanner_pause_backlog_status,
+};
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
 pub use cycle_state::{

crates/scanner/src/scanner/tests.rs

@@ -39,30 +39,46 @@ async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
 }
 
 async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool) -> (tempfile::TempDir, Arc<ECStore>) {
+    setup_scanner_cycle_store_with_pool_count(seed_usage_baseline, 1).await
+}
+
+async fn setup_scanner_cycle_store_with_pool_count(
+    seed_usage_baseline: bool,
+    pool_count: usize,
+) -> (tempfile::TempDir, Arc<ECStore>) {
     init_ecstore_config_for_scanner_tests();
     let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
-    let mut endpoints = Vec::new();
-    for disk_index in 0..4 {
-        let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
-        tokio::fs::create_dir_all(&disk_path)
-            .await
-            .expect("scanner cycle test disk should be created");
-        let mut endpoint =
-            Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(disk_index);
-        endpoints.push(endpoint);
+    let mut pools = Vec::with_capacity(pool_count);
+    for pool_index in 0..pool_count {
+        let mut endpoints = Vec::new();
+        for disk_index in 0..4 {
+            let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}"));
+            tokio::fs::create_dir_all(&disk_path)
+                .await
+                .expect("scanner cycle test disk should be created");
+            let mut endpoint =
+                Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
+            endpoint.set_pool_index(pool_index);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            endpoints.push(endpoint);
+        }
+        pools.push(PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: if pool_count == 1 {
+                "scanner-cycle-metrics".to_string()
+            } else {
+                format!("scanner-cycle-metrics-pool-{pool_index}")
+            },
+            platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+        });
     }
-    let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: 4,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "scanner-cycle-metrics".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }]);
+    let endpoint_pools = EndpointServerPools::from(pools);
     let instance_ctx = Arc::new(InstanceContext::new());
+    instance_ctx.set_endpoints(endpoint_pools.clone());
     init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
         .await
         .expect("scanner cycle test disks should initialize");
@@ -89,6 +105,27 @@ async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool
     (temp_dir, store)
 }
 
+async fn restart_scanner_cycle_store_from(store: &Arc<ECStore>) -> Arc<ECStore> {
+    let endpoint_pools = store
+        .instance_endpoints()
+        .expect("scanner restart test store should retain its endpoint topology");
+    let instance_ctx = Arc::new(InstanceContext::new());
+    instance_ctx.set_endpoints(endpoint_pools.clone());
+    init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+        .await
+        .expect("scanner restart test disks should reinitialize");
+    let restarted = ECStore::new_with_instance_ctx(
+        "127.0.0.1:0".parse().expect("test address should parse"),
+        endpoint_pools,
+        CancellationToken::new(),
+        instance_ctx,
+    )
+    .await
+    .expect("restarted scanner cycle test ECStore should initialize");
+    init_bucket_metadata_sys_for_scanner_tests(restarted.clone()).await;
+    restarted
+}
+
 fn assert_run_data_scanner_signature<F, Fut>(_run: F)
 where
     F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
@@ -101,6 +138,190 @@ fn run_data_scanner_keeps_its_two_argument_api() {
     assert_run_data_scanner_signature(run_data_scanner);
 }
 
+#[tokio::test]
+async fn restarted_main_loop_completes_durable_pause_backlog_catch_up() {
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    global_metrics().set_cycle(None).await;
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+
+    let paused_at = scanner_pause_backlog_now();
+    let mut seeded = ScannerPauseBacklogController::claim(store.clone(), paused_at)
+        .await
+        .expect("seed writer should claim the durable pause backlog");
+    seeded
+        .observe(ScannerPauseBacklogObservation {
+            now_unix_secs: paused_at.saturating_add(1),
+            paused: true,
+            movement_generation: store.scanner_data_movement_generation().saturating_add(1),
+            movement_work_items: 1,
+            pause_started_at_unix_secs: paused_at.saturating_add(1),
+            dirty_usage_buckets: 0,
+            discovered_expiry_items: 0,
+            discovered_transition_items: 0,
+        })
+        .await;
+    drop(seeded);
+
+    let seeded_status = scanner_pause_backlog_status(store.clone()).await;
+    assert!(seeded_status.durable, "seeded pause backlog must be set-backed");
+    assert_eq!(seeded_status.phase, ScannerPauseBacklogPhase::Paused);
+    assert!(seeded_status.pending_full_scan);
+    assert_eq!(seeded_status.catch_up_attempts, 0);
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    assert!(
+        restarted.instance_endpoints().is_some(),
+        "restarted scanner store must retain instance endpoints"
+    );
+    let restarted_status = scanner_pause_backlog_status(restarted.clone()).await;
+    assert_eq!(restarted_status.phase, ScannerPauseBacklogPhase::Paused);
+    assert_eq!(restarted_status.generation, seeded_status.generation);
+
+    let ctx = CancellationToken::new();
+    let scanner_ctx = ctx.clone();
+    let scanner_store = restarted.clone();
+    let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await });
+
+    let final_status = match tokio::time::timeout(Duration::from_secs(30), async {
+        loop {
+            let status = scanner_pause_backlog_status(restarted.clone()).await;
+            if status.phase == ScannerPauseBacklogPhase::Idle
+                && status.writer_epoch > seeded_status.writer_epoch
+                && status.catch_up_attempts > seeded_status.catch_up_attempts
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    {
+        Ok(status) => status,
+        Err(err) => {
+            ctx.cancel();
+            scanner_task.abort();
+            panic!("restarted scanner did not complete durable catch-up through the main loop: {err}");
+        }
+    };
+
+    ctx.cancel();
+    tokio::time::timeout(Duration::from_secs(5), scanner_task)
+        .await
+        .expect("scanner loop should stop after cancellation")
+        .expect("scanner task should not panic")
+        .expect("scanner loop should exit cleanly");
+
+    assert!(final_status.durable);
+    assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle);
+    assert!(!final_status.pending_full_scan);
+    assert_eq!(final_status.pending_work_items, 0);
+    assert_eq!(final_status.consecutive_failures, 0);
+    assert!(final_status.pause_ended_at_unix_secs >= final_status.pause_started_at_unix_secs);
+
+    let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("the catch-up scanner cycle should leave an authoritative usage snapshot readable");
+    let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("authoritative usage snapshot should decode");
+    assert!(
+        usage.is_complete_bucket_usage_snapshot(),
+        "durable catch-up must run a complete scanner cycle before clearing the backlog"
+    );
+
+    global_metrics().set_cycle(None).await;
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+}
+
+#[tokio::test]
+#[serial_test::serial(scanner_runtime_env)]
+async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
+    temp_env::async_with_vars([(ENV_SCANNER_CYCLE, Some("1")), (ENV_SCANNER_START_DELAY_SECS, Some("0"))], async {
+        crate::runtime_config::refresh_scanner_runtime_config_for_tests();
+        crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+        global_metrics().set_cycle(None).await;
+        let (_temp_dir, store) = setup_scanner_cycle_store_with_pool_count(true, 2).await;
+
+        let ctx = CancellationToken::new();
+        let scanner_ctx = ctx.clone();
+        let scanner_store = store.clone();
+        let startup_probe = ScannerStartupObservedProbe::install();
+        let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await });
+        startup_probe.wait().await;
+        let ready_probe = ScannerRuntimeObservedProbe::install(&store, false);
+        startup_probe.resume();
+        drop(startup_probe);
+        ready_probe.wait().await;
+        drop(ready_probe);
+
+        let paused_probe = ScannerRuntimeObservedProbe::install(&store, true);
+        let paused_at = time::OffsetDateTime::now_utc();
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            pool_meta.pools[0].last_update = paused_at;
+            pool_meta.pools[0].decommission = Some(crate::storage_api::owner::EcstorePoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            });
+        }
+        let pause_status = store.scanner_data_movement_pause_status().await;
+        assert!(pause_status.paused);
+        paused_probe.wait().await;
+        drop(paused_probe);
+
+        let paused_backlog = scanner_pause_backlog_status(store.clone()).await;
+        assert_eq!(paused_backlog.phase, ScannerPauseBacklogPhase::Paused);
+        assert!(paused_backlog.pending_full_scan);
+
+        let resumed_probe = ScannerRuntimeObservedProbe::install(&store, false);
+        store
+            .clear_decommission(0)
+            .await
+            .expect("terminal decommission clear should publish a movement generation");
+        resumed_probe.wait().await;
+        drop(resumed_probe);
+
+        let final_status = match tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let status = scanner_pause_backlog_status(store.clone()).await;
+                if status.phase == ScannerPauseBacklogPhase::Idle
+                    && status.writer_epoch == paused_backlog.writer_epoch
+                    && status.catch_up_attempts > paused_backlog.catch_up_attempts
+                {
+                    break status;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        {
+            Ok(status) => status,
+            Err(err) => {
+                ctx.cancel();
+                scanner_task.abort();
+                panic!("running scanner did not complete durable catch-up after a runtime movement clear: {err}");
+            }
+        };
+
+        ctx.cancel();
+        tokio::time::timeout(Duration::from_secs(5), scanner_task)
+            .await
+            .expect("scanner loop should stop after cancellation")
+            .expect("scanner task should not panic")
+            .expect("scanner loop should exit cleanly");
+
+        assert!(final_status.durable);
+        assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle);
+        assert_eq!(final_status.writer_epoch, paused_backlog.writer_epoch);
+        assert!(!final_status.pending_full_scan);
+        assert_eq!(final_status.pending_work_items, 0);
+        assert_eq!(final_status.consecutive_failures, 0);
+
+        global_metrics().set_cycle(None).await;
+        crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    })
+    .await;
+    crate::runtime_config::refresh_scanner_runtime_config_for_tests();
+}
+
 #[tokio::test]
 async fn scanner_cycle_lock_fence_cancels_cycle_context() {
     let cycle_ctx = CancellationToken::new();