fix(scanner): persist decommission catch-up debt (#6922)

crates/scanner/Cargo.toml

@@ -106,7 +106,7 @@ hex-simd.workspace = true
 [dev-dependencies]
 tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
 serial_test = { workspace = true }
-temp-env = { workspace = true }
+temp-env = { workspace = true, features = ["async_closure"] }
 tempfile = { workspace = true }
 uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
 tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }

crates/scanner/src/lib.rs

@@ -82,8 +82,10 @@ pub use remote_scanner::{
 pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
 pub use rustfs_scanner_contracts::last_minute;
 pub use scanner::{
-    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
-    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
+    ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
+    ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, init_data_scanner,
+    reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status,
+    scanner_topology_digest,
 };
 pub use scanner_io::{
     ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,

crates/scanner/src/scanner.rs

@@ -90,6 +90,144 @@ const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state
 const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total";
 const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
 const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60);
+
+#[cfg(test)]
+static SCANNER_STARTUP_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerStartupObservedProbeState>>>> =
+    LazyLock::new(|| StdMutex::new(None));
+
+#[cfg(test)]
+struct ScannerStartupObservedProbeState {
+    observed: Notify,
+    resume: Notify,
+}
+
+#[cfg(test)]
+struct ScannerObservedProbeState {
+    store_key: usize,
+    paused: bool,
+    notify: Notify,
+}
+
+#[cfg(test)]
+pub(super) struct ScannerStartupObservedProbe {
+    state: Arc<ScannerStartupObservedProbeState>,
+}
+
+#[cfg(test)]
+static SCANNER_RUNTIME_OBSERVED_PROBE: LazyLock<StdMutex<Option<Arc<ScannerObservedProbeState>>>> =
+    LazyLock::new(|| StdMutex::new(None));
+
+#[cfg(test)]
+pub(super) struct ScannerRuntimeObservedProbe {
+    state: Arc<ScannerObservedProbeState>,
+}
+
+#[cfg(test)]
+impl ScannerStartupObservedProbe {
+    pub(super) fn install() -> Self {
+        let state = Arc::new(ScannerStartupObservedProbeState {
+            observed: Notify::new(),
+            resume: Notify::new(),
+        });
+        let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned");
+        assert!(probe.is_none(), "scanner startup observed probe must be unique");
+        *probe = Some(state.clone());
+        Self { state }
+    }
+
+    pub(super) async fn wait(&self) {
+        tokio::time::timeout(Duration::from_secs(5), self.state.observed.notified())
+            .await
+            .expect("scanner should complete startup pause-backlog observation");
+    }
+
+    pub(super) fn resume(&self) {
+        self.state.resume.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl ScannerRuntimeObservedProbe {
+    pub(super) fn install(storeapi: &Arc<ECStore>, paused: bool) -> Self {
+        let state = Arc::new(ScannerObservedProbeState {
+            store_key: scanner_observed_probe_store_key(storeapi),
+            paused,
+            notify: Notify::new(),
+        });
+        let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
+            .lock()
+            .expect("scanner runtime observed probe should not be poisoned");
+        assert!(probe.is_none(), "scanner runtime observed probe must be unique");
+        *probe = Some(state.clone());
+        Self { state }
+    }
+
+    pub(super) async fn wait(&self) {
+        tokio::time::timeout(Duration::from_secs(10), self.state.notify.notified())
+            .await
+            .expect("scanner should complete runtime pause-backlog observation");
+    }
+}
+
+#[cfg(test)]
+impl Drop for ScannerStartupObservedProbe {
+    fn drop(&mut self) {
+        let mut probe = SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned");
+        if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *probe = None;
+        }
+        self.state.resume.notify_one();
+    }
+}
+
+#[cfg(test)]
+impl Drop for ScannerRuntimeObservedProbe {
+    fn drop(&mut self) {
+        let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE
+            .lock()
+            .expect("scanner runtime observed probe should not be poisoned");
+        if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
+            *probe = None;
+        }
+    }
+}
+
+#[cfg(test)]
+async fn notify_scanner_startup_observed_for_test() {
+    let probe = {
+        SCANNER_STARTUP_OBSERVED_PROBE
+            .lock()
+            .expect("scanner startup observed probe should not be poisoned")
+            .clone()
+    };
+    if let Some(probe) = probe {
+        probe.observed.notify_one();
+        probe.resume.notified().await;
+    }
+}
+
+#[cfg(test)]
+fn scanner_observed_probe_store_key(storeapi: &Arc<ECStore>) -> usize {
+    Arc::as_ptr(storeapi).cast::<()>() as usize
+}
+
+#[cfg(test)]
+fn notify_scanner_runtime_observed_for_test(storeapi: &Arc<ECStore>, observation: ScannerPauseBacklogObservation) {
+    if let Some(probe) = SCANNER_RUNTIME_OBSERVED_PROBE
+        .lock()
+        .expect("scanner runtime observed probe should not be poisoned")
+        .clone()
+        && probe.store_key == scanner_observed_probe_store_key(storeapi)
+        && probe.paused == observation.paused
+    {
+        probe.notify.notify_one();
+    }
+}
+
 const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
 /// First-retry delay after a scanner cycle cannot publish authoritative usage.
 ///
@@ -2190,6 +2328,74 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) ->
     run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await
 }
 
+async fn current_scanner_pause_backlog_observation(storeapi: &Arc<ECStore>) -> ScannerPauseBacklogObservation {
+    let now_unix_secs = scanner_pause_backlog_now();
+    let pause = storeapi.scanner_data_movement_pause_status().await;
+    let metrics = global_metrics().report().await;
+    ScannerPauseBacklogObservation {
+        now_unix_secs,
+        paused: pause.paused,
+        movement_generation: pause.movement_generation,
+        movement_work_items: pause.movement_backlog_work_items,
+        pause_started_at_unix_secs: pause.started_at_unix_secs,
+        dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets,
+        discovered_expiry_items: metrics
+            .lifecycle_expiry
+            .current_queued
+            .saturating_add(metrics.lifecycle_expiry.current_active),
+        discovered_transition_items: metrics
+            .lifecycle_transition
+            .current_queued
+            .saturating_add(metrics.lifecycle_transition.current_active)
+            .saturating_add(metrics.lifecycle_transition.compensation_pending)
+            .saturating_add(metrics.lifecycle_transition.compensation_running),
+    }
+}
+
+async fn wait_for_scanner_data_movement_resume(
+    ctx: &CancellationToken,
+    storeapi: &Arc<ECStore>,
+    guard: &NamespaceLockGuard,
+    pause_backlog: &mut ScannerPauseBacklogController,
+) -> bool {
+    loop {
+        let observation = current_scanner_pause_backlog_observation(storeapi).await;
+        pause_backlog.observe(observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(storeapi, observation);
+        if !observation.paused {
+            return !ctx.is_cancelled() && !guard.is_lock_lost();
+        }
+
+        let movement_changed = storeapi.scanner_data_movement_changed();
+        if storeapi.scanner_data_movement_generation() != observation.movement_generation {
+            continue;
+        }
+        tokio::select! {
+            _ = ctx.cancelled() => return false,
+            _ = guard.lock_lost_notified() => return false,
+            _ = movement_changed.notified() => {},
+            _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
+        }
+    }
+}
+
+async fn finish_scanner_pause_backlog_cycle(
+    pause_backlog: &mut ScannerPauseBacklogController,
+    storeapi: &Arc<ECStore>,
+    attempt: ScannerPauseBacklogAttemptDecision,
+    outcome: ScannerCycleOutcome,
+) {
+    let observation = current_scanner_pause_backlog_observation(storeapi).await;
+    if let ScannerPauseBacklogAttemptDecision::Tracked(serial) = attempt {
+        pause_backlog.finish_attempt(serial, outcome, observation).await;
+    } else {
+        pause_backlog.observe_cycle_outcome(outcome, observation).await;
+    }
+    #[cfg(test)]
+    notify_scanner_runtime_observed_for_test(storeapi, observation);
+}
+
 async fn run_data_scanner_with_maintenance_state(
     ctx: CancellationToken,
     storeapi: Arc<ECStore>,
@@ -2269,6 +2475,28 @@ async fn run_data_scanner_with_maintenance_state(
             return Ok(());
         }
     };
+    let pause_backlog_now = scanner_pause_backlog_now();
+    let mut pause_backlog = match ScannerPauseBacklogController::claim(storeapi.clone(), pause_backlog_now).await {
+        Ok(controller) => controller,
+        Err(err) => {
+            error!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                state = "pause_backlog_claim_failed",
+                error = %err,
+                "Scanner pause backlog persistence is unavailable"
+            );
+            ScannerPauseBacklogController::unavailable(storeapi.clone(), err, pause_backlog_now)
+        }
+    };
+    if !wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
+        global_metrics().set_cycle(None).await;
+        return Ok(());
+    }
+    #[cfg(test)]
+    notify_scanner_startup_observed_for_test().await;
     let single_disk = storeapi.setup_is_erasure_sd().await;
     let erasure = storeapi.setup_is_erasure().await;
     let distributed = storeapi.setup_is_dist_erasure().await;
@@ -2416,6 +2644,19 @@ async fn run_data_scanner_with_maintenance_state(
         return Ok(());
     }
     if !leadership_claimed {
+        let observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, observation);
+        if observation.paused {
+            if wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await {
+                return Err(ScannerError::Other(
+                    "scanner startup was fenced by data movement; retrying from durable state".to_string(),
+                ));
+            }
+            global_metrics().set_cycle(None).await;
+            return Ok(());
+        }
         error!(
             target: "rustfs::scanner",
             event = EVENT_SCANNER_LOCK_STATE,
@@ -2448,7 +2689,13 @@ async fn run_data_scanner_with_maintenance_state(
         return Ok(());
     }
 
-    if !ctx.is_cancelled() {
+    let initial_pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
+    if !ctx.is_cancelled()
+        && matches!(
+            initial_pause_backlog_attempt,
+            ScannerPauseBacklogAttemptDecision::Untracked | ScannerPauseBacklogAttemptDecision::Tracked(_)
+        )
+    {
         // Preserve previous behavior: run one cycle immediately after lock acquisition.
         let dirty_generation_before_cycle = dirty_usage_generation();
         let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending();
@@ -2506,6 +2753,7 @@ async fn run_data_scanner_with_maintenance_state(
                 return Ok(());
             }
         };
+        finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, initial_pause_backlog_attempt, initial_outcome).await;
         superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -2558,6 +2806,10 @@ async fn run_data_scanner_with_maintenance_state(
             break;
         }
 
+        let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(pause_backlog_observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
         let runtime_config = resolve_scanner_runtime_config();
         if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) {
             let current_generation = scanner_maintenance_generation();
@@ -2594,11 +2846,16 @@ async fn run_data_scanner_with_maintenance_state(
             scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
         let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval);
         let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
-        let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
+        let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
         if let Some(retry_interval) = convergence_retry_interval {
             wait_plan.effective_interval = retry_interval;
             wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval);
         }
+        if let Some(pause_backlog_delay) = pause_backlog.scheduling_delay(scanner_pause_backlog_now()) {
+            wait_plan.effective_interval = pause_backlog_delay.max(Duration::from_secs(1));
+            wait_plan.delay = pause_backlog_delay;
+            convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1)));
+        }
         let dirty_generation_before_wait = dirty_usage_generation();
         let dirty_usage_pending_before_wait = dirty_usage_buckets_pending();
         let maintenance_generation_before_wait = scanner_maintenance_generation();
@@ -2723,6 +2980,20 @@ async fn run_data_scanner_with_maintenance_state(
             record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await;
             break;
         }
+        let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await;
+        pause_backlog.observe(pause_backlog_observation).await;
+        #[cfg(test)]
+        notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation);
+        if pause_backlog_observation.paused {
+            continue;
+        }
+        let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await;
+        if matches!(
+            pause_backlog_attempt,
+            ScannerPauseBacklogAttemptDecision::RateLimited | ScannerPauseBacklogAttemptDecision::PersistenceUnavailable
+        ) {
+            continue;
+        }
         let dirty_generation_before_cycle = dirty_usage_generation();
         let cycle_ctx = ctx.child_token();
         let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
@@ -2771,6 +3042,7 @@ async fn run_data_scanner_with_maintenance_state(
                 return Ok(());
             }
         };
+        finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, pause_backlog_attempt, outcome).await;
         superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
         deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
         dirty_usage_generation_seen = dirty_generation_before_cycle;
@@ -3088,12 +3360,14 @@ fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Op
 
 /// Store data usage info in backend. Will store all objects sent on the receiver until closed.
 mod activity;
+mod backlog;
 mod cycle_state;
 mod heal_info;
 mod leadership;
 mod usage_store;
 
 use activity::*;
+use backlog::*;
 use cycle_state::*;
 use leadership::*;
 use usage_store::*;
@@ -3104,6 +3378,10 @@ pub(crate) use activity::{
     scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
 };
 pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
+pub use backlog::{
+    ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds,
+    scanner_pause_backlog_status,
+};
 #[cfg(test)]
 pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
 pub use cycle_state::{

crates/scanner/src/scanner/tests.rs

@@ -39,30 +39,46 @@ async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
 }
 
 async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool) -> (tempfile::TempDir, Arc<ECStore>) {
+    setup_scanner_cycle_store_with_pool_count(seed_usage_baseline, 1).await
+}
+
+async fn setup_scanner_cycle_store_with_pool_count(
+    seed_usage_baseline: bool,
+    pool_count: usize,
+) -> (tempfile::TempDir, Arc<ECStore>) {
     init_ecstore_config_for_scanner_tests();
     let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
-    let mut endpoints = Vec::new();
-    for disk_index in 0..4 {
-        let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
-        tokio::fs::create_dir_all(&disk_path)
-            .await
-            .expect("scanner cycle test disk should be created");
-        let mut endpoint =
-            Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
-        endpoint.set_pool_index(0);
-        endpoint.set_set_index(0);
-        endpoint.set_disk_index(disk_index);
-        endpoints.push(endpoint);
+    let mut pools = Vec::with_capacity(pool_count);
+    for pool_index in 0..pool_count {
+        let mut endpoints = Vec::new();
+        for disk_index in 0..4 {
+            let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}"));
+            tokio::fs::create_dir_all(&disk_path)
+                .await
+                .expect("scanner cycle test disk should be created");
+            let mut endpoint =
+                Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
+            endpoint.set_pool_index(pool_index);
+            endpoint.set_set_index(0);
+            endpoint.set_disk_index(disk_index);
+            endpoints.push(endpoint);
+        }
+        pools.push(PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 4,
+            endpoints: Endpoints::from(endpoints),
+            cmd_line: if pool_count == 1 {
+                "scanner-cycle-metrics".to_string()
+            } else {
+                format!("scanner-cycle-metrics-pool-{pool_index}")
+            },
+            platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
+        });
     }
-    let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
-        legacy: false,
-        set_count: 1,
-        drives_per_set: 4,
-        endpoints: Endpoints::from(endpoints),
-        cmd_line: "scanner-cycle-metrics".to_string(),
-        platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
-    }]);
+    let endpoint_pools = EndpointServerPools::from(pools);
     let instance_ctx = Arc::new(InstanceContext::new());
+    instance_ctx.set_endpoints(endpoint_pools.clone());
     init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
         .await
         .expect("scanner cycle test disks should initialize");
@@ -89,6 +105,27 @@ async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool
     (temp_dir, store)
 }
 
+async fn restart_scanner_cycle_store_from(store: &Arc<ECStore>) -> Arc<ECStore> {
+    let endpoint_pools = store
+        .instance_endpoints()
+        .expect("scanner restart test store should retain its endpoint topology");
+    let instance_ctx = Arc::new(InstanceContext::new());
+    instance_ctx.set_endpoints(endpoint_pools.clone());
+    init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
+        .await
+        .expect("scanner restart test disks should reinitialize");
+    let restarted = ECStore::new_with_instance_ctx(
+        "127.0.0.1:0".parse().expect("test address should parse"),
+        endpoint_pools,
+        CancellationToken::new(),
+        instance_ctx,
+    )
+    .await
+    .expect("restarted scanner cycle test ECStore should initialize");
+    init_bucket_metadata_sys_for_scanner_tests(restarted.clone()).await;
+    restarted
+}
+
 fn assert_run_data_scanner_signature<F, Fut>(_run: F)
 where
     F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
@@ -101,6 +138,190 @@ fn run_data_scanner_keeps_its_two_argument_api() {
     assert_run_data_scanner_signature(run_data_scanner);
 }
 
+#[tokio::test]
+async fn restarted_main_loop_completes_durable_pause_backlog_catch_up() {
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    global_metrics().set_cycle(None).await;
+    let (_temp_dir, store) = setup_scanner_cycle_store().await;
+
+    let paused_at = scanner_pause_backlog_now();
+    let mut seeded = ScannerPauseBacklogController::claim(store.clone(), paused_at)
+        .await
+        .expect("seed writer should claim the durable pause backlog");
+    seeded
+        .observe(ScannerPauseBacklogObservation {
+            now_unix_secs: paused_at.saturating_add(1),
+            paused: true,
+            movement_generation: store.scanner_data_movement_generation().saturating_add(1),
+            movement_work_items: 1,
+            pause_started_at_unix_secs: paused_at.saturating_add(1),
+            dirty_usage_buckets: 0,
+            discovered_expiry_items: 0,
+            discovered_transition_items: 0,
+        })
+        .await;
+    drop(seeded);
+
+    let seeded_status = scanner_pause_backlog_status(store.clone()).await;
+    assert!(seeded_status.durable, "seeded pause backlog must be set-backed");
+    assert_eq!(seeded_status.phase, ScannerPauseBacklogPhase::Paused);
+    assert!(seeded_status.pending_full_scan);
+    assert_eq!(seeded_status.catch_up_attempts, 0);
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    assert!(
+        restarted.instance_endpoints().is_some(),
+        "restarted scanner store must retain instance endpoints"
+    );
+    let restarted_status = scanner_pause_backlog_status(restarted.clone()).await;
+    assert_eq!(restarted_status.phase, ScannerPauseBacklogPhase::Paused);
+    assert_eq!(restarted_status.generation, seeded_status.generation);
+
+    let ctx = CancellationToken::new();
+    let scanner_ctx = ctx.clone();
+    let scanner_store = restarted.clone();
+    let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await });
+
+    let final_status = match tokio::time::timeout(Duration::from_secs(30), async {
+        loop {
+            let status = scanner_pause_backlog_status(restarted.clone()).await;
+            if status.phase == ScannerPauseBacklogPhase::Idle
+                && status.writer_epoch > seeded_status.writer_epoch
+                && status.catch_up_attempts > seeded_status.catch_up_attempts
+            {
+                break status;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    {
+        Ok(status) => status,
+        Err(err) => {
+            ctx.cancel();
+            scanner_task.abort();
+            panic!("restarted scanner did not complete durable catch-up through the main loop: {err}");
+        }
+    };
+
+    ctx.cancel();
+    tokio::time::timeout(Duration::from_secs(5), scanner_task)
+        .await
+        .expect("scanner loop should stop after cancellation")
+        .expect("scanner task should not panic")
+        .expect("scanner loop should exit cleanly");
+
+    assert!(final_status.durable);
+    assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle);
+    assert!(!final_status.pending_full_scan);
+    assert_eq!(final_status.pending_work_items, 0);
+    assert_eq!(final_status.consecutive_failures, 0);
+    assert!(final_status.pause_ended_at_unix_secs >= final_status.pause_started_at_unix_secs);
+
+    let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
+        .await
+        .expect("the catch-up scanner cycle should leave an authoritative usage snapshot readable");
+    let usage = serde_json::from_slice::<DataUsageInfo>(&usage).expect("authoritative usage snapshot should decode");
+    assert!(
+        usage.is_complete_bucket_usage_snapshot(),
+        "durable catch-up must run a complete scanner cycle before clearing the backlog"
+    );
+
+    global_metrics().set_cycle(None).await;
+    crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+}
+
+#[tokio::test]
+#[serial_test::serial(scanner_runtime_env)]
+async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() {
+    temp_env::async_with_vars([(ENV_SCANNER_CYCLE, Some("1")), (ENV_SCANNER_START_DELAY_SECS, Some("0"))], async {
+        crate::runtime_config::refresh_scanner_runtime_config_for_tests();
+        crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+        global_metrics().set_cycle(None).await;
+        let (_temp_dir, store) = setup_scanner_cycle_store_with_pool_count(true, 2).await;
+
+        let ctx = CancellationToken::new();
+        let scanner_ctx = ctx.clone();
+        let scanner_store = store.clone();
+        let startup_probe = ScannerStartupObservedProbe::install();
+        let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await });
+        startup_probe.wait().await;
+        let ready_probe = ScannerRuntimeObservedProbe::install(&store, false);
+        startup_probe.resume();
+        drop(startup_probe);
+        ready_probe.wait().await;
+        drop(ready_probe);
+
+        let paused_probe = ScannerRuntimeObservedProbe::install(&store, true);
+        let paused_at = time::OffsetDateTime::now_utc();
+        {
+            let mut pool_meta = store.pool_meta.write().await;
+            pool_meta.pools[0].last_update = paused_at;
+            pool_meta.pools[0].decommission = Some(crate::storage_api::owner::EcstorePoolDecommissionInfo {
+                failed: true,
+                ..Default::default()
+            });
+        }
+        let pause_status = store.scanner_data_movement_pause_status().await;
+        assert!(pause_status.paused);
+        paused_probe.wait().await;
+        drop(paused_probe);
+
+        let paused_backlog = scanner_pause_backlog_status(store.clone()).await;
+        assert_eq!(paused_backlog.phase, ScannerPauseBacklogPhase::Paused);
+        assert!(paused_backlog.pending_full_scan);
+
+        let resumed_probe = ScannerRuntimeObservedProbe::install(&store, false);
+        store
+            .clear_decommission(0)
+            .await
+            .expect("terminal decommission clear should publish a movement generation");
+        resumed_probe.wait().await;
+        drop(resumed_probe);
+
+        let final_status = match tokio::time::timeout(Duration::from_secs(30), async {
+            loop {
+                let status = scanner_pause_backlog_status(store.clone()).await;
+                if status.phase == ScannerPauseBacklogPhase::Idle
+                    && status.writer_epoch == paused_backlog.writer_epoch
+                    && status.catch_up_attempts > paused_backlog.catch_up_attempts
+                {
+                    break status;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        {
+            Ok(status) => status,
+            Err(err) => {
+                ctx.cancel();
+                scanner_task.abort();
+                panic!("running scanner did not complete durable catch-up after a runtime movement clear: {err}");
+            }
+        };
+
+        ctx.cancel();
+        tokio::time::timeout(Duration::from_secs(5), scanner_task)
+            .await
+            .expect("scanner loop should stop after cancellation")
+            .expect("scanner task should not panic")
+            .expect("scanner loop should exit cleanly");
+
+        assert!(final_status.durable);
+        assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle);
+        assert_eq!(final_status.writer_epoch, paused_backlog.writer_epoch);
+        assert!(!final_status.pending_full_scan);
+        assert_eq!(final_status.pending_work_items, 0);
+        assert_eq!(final_status.consecutive_failures, 0);
+
+        global_metrics().set_cycle(None).await;
+        crate::scanner_io::clear_dirty_usage_buckets_for_tests();
+    })
+    .await;
+    crate::runtime_config::refresh_scanner_runtime_config_for_tests();
+}
+
 #[tokio::test]
 async fn scanner_cycle_lock_fence_cancels_cycle_context() {
     let cycle_ctx = CancellationToken::new();