Merge branch 'fix/replication-check-ledger-probe' into fix/replication-orphaned-purge-lifecycle

crates/scanner/src/scanner.rs

@@ -70,12 +70,13 @@ use crate::storage_api::scan::{
     SCANNER_ACTIVITY_PROTOCOL_VERSION,
 };
 use crate::{
-    ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _,
-    ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config,
-    get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config,
-    replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions_and_lease_fence_and_scope,
-    save_config_with_preconditions, save_config_with_publication_admission_for_epoch, scanner_publication_admission_for_epoch,
-    scanner_publication_epoch, scanner_publication_epoch_changed,
+    DiskError, ECStore, EcstoreError, ListPathRawOptions, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED,
+    ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch,
+    get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache,
+    invalidate_data_usage_snapshot_cache, list_path_raw, read_config, replace_bucket_usage_memory_from_info, save_config,
+    save_config_shared_with_preconditions_and_lease_fence_and_scope, save_config_with_preconditions,
+    save_config_with_publication_admission_for_epoch, scanner_publication_admission_for_epoch, scanner_publication_epoch,
+    scanner_publication_epoch_changed,
 };
 
 const LOG_COMPONENT_SCANNER: &str = "scanner";
@@ -947,6 +948,22 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
     init_data_scanner_with_storage(ctx, storeapi).await;
 }
 
+async fn run_scanner_usage_recovery_intents_for_startup(
+    ctx: CancellationToken,
+    storeapi: Arc<ECStore>,
+) -> Result<usize, ScannerError> {
+    let intent_ids = scanner_usage_recovery_intents_for_startup(&ctx, storeapi.clone()).await?;
+    let mut attempted = 0usize;
+    for intent_id in intent_ids {
+        if ctx.is_cancelled() {
+            break;
+        }
+        run_scanner_usage_recovery_intent(ctx.child_token(), storeapi.clone(), intent_id).await?;
+        attempted = attempted.saturating_add(1);
+    }
+    Ok(attempted)
+}
+
 /// Start normal scanning when enabled, or one resume-only cleanup attempt.
 /// The disabled branch returns a finite task for the startup owner to join;
 /// it never enables ordinary namespace scanning or accepts a new reset intent.
@@ -956,10 +973,32 @@ pub async fn init_scanner_with_recovery(
     enabled: bool,
 ) -> Option<tokio::task::JoinHandle<()>> {
     if enabled {
+        if let Err(error) = run_scanner_usage_recovery_intents_for_startup(ctx.clone(), storeapi.clone()).await {
+            warn!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                state = "recovery_intent_startup_discovery_failed",
+                error = %error,
+                "Scanner recovery intent startup discovery failed"
+            );
+        }
         init_data_scanner(ctx, storeapi).await;
         return None;
     }
     Some(tokio::spawn(async move {
+        if let Err(error) = run_scanner_usage_recovery_intents_for_startup(ctx.clone(), storeapi.clone()).await {
+            warn!(
+                target: "rustfs::scanner",
+                event = EVENT_SCANNER_PERSIST_STATE,
+                component = LOG_COMPONENT_SCANNER,
+                subsystem = LOG_SUBSYSTEM_RUNTIME,
+                state = "recovery_intent_startup_discovery_failed",
+                error = %error,
+                "Disabled scanner recovery intent startup discovery failed"
+            );
+        }
         if let Err(error) = resume_scanner_cycle_cleanup(ctx, storeapi).await {
             warn!(
                 target: "rustfs::scanner",
@@ -2216,11 +2255,14 @@ where
         false
     } else if let Some(notification_system) = storeapi.scanner_notification_system() {
         let acknowledgement_count = remote_dirty_usage_acknowledgements.len();
+        let acknowledgement_proof = remote_dirty_usage_acknowledgements.clone();
         let acknowledgements = remote_dirty_usage_acknowledgements.into_iter().map(Into::into).collect();
         remote_dirty_usage_acknowledgement_pending(
             cycle_info.current,
             acknowledgement_count,
+            &acknowledgement_proof,
             notification_system.acknowledge_scanner_dirty_usage(acknowledgements),
+            || probe_scanner_activity(storeapi.as_ref(), true),
         )
         .await
     } else {

crates/scanner/src/scanner/activity.rs

@@ -15,6 +15,7 @@
 use super::*;
 use crate::storage_api::ScannerStorage;
 use crate::storage_api::scan::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
+use std::collections::HashSet;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub(super) enum ScannerCycleWakeReason {
@@ -51,18 +52,25 @@ pub(crate) fn scanner_cycle_outcome_with_pending_maintenance(
     }
 }
 
-pub(super) async fn remote_dirty_usage_acknowledgement_pending<F, E>(
+pub(super) async fn remote_dirty_usage_acknowledgement_pending<F, E, C, CF>(
     cycle: u64,
     acknowledgement_count: usize,
+    acknowledgements: &[ScannerDirtyUsageAcknowledgement],
     acknowledgement: F,
+    confirm_after_error: C,
 ) -> bool
 where
     F: Future<Output = Result<bool, E>>,
     E: std::fmt::Display,
+    C: FnOnce() -> CF,
+    CF: Future<Output = Result<ScannerActivitySnapshot, String>>,
 {
     match acknowledgement.await {
         Ok(dirty_usage_pending) => dirty_usage_pending,
         Err(err) => {
+            if remote_dirty_usage_acknowledgement_loss_reconciled(acknowledgements, confirm_after_error().await) {
+                return false;
+            }
             warn!(
                 target: "rustfs::scanner",
                 event = EVENT_SCANNER_PERSIST_STATE,
@@ -79,6 +87,29 @@ where
     }
 }
 
+pub(super) fn remote_dirty_usage_acknowledgement_loss_reconciled(
+    acknowledgements: &[ScannerDirtyUsageAcknowledgement],
+    activity_after_error: Result<ScannerActivitySnapshot, String>,
+) -> bool {
+    if acknowledgements.is_empty() {
+        return false;
+    }
+    let Ok(activity_after_error) = activity_after_error else {
+        return false;
+    };
+    if !scanner_activity_allows_usage_publication(&activity_after_error) {
+        return false;
+    }
+    let mut acknowledged_hosts = HashSet::with_capacity(acknowledgements.len());
+    acknowledgements.iter().all(|acknowledgement| {
+        if !acknowledged_hosts.insert(acknowledgement.host.as_str()) {
+            return false;
+        }
+        scanner_activity_dirty_usage_state_for_host(&activity_after_error, &acknowledgement.host)
+            .is_some_and(|(instance_id, _generation, pending)| instance_id == acknowledgement.instance_id && !pending)
+    })
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub(super) struct ScannerCleanIdleBackoff {
     pub(super) interval_multiplier: u32,

crates/scanner/src/scanner/cycle_state.rs

@@ -18,6 +18,8 @@ use crate::data_usage_define::{
     DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH, usage_floor_primary_read_error_allows_backup,
 };
 use crate::storage_api::owner::ObjectIO as _;
+use rustfs_filemeta::MetaCacheEntry;
+use std::collections::BTreeSet;
 use std::sync::atomic::AtomicU64;
 use tokio::io::AsyncReadExt as _;
 
@@ -42,6 +44,9 @@ const SCANNER_RECOVERY_INTENT_STATE_RUNNING: &str = "running";
 const SCANNER_RECOVERY_INTENT_STATE_COMPLETED: &str = "completed";
 const SCANNER_RECOVERY_INTENT_STATE_FAILED: &str = "failed";
 pub const SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD: &str = "scanner-usage-full-rebuild";
+const MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES: usize = 4096;
+const SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE: usize = 128;
+const MAX_SCANNER_RECOVERY_INTENT_STARTUP_REPLAY: usize = 64;
 
 #[cfg(test)]
 pub(super) mod cleanup_io_fault {
@@ -465,6 +470,14 @@ fn scanner_recovery_intent_path(intent_id: &str) -> Result<String, ScannerError>
     Ok(format!("{SCANNER_RECOVERY_INTENT_PREFIX}/{intent_id}.json"))
 }
 
+fn scanner_recovery_intent_id_from_entry_name(entry_name: &str) -> Option<String> {
+    let relative = entry_name
+        .strip_prefix(SCANNER_RECOVERY_INTENT_PREFIX)
+        .and_then(|name| name.strip_prefix('/'))?;
+    let intent_id = relative.strip_suffix(".json")?;
+    (!intent_id.contains('/') && is_canonical_sha256(intent_id)).then(|| intent_id.to_string())
+}
+
 pub fn scanner_recovery_actor_sha256(actor: &str) -> String {
     sha256_hex(&[b"scanner-recovery-actor-v1", actor.as_bytes()])
 }
@@ -623,6 +636,117 @@ pub async fn get_scanner_usage_recovery_intent(
     read_recovery_intent_record(storeapi, &path).await
 }
 
+fn scanner_recovery_intent_is_replayable(record: &ScannerRecoveryIntentRecord) -> bool {
+    matches!(
+        record.state.as_str(),
+        SCANNER_RECOVERY_INTENT_STATE_ACCEPTED | SCANNER_RECOVERY_INTENT_STATE_RUNNING
+    )
+}
+
+pub(super) async fn scanner_usage_recovery_intents_for_startup(
+    ctx: &CancellationToken,
+    storeapi: Arc<ECStore>,
+) -> Result<Vec<String>, ScannerError> {
+    let discovered = Arc::new(StdMutex::new(BTreeSet::<String>::new()));
+    for set in storeapi.all_set_disks() {
+        if ctx.is_cancelled() {
+            break;
+        }
+        let disks = set.get_local_disks().await;
+        if disks.is_empty() {
+            continue;
+        }
+        let read_quorum = disks.len().saturating_sub(set.default_parity_count).clamp(1, disks.len());
+        let mut forward_to = None;
+        loop {
+            let page_entries = Arc::new(StdMutex::new(Vec::<String>::new()));
+            let page_entries_for_set = page_entries.clone();
+            let list_result = list_path_raw(
+                ctx.child_token(),
+                ListPathRawOptions {
+                    disks: disks.clone(),
+                    bucket: RUSTFS_META_BUCKET.to_string(),
+                    path: SCANNER_RECOVERY_INTENT_PREFIX.to_string(),
+                    recursive: true,
+                    skip_hidden_prefix_check: true,
+                    forward_to: forward_to.clone(),
+                    min_disks: read_quorum,
+                    report_not_found: true,
+                    per_disk_limit: i32::try_from(SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE).unwrap_or(i32::MAX),
+                    agreed: Some(Box::new(move |entry: MetaCacheEntry| {
+                        let page_entries = page_entries_for_set.clone();
+                        Box::pin(async move {
+                            if scanner_recovery_intent_id_from_entry_name(&entry.name).is_some() {
+                                page_entries
+                                    .lock()
+                                    .unwrap_or_else(|poisoned| poisoned.into_inner())
+                                    .push(entry.name);
+                            }
+                        })
+                    })),
+                    ..Default::default()
+                },
+            )
+            .await;
+            match list_result {
+                Ok(()) => {}
+                Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => break,
+                Err(err) => {
+                    return Err(ScannerError::Other(format!("failed to list scanner recovery intents for startup: {err}")));
+                }
+            }
+
+            let page = {
+                let mut guard = page_entries.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+                guard.sort();
+                guard.dedup();
+                std::mem::take(&mut *guard)
+            };
+            if page.is_empty() {
+                break;
+            }
+
+            {
+                let mut all = discovered.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+                for entry_name in &page {
+                    if let Some(intent_id) = scanner_recovery_intent_id_from_entry_name(entry_name) {
+                        all.insert(intent_id);
+                    }
+                    if all.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES {
+                        break;
+                    }
+                }
+                if all.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_CANDIDATES {
+                    break;
+                }
+            }
+
+            if page.len() < SCANNER_RECOVERY_INTENT_STARTUP_PAGE_SIZE {
+                break;
+            }
+            forward_to = page.last().cloned();
+        }
+    }
+
+    let intent_ids = {
+        let guard = discovered.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+        guard.iter().cloned().collect::<Vec<_>>()
+    };
+    let mut replayable = Vec::new();
+    for intent_id in intent_ids {
+        let Some(record) = get_scanner_usage_recovery_intent(storeapi.clone(), &intent_id).await? else {
+            continue;
+        };
+        if scanner_recovery_intent_is_replayable(&record) {
+            replayable.push(record.intent_id);
+            if replayable.len() >= MAX_SCANNER_RECOVERY_INTENT_STARTUP_REPLAY {
+                break;
+            }
+        }
+    }
+    Ok(replayable)
+}
+
 pub async fn accept_scanner_usage_recovery_intent(
     storeapi: Arc<impl ScannerObjectIO>,
     request: ScannerRecoveryIntentRequest,

crates/scanner/src/scanner/tests.rs

@@ -7498,13 +7498,28 @@ fn finalizing_post_scan_observation_advances_partially_without_dirty_ack() {
 
 #[tokio::test]
 async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
-    let pending = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(true))).await;
+    let acknowledgements = Vec::new();
+    let pending = remote_dirty_usage_acknowledgement_pending(
+        7,
+        1,
+        &acknowledgements,
+        std::future::ready(Ok::<bool, std::io::Error>(true)),
+        || async { Ok(BTreeMap::new()) },
+    )
+    .await;
     assert_eq!(
         scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending),
         ScannerCycleOutcome::CompletedWithPendingMaintenance
     );
 
-    let cleared = remote_dirty_usage_acknowledgement_pending(7, 1, std::future::ready(Ok::<bool, std::io::Error>(false))).await;
+    let cleared = remote_dirty_usage_acknowledgement_pending(
+        7,
+        1,
+        &acknowledgements,
+        std::future::ready(Ok::<bool, std::io::Error>(false)),
+        || async { Ok(BTreeMap::new()) },
+    )
+    .await;
     assert_eq!(
         scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, cleared),
         ScannerCycleOutcome::Completed
@@ -7513,7 +7528,9 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
     let failed = remote_dirty_usage_acknowledgement_pending(
         7,
         1,
+        &acknowledgements,
         std::future::ready(Err::<bool, _>(std::io::Error::other("injected acknowledgement failure"))),
+        || async { Err("confirmation probe failed".to_string()) },
     )
     .await;
     assert_eq!(
@@ -7522,6 +7539,76 @@ async fn scanner_cycle_keeps_remote_pending_acknowledgement() {
     );
 }
 
+#[tokio::test]
+async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() {
+    let acknowledgement = ScannerDirtyUsageAcknowledgement {
+        host: "node-2".to_string(),
+        instance_id: "epoch-a".to_string(),
+        kind: ScannerDirtyUsageAcknowledgementKind::Generation(5),
+    };
+    let cleared_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
+    let response_lost = remote_dirty_usage_acknowledgement_pending(
+        8,
+        1,
+        std::slice::from_ref(&acknowledgement),
+        std::future::ready(Err::<bool, _>(std::io::Error::other("response lost after peer ack"))),
+        || async { Ok(cleared_activity) },
+    )
+    .await;
+    assert_eq!(
+        scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, response_lost),
+        ScannerCycleOutcome::Completed,
+        "a same-instance activity confirmation with no dirty work closes the uncertain ACK"
+    );
+
+    let duplicate_acknowledgements = vec![acknowledgement.clone(), acknowledgement.clone()];
+    let duplicate_target = remote_dirty_usage_acknowledgement_pending(
+        8,
+        duplicate_acknowledgements.len(),
+        &duplicate_acknowledgements,
+        std::future::ready(Err::<bool, _>(std::io::Error::other("duplicate target rejected before peer ack"))),
+        || async { Ok(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))])) },
+    )
+    .await;
+    assert_eq!(
+        scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, duplicate_target),
+        ScannerCycleOutcome::CompletedWithPendingMaintenance,
+        "a request rejected before peer delivery cannot be recovered by a clean activity snapshot"
+    );
+
+    let restarted_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 7, 3))]);
+    let peer_restarted = remote_dirty_usage_acknowledgement_pending(
+        8,
+        1,
+        std::slice::from_ref(&acknowledgement),
+        std::future::ready(Err::<bool, _>(std::io::Error::other("response lost before restart was observed"))),
+        || async { Ok(restarted_activity) },
+    )
+    .await;
+    assert_eq!(
+        scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, peer_restarted),
+        ScannerCycleOutcome::CompletedWithPendingMaintenance,
+        "a new peer instance cannot confirm whether the old ACK reached durable dirty state"
+    );
+
+    let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
+    written_activity.dirty_usage_generation = 6;
+    written_activity.dirty_usage_pending = true;
+    let concurrent_write = remote_dirty_usage_acknowledgement_pending(
+        8,
+        1,
+        &[acknowledgement],
+        std::future::ready(Err::<bool, _>(std::io::Error::other("response lost before concurrent write"))),
+        || async { Ok(BTreeMap::from([("node-2".to_string(), written_activity)])) },
+    )
+    .await;
+    assert_eq!(
+        scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, concurrent_write),
+        ScannerCycleOutcome::CompletedWithPendingMaintenance,
+        "new dirty usage on the same peer must keep maintenance pending"
+    );
+}
+
 #[test]
 #[serial]
 fn finalizing_an_already_durable_enum_without_proof_keeps_dirty_pending() {

crates/scanner/src/scanner/tests/recovery_control.rs

@@ -106,6 +106,20 @@ fn recovery_intent_request(key: &str, actor: &str) -> ScannerRecoveryIntentReque
     }
 }
 
+fn synthetic_recovery_intent_record(intent_id: String, state: &str) -> ScannerRecoveryIntentRecord {
+    ScannerRecoveryIntentRecord {
+        schema_version: 1,
+        intent_id,
+        action: SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD.to_string(),
+        mode: "full-rebuild".to_string(),
+        state: state.to_string(),
+        actor_sha256: "a".repeat(64),
+        idempotency_key_sha256: "b".repeat(64),
+        request_sha256: "c".repeat(64),
+        accepted_at_unix_secs: 1,
+    }
+}
+
 async fn assert_reset_fences(store: &Arc<ECStore>) {
     let data = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
         .await
@@ -311,6 +325,162 @@ async fn scanner_recovery_intent_executor_persists_failed_progress() {
     assert_eq!(failed.intent_id, record.intent_id);
 }
 
+#[tokio::test]
+#[serial]
+async fn scanner_recovery_intent_startup_discovers_only_non_terminal_records() {
+    let (_dir, store) = setup_scanner_cycle_store().await;
+    let accepted = match accept_scanner_usage_recovery_intent(
+        store.clone(),
+        recovery_intent_request("intent-key-0001-startup-accepted", "operator-a"),
+    )
+    .await
+    .expect("accepted startup intent should persist")
+    {
+        ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
+        other => panic!("first request must create accepted startup intent: {other:?}"),
+    };
+    let mut running = match accept_scanner_usage_recovery_intent(
+        store.clone(),
+        recovery_intent_request("intent-key-0001-startup-running", "operator-a"),
+    )
+    .await
+    .expect("running startup intent should persist")
+    {
+        ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
+        other => panic!("first request must create running startup intent: {other:?}"),
+    };
+    running.state = "running".to_string();
+    let running_path = format!(".usage.v2.recovery-intents/{}.json", running.intent_id);
+    save_config(
+        store.clone(),
+        &running_path,
+        serde_json::to_vec(&running).expect("running intent should encode"),
+    )
+    .await
+    .expect("running intent override should persist");
+
+    for (key, state) in [
+        ("intent-key-0001-startup-completed", "completed"),
+        ("intent-key-0001-startup-failed", "failed"),
+    ] {
+        let mut terminal = match accept_scanner_usage_recovery_intent(store.clone(), recovery_intent_request(key, "operator-a"))
+            .await
+            .expect("terminal startup intent should persist")
+        {
+            ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
+            other => panic!("first request must create terminal startup intent: {other:?}"),
+        };
+        terminal.state = state.to_string();
+        let path = format!(".usage.v2.recovery-intents/{}.json", terminal.intent_id);
+        save_config(
+            store.clone(),
+            &path,
+            serde_json::to_vec(&terminal).expect("terminal intent should encode"),
+        )
+        .await
+        .expect("terminal intent override should persist");
+    }
+    save_config(store.clone(), ".usage.v2.recovery-intents/not-a-sha.json", b"{}".to_vec())
+        .await
+        .expect("foreign startup key should persist");
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    let replayable = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
+        .await
+        .expect("startup discovery should tolerate terminal and foreign records");
+    let replayable = replayable.into_iter().collect::<std::collections::BTreeSet<_>>();
+    assert_eq!(
+        replayable,
+        [accepted.intent_id, running.intent_id].into_iter().collect(),
+        "startup discovery must only re-drive accepted/running durable intents"
+    );
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_recovery_intent_startup_pages_past_terminal_records() {
+    let (_dir, store) = setup_scanner_cycle_store().await;
+    for index in 0..70 {
+        let intent_id = format!("{index:064x}");
+        let path = format!(".usage.v2.recovery-intents/{intent_id}.json");
+        let terminal = synthetic_recovery_intent_record(intent_id, "completed");
+        save_config(
+            store.clone(),
+            &path,
+            serde_json::to_vec(&terminal).expect("terminal paging fixture should encode"),
+        )
+        .await
+        .expect("terminal paging fixture should persist");
+    }
+    let pending_id = "f".repeat(64);
+    let pending = synthetic_recovery_intent_record(pending_id.clone(), "accepted");
+    save_config(
+        store.clone(),
+        &format!(".usage.v2.recovery-intents/{pending_id}.json"),
+        serde_json::to_vec(&pending).expect("pending paging fixture should encode"),
+    )
+    .await
+    .expect("pending paging fixture should persist");
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    let replayable = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
+        .await
+        .expect("startup discovery should page past terminal records");
+    assert_eq!(replayable, vec![pending_id]);
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_recovery_intent_startup_rejects_corrupt_pending_record() {
+    let (_dir, store) = setup_scanner_cycle_store().await;
+    let record = match accept_scanner_usage_recovery_intent(
+        store.clone(),
+        recovery_intent_request("intent-key-0001-startup-corrupt", "operator-a"),
+    )
+    .await
+    .expect("corrupt startup fixture intent should persist")
+    {
+        ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
+        other => panic!("first request must create corrupt startup fixture: {other:?}"),
+    };
+    let path = format!(".usage.v2.recovery-intents/{}.json", record.intent_id);
+    save_config(store.clone(), &path, b"{corrupt".to_vec())
+        .await
+        .expect("corrupt intent payload should persist");
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    let error = scanner_usage_recovery_intents_for_startup(&CancellationToken::new(), restarted)
+        .await
+        .expect_err("startup discovery must not silently drop corrupt pending intent records");
+    assert!(error.to_string().contains("scanner recovery intent is invalid"), "{error}");
+}
+
+#[tokio::test]
+#[serial]
+async fn scanner_recovery_intent_disabled_startup_executes_persisted_non_terminal_intent() {
+    let (_dir, store) = setup_scanner_cycle_store().await;
+    let record = match accept_scanner_usage_recovery_intent(
+        store.clone(),
+        recovery_intent_request("intent-key-0001-startup-exec", "operator-a"),
+    )
+    .await
+    .expect("startup execution intent should persist")
+    {
+        ScannerRecoveryIntentAcceptResult::Accepted { record } => record,
+        other => panic!("first request must create startup execution intent: {other:?}"),
+    };
+
+    let restarted = restart_scanner_cycle_store_from(&store).await;
+    run_disabled_startup(CancellationToken::new(), restarted.clone()).await;
+
+    let completed = get_scanner_usage_recovery_intent(restarted, &record.intent_id)
+        .await
+        .expect("startup-executed intent should read")
+        .expect("startup-executed intent should remain durable");
+    assert_eq!(completed.state, "completed");
+    assert_eq!(completed.intent_id, record.intent_id);
+}
+
 #[tokio::test]
 #[serial]
 async fn scanner_recovery_intent_rejects_same_namespace_conflict() {

crates/scanner/src/scanner_folder.rs

@@ -21,8 +21,8 @@ use std::time::{Duration, Instant, SystemTime};
 use crate::ReplTargetSizeSummary;
 use crate::data_usage_define::{
     DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
-    DataUsageScanCheckpoint, DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt,
-    SizeReconciliationEntry, SizeSummary, hash_path,
+    DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, PendingScannerHeal,
+    PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
 };
 use crate::error::ScannerError;
 use crate::runtime_config::{
@@ -55,6 +55,7 @@ use rustfs_scanner_metrics::metrics::{
     UpdateCurrentPathFn, current_path_updater, global_metrics,
 };
 use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
+use sha2::{Digest, Sha256};
 use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc;
@@ -733,6 +734,7 @@ pub struct FolderScanner {
     coverage_frontier: Option<String>,
     resume_frontier: Option<String>,
     coverage_gap: bool,
+    raw_enumeration_progress: Vec<RawEnumerationProgress>,
     pending_heal_sync_deferred: bool,
     pending_heal_batch_dirty: bool,
     #[cfg(test)]
@@ -744,6 +746,50 @@ pub struct FolderScanner {
     list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
 }
 
+struct RawEnumerationProgress {
+    parent: String,
+    last_entry: Option<String>,
+    entries_seen: u64,
+    digest: Sha256,
+}
+
+impl RawEnumerationProgress {
+    fn new(parent: &str) -> Self {
+        let mut digest = Sha256::new();
+        update_raw_enumeration_digest(&mut digest, b"parent", parent.as_bytes());
+        Self {
+            parent: parent.to_string(),
+            last_entry: None,
+            entries_seen: 0,
+            digest,
+        }
+    }
+
+    fn record_entry(&mut self, entry: &str) {
+        update_raw_enumeration_digest(&mut self.digest, b"entry", entry.as_bytes());
+        self.last_entry = Some(entry.to_string());
+        self.entries_seen = self.entries_seen.saturating_add(1);
+    }
+
+    fn into_cursor(self) -> Option<DataUsageRawEnumerationCursor> {
+        if self.entries_seen == 0 {
+            return None;
+        }
+        Some(DataUsageRawEnumerationCursor::new(
+            self.parent,
+            self.last_entry,
+            self.entries_seen,
+            self.digest.finalize().into(),
+        ))
+    }
+}
+
+fn update_raw_enumeration_digest(digest: &mut Sha256, label: &[u8], value: &[u8]) {
+    digest.update(label);
+    digest.update(u64::try_from(value.len()).unwrap_or(u64::MAX).to_le_bytes());
+    digest.update(value);
+}
+
 fn size_reconciliation_entry_bytes(entry: &SizeReconciliationEntry) -> usize {
     entry.key.len()
         + entry.bucket.len()
@@ -999,6 +1045,41 @@ impl FolderScanner {
         }
     }
 
+    fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) {
+        if self.old_cache.info.scan_progress.is_none() {
+            return;
+        }
+        if let Some(position) = self
+            .raw_enumeration_progress
+            .iter()
+            .position(|progress| progress.parent == parent)
+        {
+            self.raw_enumeration_progress.truncate(position + 1);
+        } else {
+            self.raw_enumeration_progress.push(RawEnumerationProgress::new(parent));
+        }
+        if let Some(progress) = self.raw_enumeration_progress.last_mut() {
+            progress.record_entry(entry);
+        }
+    }
+
+    fn finish_raw_enumeration_parent(&mut self, parent: &str) {
+        self.raw_enumeration_progress.retain(|progress| {
+            progress.parent != parent
+                && !progress
+                    .parent
+                    .strip_prefix(parent)
+                    .is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
+        });
+    }
+
+    fn take_raw_enumeration_cursor(&mut self) -> Option<DataUsageRawEnumerationCursor> {
+        self.raw_enumeration_progress
+            .drain(..)
+            .next()
+            .and_then(RawEnumerationProgress::into_cursor)
+    }
+
     fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {
         if entry.compacted {
             // Compacted entries store child totals directly; child links would be flattened twice.
@@ -1329,11 +1410,15 @@ impl FolderScanner {
             };
             let mut pending_entry_progress = 0_u64;
             let mut last_entry_progress = Instant::now();
+            let mut raw_enumeration_complete = false;
 
             loop {
                 let entry = match dir_reader.next_entry().await {
                     Ok(Some(entry)) => entry,
-                    Ok(None) => break,
+                    Ok(None) => {
+                        raw_enumeration_complete = true;
+                        break;
+                    }
                     Err(e) if e.kind() == ErrorKind::NotFound => {
                         debug!(
                             target: "rustfs::scanner::folder",
@@ -1345,6 +1430,7 @@ impl FolderScanner {
                             error = %e,
                             "Scanner folder state updated"
                         );
+                        raw_enumeration_complete = true;
                         break;
                     }
                     Err(e) if e.kind() == ErrorKind::NotADirectory => {
@@ -1358,6 +1444,7 @@ impl FolderScanner {
                             error = %e,
                             "Scanner folder state updated"
                         );
+                        raw_enumeration_complete = true;
                         break;
                     }
                     Err(e) => return Err(ScannerError::Io(e)),
@@ -1376,6 +1463,7 @@ impl FolderScanner {
                 if file_name.is_empty() || file_name == "." || file_name == ".." {
                     continue;
                 }
+                self.record_raw_enumeration_entry(&folder.name, &file_name);
                 let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
 
                 let file_path = entry.path().to_string_lossy().to_string();
@@ -1686,6 +1774,9 @@ impl FolderScanner {
                 }
             }
             self.budget.record_entries_visited(pending_entry_progress);
+            if raw_enumeration_complete {
+                self.finish_raw_enumeration_parent(&folder.name);
+            }
 
             let mut found_erasure_data_directory = false;
             if self.is_erasure_mode && !found_object_metadata {
@@ -2533,6 +2624,7 @@ pub(crate) async fn scan_data_folder_scoped(
         coverage_gap: false,
         pending_heal_sync_deferred: false,
         pending_heal_batch_dirty: false,
+        raw_enumeration_progress: Vec::new(),
         #[cfg(test)]
         pending_heal_sync_count: 0,
         pending_size_reconciliation_keys: HashSet::new(),
@@ -2593,6 +2685,7 @@ pub(crate) async fn scan_data_folder_scoped(
             let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some();
             new_cache.info.scan_resume_after = None;
             new_cache.info.scan_checkpoint = None;
+            new_cache.info.scan_raw_enumeration_cursor = None;
             new_cache.info.scan_coverage_receipt = None;
             if had_scan_checkpoint {
                 global_metrics().record_scanner_checkpoint_cleared();
@@ -2610,6 +2703,9 @@ pub(crate) async fn scan_data_folder_scoped(
                 let root_hash = hash_path(&cache.info.name);
                 let root_has_progress = data_usage_root_has_progress(&root);
                 let pending_heals_changed = scanner.pending_heals_changed;
+                let raw_enumeration_cursor = scanner.take_raw_enumeration_cursor();
+                let carry_forward_cache =
+                    (raw_enumeration_cursor.is_some() && !root_has_progress).then(|| scanner.old_cache.cache.clone());
                 if root_has_progress {
                     scanner.carry_forward_old_children(&root_hash, &mut root);
                 }
@@ -2617,8 +2713,19 @@ pub(crate) async fn scan_data_folder_scoped(
                 let new_cache = scanner.as_mut_new_cache();
                 if root_has_progress {
                     new_cache.replace_hashed(&root_hash, &None, &root);
+                } else if let Some(cache) = carry_forward_cache {
+                    new_cache.cache = cache;
                 }
-                if partial_cache_is_useful(&root, pending_heals_changed) || !new_cache.info.size_reconciliation.is_empty() {
+                if raw_enumeration_cursor.is_some() {
+                    new_cache.info.scan_raw_enumeration_cursor = raw_enumeration_cursor;
+                    new_cache.info.scan_checkpoint = None;
+                    new_cache.info.scan_resume_after = None;
+                    new_cache.info.scan_coverage_receipt = None;
+                }
+                if partial_cache_is_useful(&root, pending_heals_changed)
+                    || new_cache.info.scan_raw_enumeration_cursor.is_some()
+                    || !new_cache.info.size_reconciliation.is_empty()
+                {
                     if new_cache.root().is_some() {
                         new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
                     }

crates/scanner/src/scanner_folder/tests.rs

@@ -353,6 +353,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
         coverage_frontier: None,
         resume_frontier: None,
         coverage_gap: false,
+        raw_enumeration_progress: Vec::new(),
         pending_heal_sync_deferred: false,
         pending_heal_batch_dirty: false,
         pending_heal_sync_count: 0,
@@ -2637,6 +2638,93 @@ async fn test_scan_data_folder_returns_partial_cache_on_budget_cancel() {
     assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
 }
 
+#[tokio::test]
+#[serial]
+async fn test_scan_data_folder_returns_raw_cursor_on_enumeration_cancel_without_root_progress() {
+    let (scanner, temp_dir) = build_test_scanner().await;
+    let _guard = TestGuard {
+        temp_dir: Some(temp_dir.clone()),
+    };
+
+    let bucket_dir = temp_dir.join("bucket");
+    tokio::fs::create_dir_all(&bucket_dir)
+        .await
+        .expect("failed to create bucket directory");
+    for entry in ["entry-a", "entry-b", "entry-c"] {
+        tokio::fs::write(bucket_dir.join(entry), b"data")
+            .await
+            .expect("failed to create raw directory entry");
+    }
+
+    let plan = crate::data_usage_define::DataUsageScanPlanDigest([11; 32]);
+    let source = crate::data_usage_define::DataUsageCacheSource::new(1, 0);
+    let identity = crate::data_usage_define::DataUsageScanIdentity {
+        version: 1,
+        bucket_incarnation: Uuid::from_u128(7),
+        set_layout: crate::data_usage_define::DataUsageScanPlanDigest([12; 32]),
+        publication_epoch: 3,
+        tier_registry_generation: 0,
+        scan_mode: HealScanMode::Normal,
+    };
+    let mut cache = DataUsageCache {
+        info: crate::data_usage_define::DataUsageCacheInfo {
+            name: "bucket".to_string(),
+            next_cycle: 7,
+            ..Default::default()
+        },
+        ..Default::default()
+    };
+    assert_eq!(
+        cache.prepare_bucket_checkpoint("bucket", 7, 3, source, plan, identity),
+        crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
+    );
+
+    let parent = CancellationToken::new();
+    let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
+    let _raw_entry_budget = enumeration_restart::install_raw_entry_budget(scanner.local_disk.path(), 1);
+
+    let result = scan_data_folder(
+        budget.token(),
+        budget.clone(),
+        vec![scanner.local_disk.clone()],
+        scanner.local_disk.clone(),
+        cache,
+        None,
+        HealScanMode::Normal,
+        SCANNER_SLEEPER.clone(),
+    )
+    .await;
+
+    let partial_cache = match result {
+        Err(ScannerError::PartialCache(partial_cache)) => partial_cache,
+        other => panic!("expected raw enumeration partial cache after cancellation, got {other:?}"),
+    };
+
+    assert!(
+        partial_cache
+            .root()
+            .is_none_or(|root| root.objects == 0 && root.versions == 0 && root.size == 0),
+        "raw cursor writer must not invent object progress"
+    );
+    assert!(partial_cache.info.last_update.is_some());
+    assert_eq!(partial_cache.info.next_cycle, 7);
+    assert!(!partial_cache.info.snapshot_complete);
+    assert!(partial_cache.info.scan_checkpoint.is_none());
+    assert!(partial_cache.info.scan_resume_after.is_none());
+
+    let raw_cursor = partial_cache
+        .info
+        .scan_raw_enumeration_cursor
+        .as_ref()
+        .expect("raw enumeration cancellation should persist a cursor");
+    assert_eq!(raw_cursor.parent, "bucket");
+    assert_eq!(raw_cursor.entries_seen, 1);
+    assert!(raw_cursor.last_entry.is_some());
+    assert_ne!(raw_cursor.page_digest, [0; 32]);
+    assert_eq!(partial_cache.validated_raw_enumeration_cursor(), Some(raw_cursor));
+    assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Runtime));
+}
+
 #[tokio::test]
 #[serial]
 async fn test_scan_data_folder_reports_invalid_checkpoint_ignored_once() {

crates/scanner/src/scanner_folder/tests/enumeration_restart.rs

@@ -40,7 +40,7 @@ pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::O
     }
 }
 
-struct ObservationGuard;
+pub(in crate::scanner_folder) struct ObservationGuard;
 
 impl Drop for ObservationGuard {
     fn drop(&mut self) {
@@ -48,6 +48,18 @@ impl Drop for ObservationGuard {
     }
 }
 
+pub(in crate::scanner_folder) fn install_raw_entry_budget(root: PathBuf, limit: u64) -> ObservationGuard {
+    *OBSERVATION.lock().expect("install raw-entry observation") = Some(Observation {
+        root,
+        limit,
+        entries: 0,
+        name_bytes: 0,
+        first_entry: None,
+        last_entry: None,
+    });
+    ObservationGuard
+}
+
 #[derive(serde::Deserialize)]
 #[serde(deny_unknown_fields)]
 struct Request {
@@ -84,8 +96,21 @@ async fn round(request: &Request) -> serde_json::Value {
         }
         let mut initial = DataUsageCache::default();
         initial.info.name = "bucket".to_string();
+        let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
+        let plan = crate::data_usage_define::DataUsageScanPlanDigest([31; 32]);
+        let identity = crate::data_usage_define::DataUsageScanIdentity {
+            version: 1,
+            bucket_incarnation: Uuid::from_u128(31),
+            set_layout: crate::data_usage_define::DataUsageScanPlanDigest([32; 32]),
+            publication_epoch: 1,
+            tier_registry_generation: 0,
+            scan_mode: HealScanMode::Normal,
+        };
+        assert_eq!(
+            initial.prepare_bucket_checkpoint("bucket", 1, 0, source, plan, identity),
+            crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
+        );
         initial.info.skip_healing = true;
-        initial.info.snapshot_complete = false;
         initial.replace("bucket", "", DataUsageEntry::default());
         tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
             .await
@@ -106,15 +131,7 @@ async fn round(request: &Request) -> serde_json::Value {
     .expect("open synthetic disk in this process");
     let parent = CancellationToken::new();
     let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
-    *OBSERVATION.lock().expect("install observation") = Some(Observation {
-        root: disk.path(),
-        limit: request.raw_entry_budget,
-        entries: 0,
-        name_bytes: 0,
-        first_entry: None,
-        last_entry: None,
-    });
-    let _observation_guard = ObservationGuard;
+    let _observation_guard = install_raw_entry_budget(disk.path(), request.raw_entry_budget);
     let result = scan_data_folder(
         budget.token(),
         budget.clone(),