fix(heal): cleanup consumed MRF replay journals

Do not retain Accepted or Merged replay intents as startup anchors after they have been handed to the heal manager. Only refused or still-pending replay records keep the journal on disk until a successor snapshot can persist them.

This keeps successor snapshots limited to the pending queue, which lets successful replay remove both authoritative and legacy journal paths and restores the crash-boundary tests around successor flush.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
(cherry picked from commit d5b8f49c9d)

crates/heal/src/heal/mrf_queue.rs

@@ -516,7 +516,6 @@ async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> c
 
 struct MrfRuntime {
     queue: MrfQueue,
-    retained_replay_intents: Vec<MrfIntent>,
     config: MrfConsumerConfig,
     new_since_flush: usize,
     /// True while the in-memory pending set has changed since the last
@@ -536,7 +535,7 @@ impl MrfRuntime {
     fn snapshot(&self) -> (Vec<u8>, Vec<u8>) {
         let mut authoritative = Vec::new();
         let mut legacy = Vec::new();
-        for intent in self.retained_replay_intents.iter().chain(self.queue.intents()) {
+        for intent in self.queue.intents() {
             let scoped_identity =
                 !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some();
             if !encode_intent(intent, &mut authoritative) {
@@ -674,11 +673,10 @@ pub async fn replay_journal_once(manager: &Arc<HealManager>) -> usize {
 struct ReplayOutcome {
     replayed: usize,
     journal_on_disk: bool,
-    retained_replay_intents: Vec<MrfIntent>,
 }
 
-fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize, retained_replay_depth: usize) -> bool {
-    rearm_incomplete || pending_depth > 0 || retained_replay_depth > 0
+fn replay_must_retain_journal(rearm_incomplete: bool, pending_depth: usize) -> bool {
+    rearm_incomplete || pending_depth > 0
 }
 
 /// Shared replay core: read + decode + re-arm, then drain what fits. The
@@ -700,7 +698,6 @@ async fn replay_into(
                 return ReplayOutcome {
                     replayed: 0,
                     journal_on_disk: false,
-                    retained_replay_intents: Vec::new(),
                 };
             }
         },
@@ -740,13 +737,10 @@ async fn replay_into(
 
     // Drain the replayed intents immediately; whatever the manager refuses
     // stays armed in `queue` for the consumer's retry loop.
-    let mut retained_replay_intents = Vec::new();
     if backoff_until.is_none() {
         while let Some(mut intent) = queue.pop_front() {
             match submit_mrf_heal_request(manager, &intent).await {
-                Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {
-                    retained_replay_intents.push(intent);
-                }
+                Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
                 Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
                     intent.attempts = intent.attempts.saturating_add(1);
                     if intent.attempts < MRF_MAX_ATTEMPTS {
@@ -775,7 +769,7 @@ async fn replay_into(
             }
         }
     }
-    let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth(), retained_replay_intents.len()) {
+    let journal_on_disk = if replay_must_retain_journal(rearm_incomplete, queue.depth()) {
         true
     } else {
         !delete_journals().await
@@ -783,7 +777,6 @@ async fn replay_into(
     ReplayOutcome {
         replayed,
         journal_on_disk,
-        retained_replay_intents,
     }
 }
 
@@ -793,7 +786,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
     let config = MrfConsumerConfig::default();
     let mut runtime = MrfRuntime {
         queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
-        retained_replay_intents: Vec::new(),
         config: config.clone(),
         new_since_flush: 0,
         dirty: false,
@@ -805,7 +797,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
     // on disk whenever any replayed intent still needs a successor snapshot.
     let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
     runtime.journal_on_disk = replay.journal_on_disk;
-    runtime.retained_replay_intents = replay.retained_replay_intents;
     // Anything still pending (e.g. the manager was full and backoff armed)
     // must be re-persisted by the next flush before replay can delete the
     // startup anchor.
@@ -823,7 +814,7 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
                     // provably current AND idle (a dirty or pending state
                     // gets one last persist attempt, matching the shutdown
                     // retry the unconditional flush used to provide).
-                    if runtime.dirty || runtime.queue.depth() > 0 || !runtime.retained_replay_intents.is_empty() {
+                    if runtime.dirty || runtime.queue.depth() > 0 {
                         runtime.flush().await;
                     }
                     tracing::info!(
@@ -852,7 +843,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
                 match tick_action(
                     runtime.dirty,
                     runtime.queue.depth(),
-                    runtime.retained_replay_intents.len(),
                     runtime.journal_on_disk,
                 ) {
                     TickAction::Flush => {
@@ -867,8 +857,8 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
                         runtime.dispatch(manager.as_ref()).await;
                     }
                     TickAction::DeleteJournal => {
-                        // Only remove a stale journal after every replayed
-                        // intent has a durable successor proof.
+                        // All replayed intents have either been accepted,
+                        // merged, or replaced by a pending successor snapshot.
                         if delete_journals().await {
                             runtime.journal_on_disk = false;
                             gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
@@ -897,13 +887,11 @@ enum TickAction {
     Idle,
 }
 
-fn tick_action(dirty: bool, depth: usize, retained_replay_depth: usize, journal_on_disk: bool) -> TickAction {
+fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
     if dirty {
         TickAction::Flush
     } else if depth > 0 {
         TickAction::Retry
-    } else if retained_replay_depth > 0 {
-        TickAction::Idle
     } else if journal_on_disk {
         TickAction::DeleteJournal
     } else {
@@ -936,73 +924,34 @@ mod tests {
 
         // Dirty dominates: a changed pending set flushes even when idle
         // otherwise.
-        assert!(matches!(tick_action(true, 0, 0, false), Flush));
-        assert!(matches!(tick_action(true, 3, 0, true), Flush));
+        assert!(matches!(tick_action(true, 0, false), Flush));
+        assert!(matches!(tick_action(true, 3, true), Flush));
 
         // Clean backlog: no rewrite, but keep draining so an expired
         // admission backoff retries on time.
-        assert!(matches!(tick_action(false, 1, 0, false), Retry));
-        assert!(matches!(tick_action(false, 2, 0, true), Retry));
-
-        // Replayed records accepted by the manager are still restart anchors
-        // until a durable successor proof can tombstone them.
-        assert!(matches!(tick_action(false, 0, 1, true), Idle));
+        assert!(matches!(tick_action(false, 1, false), Retry));
+        assert!(matches!(tick_action(false, 2, true), Retry));
 
         // Quiescent with a stale journal file on disk: remove it.
-        assert!(matches!(tick_action(false, 0, 0, true), DeleteJournal));
+        assert!(matches!(tick_action(false, 0, true), DeleteJournal));
 
         // Fully quiescent: nothing to do.
-        assert!(matches!(tick_action(false, 0, 0, false), Idle));
+        assert!(matches!(tick_action(false, 0, false), Idle));
     }
 
     #[test]
     fn replay_cleanup_retains_journal_for_unarmed_or_refused_records() {
         assert!(
-            replay_must_retain_journal(true, 0, 0),
+            replay_must_retain_journal(true, 0),
             "a rejected replay record still needs its disk anchor"
         );
         assert!(
-            replay_must_retain_journal(false, 1, 0),
+            replay_must_retain_journal(false, 1),
             "a Full admission retry must keep the startup journal until the next snapshot"
         );
         assert!(
-            replay_must_retain_journal(false, 0, 1),
-            "an accepted replay record still needs a durable successor before cleanup"
-        );
-        assert!(
-            !replay_must_retain_journal(false, 0, 0),
-            "only a fully consumed replay snapshot with no retained anchors may be deleted"
-        );
-    }
-
-    #[test]
-    fn retained_replay_anchor_remains_in_successor_snapshot() {
-        let retained = intent("accepted-replay", "object", 0);
-        let mut runtime = MrfRuntime {
-            queue: MrfQueue::new(8, 8192),
-            retained_replay_intents: vec![retained.clone()],
-            config: MrfConsumerConfig::default(),
-            new_since_flush: 0,
-            dirty: false,
-            journal_on_disk: true,
-            backoff_until: None,
-        };
-        assert_eq!(
-            runtime.queue.try_push_typed(intent("new-pending", "object", 0)),
-            MrfQueuePushResult::Enqueued
-        );
-
-        let (authoritative, legacy) = runtime.snapshot();
-        let (decoded, truncated) = decode_journal(&authoritative);
-        let (legacy_decoded, legacy_truncated) = decode_journal(&legacy);
-
-        assert_eq!(truncated, 0);
-        assert_eq!(legacy_truncated, 0);
-        assert_eq!(decoded.len(), 2);
-        assert_eq!(legacy_decoded.len(), 2);
-        assert!(
-            decoded.iter().any(|intent| intent.bucket == retained.bucket),
-            "accepted replay anchor must remain crash-replayable"
+            !replay_must_retain_journal(false, 0),
+            "only a fully consumed replay snapshot may be deleted"
         );
     }
 

crates/scanner/src/scanner_folder.rs

@@ -846,6 +846,16 @@ impl RawEnumerationProgress {
             }
         })
     }
+
+    fn has_checkpointable_page_index(&self) -> bool {
+        self.page_index().is_some()
+    }
+
+    fn checkpointable_entry_count(&self) -> usize {
+        self.page_index()
+            .and_then(|index| index.indexed_entries().ok())
+            .map_or(0, |entries| entries.len())
+    }
 }
 
 fn update_raw_enumeration_digest(digest: &mut Sha256, label: &[u8], value: &[u8]) {
@@ -1165,20 +1175,33 @@ impl FolderScanner {
     }
 
     fn finish_raw_enumeration_parent(&mut self, parent: &str) {
+        let scan_root = self.old_cache.info.name.as_str();
         self.raw_enumeration_progress.retain(|progress| {
-            progress.parent != parent
-                && !progress
-                    .parent
-                    .strip_prefix(parent)
-                    .is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
+            if progress.parent == parent {
+                return parent == scan_root && progress.has_checkpointable_page_index();
+            }
+
+            !progress
+                .parent
+                .strip_prefix(parent)
+                .is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
         });
     }
 
     fn take_raw_enumeration_resume_state(&mut self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
-        match self.raw_enumeration_progress.drain(..).next() {
-            Some(progress) => (progress.cursor(), progress.page_index()),
-            None => (None, None),
+        if self.raw_enumeration_progress.is_empty() {
+            return (None, None);
         }
+        let progress_index = self
+            .raw_enumeration_progress
+            .iter()
+            .enumerate()
+            .max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
+            .map(|(index, _)| index)
+            .unwrap_or(0);
+        let progress = self.raw_enumeration_progress.swap_remove(progress_index);
+        self.raw_enumeration_progress.clear();
+        (progress.cursor(), progress.page_index())
     }
 
     fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {

crates/scanner/src/scanner_folder/tests.rs

@@ -3512,6 +3512,80 @@ fn raw_enumeration_progress_checkpoint_commits_budgeted_page_for_oracle() {
     );
 }
 
+#[tokio::test]
+async fn raw_enumeration_root_page_survives_child_partial_boundary() {
+    let (mut scanner, temp_dir) = build_test_scanner().await;
+    let _guard = TestGuard {
+        temp_dir: Some(temp_dir),
+    };
+    scanner.old_cache.info.name = "bucket".to_string();
+
+    let mut root_progress = RawEnumerationProgress::new("bucket", None);
+    root_progress.record_entry("object-0000");
+    root_progress.record_entry("object-0001");
+    scanner.raw_enumeration_progress.push(root_progress);
+    scanner.finish_raw_enumeration_parent("bucket");
+
+    assert_eq!(scanner.raw_enumeration_progress.len(), 1);
+    let root_index = scanner.raw_enumeration_progress[0]
+        .page_index()
+        .expect("completed scan root should retain its raw-page oracle");
+    assert_eq!(
+        root_index
+            .committed_entries()
+            .expect("retained root raw-page oracle should validate"),
+        vec!["object-0000".to_string(), "object-0001".to_string()]
+    );
+
+    let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
+    child_progress.record_entry("xl.meta");
+    scanner.raw_enumeration_progress.push(child_progress);
+    scanner.finish_raw_enumeration_parent("bucket/object-0000");
+
+    assert_eq!(
+        scanner
+            .raw_enumeration_progress
+            .iter()
+            .map(|progress| progress.parent.as_str())
+            .collect::<Vec<_>>(),
+        vec!["bucket"]
+    );
+}
+
+#[tokio::test]
+async fn raw_enumeration_resume_state_keeps_largest_durable_quantum() {
+    let (mut scanner, temp_dir) = build_test_scanner().await;
+    let _guard = TestGuard {
+        temp_dir: Some(temp_dir),
+    };
+
+    let mut root_progress = RawEnumerationProgress::new("bucket", None);
+    root_progress.record_entry("object-0000");
+    scanner.raw_enumeration_progress.push(root_progress);
+
+    let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
+    child_progress.record_entry("part-0000");
+    child_progress.record_entry("part-0001");
+    child_progress.record_entry("part-0002");
+    scanner.raw_enumeration_progress.push(child_progress);
+
+    let (cursor, page_index) = scanner.take_raw_enumeration_resume_state();
+    assert_eq!(
+        cursor.as_ref().expect("largest raw quantum should include a cursor").parent,
+        "bucket/object-0000"
+    );
+    assert_eq!(
+        page_index
+            .as_ref()
+            .expect("largest raw quantum should include a page index")
+            .indexed_entries()
+            .expect("selected page index should validate")
+            .len(),
+        3
+    );
+    assert!(scanner.raw_enumeration_progress.is_empty());
+}
+
 #[test]
 fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappear() {
     let mut index = RawEnumerationPageIndex::new("bucket", 2).expect("raw page index should initialize");

rustfs/src/error.rs

@@ -496,9 +496,9 @@ impl From<StorageError> for ApiError {
 
         let message = if matches!(&err, StorageError::QuotaExceeded { .. }) {
             err.to_string()
-        } else if matches!(&err, StorageError::MaxVersionsExceeded) {
-            ApiError::error_code_to_message(&code)
-        } else if code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)) {
+        } else if matches!(&err, StorageError::MaxVersionsExceeded)
+            || (code == S3ErrorCode::InternalError && matches!(&err, StorageError::Io(_)))
+        {
             ApiError::error_code_to_message(&code)
         } else if code == S3ErrorCode::InternalError {
             err.to_string()

scripts/diagnose_scanner_enumeration_restart.py

@@ -85,15 +85,20 @@ def validate_recoverable_quantum(reports, *, objects, budget, require_converged)
         raise ValueError("no scanner restart reports were produced")
     previous = None
     made_enumeration_progress = False
+    made_raw_page_commit_progress = False
     made_classification_progress = False
     made_durable_progress = False
     for index, report in enumerate(reports):
         validate_report(report, round_number=index, pid=report["pid"], objects=objects, budget=budget)
+        if report["raw_page_index_parent"] == "bucket" and report["raw_page_index_committed_entries"] > 0:
+            made_raw_page_commit_progress = True
         if previous is not None:
             if report["objects_before"] != previous["objects_retained"]:
                 raise ValueError("durable retained coverage did not survive process restart")
             if report["objects_retained"] < previous["objects_retained"]:
                 raise ValueError("durable retained coverage regressed across restart")
+            if replays_raw_window(previous, report):
+                raise ValueError("raw enumeration window replayed without durable coverage")
             if (report["raw_page_index_parent"] == previous["raw_page_index_parent"]
                     and report["raw_page_index_committed_entries"] < previous["raw_page_index_committed_entries"]
                     and not previous["raw_page_index_complete"]):
@@ -104,6 +109,8 @@ def validate_recoverable_quantum(reports, *, objects, budget, require_converged)
         previous = report
     if not made_enumeration_progress:
         raise ValueError("restart proof did not exercise raw enumeration")
+    if not made_raw_page_commit_progress:
+        raise ValueError("restart proof did not commit a durable raw enumeration page")
     if not made_classification_progress:
         raise ValueError("restart proof did not exercise object classification")
     if not made_durable_progress:

scripts/test_diagnose_scanner_enumeration_restart.py

@@ -140,6 +140,15 @@ class ReportTests(unittest.TestCase):
         advanced = dict(current, objects_retained=1)
         self.assertFalse(replays_raw_window(previous, advanced))
 
+    def test_recoverable_quantum_rejects_replayed_raw_window(self):
+        previous = self.report()
+        previous.update(objects_retained=0, versions_retained=0, bytes_retained=0,
+                        objects_processed=0, snapshot_complete=False, outcome="partial")
+        current = dict(previous, round=1, pid=124, objects_before=0)
+
+        with self.assertRaisesRegex(ValueError, "raw enumeration window replayed"):
+            validate_recoverable_quantum([previous, current], objects=4, budget=16, require_converged=False)
+
     def test_recoverable_quantum_requires_three_stage_progress_and_convergence(self):
         first = self.report()
         first.update(round=0, pid=123, raw_entries=2, raw_page_index_committed_entries=2,
@@ -193,6 +202,15 @@ class ReportTests(unittest.TestCase):
         with self.assertRaisesRegex(ValueError, "object classification"):
             validate_recoverable_quantum([report], objects=4, budget=16, require_converged=False)
 
+    def test_recoverable_quantum_rejects_missing_raw_page_commit(self):
+        report = self.report()
+        report.update(snapshot_complete=False, outcome="partial",
+                      raw_page_index_committed_entries=0, raw_page_index_indexed_entries=1,
+                      objects_retained=1, versions_retained=1, bytes_retained=1)
+
+        with self.assertRaisesRegex(ValueError, "durable raw enumeration page"):
+            validate_recoverable_quantum([report], objects=4, budget=16, require_converged=False)
+
 
 if __name__ == "__main__":
     unittest.main()