fix(scanner): retain raw enumeration quantum

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-08 01:03:38 +08:00
parent 7c85c72fd1
commit 47e2407362
4 changed files with 130 additions and 8 deletions
+31 -8
View File
@@ -846,6 +846,16 @@ impl RawEnumerationProgress {
}
})
}
fn has_checkpointable_page_index(&self) -> bool {
self.page_index().is_some()
}
fn checkpointable_entry_count(&self) -> usize {
self.page_index()
.and_then(|index| index.indexed_entries().ok())
.map_or(0, |entries| entries.len())
}
}
fn update_raw_enumeration_digest(digest: &mut Sha256, label: &[u8], value: &[u8]) {
@@ -1165,20 +1175,33 @@ impl FolderScanner {
}
fn finish_raw_enumeration_parent(&mut self, parent: &str) {
let scan_root = self.old_cache.info.name.as_str();
self.raw_enumeration_progress.retain(|progress| {
progress.parent != parent
&& !progress
.parent
.strip_prefix(parent)
.is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
if progress.parent == parent {
return parent == scan_root && progress.has_checkpointable_page_index();
}
!progress
.parent
.strip_prefix(parent)
.is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
});
}
fn take_raw_enumeration_resume_state(&mut self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
match self.raw_enumeration_progress.drain(..).next() {
Some(progress) => (progress.cursor(), progress.page_index()),
None => (None, None),
if self.raw_enumeration_progress.is_empty() {
return (None, None);
}
let progress_index = self
.raw_enumeration_progress
.iter()
.enumerate()
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
.map(|(index, _)| index)
.unwrap_or(0);
let progress = self.raw_enumeration_progress.swap_remove(progress_index);
self.raw_enumeration_progress.clear();
(progress.cursor(), progress.page_index())
}
fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {
@@ -3512,6 +3512,80 @@ fn raw_enumeration_progress_checkpoint_commits_budgeted_page_for_oracle() {
);
}
#[tokio::test]
async fn raw_enumeration_root_page_survives_child_partial_boundary() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
scanner.old_cache.info.name = "bucket".to_string();
let mut root_progress = RawEnumerationProgress::new("bucket", None);
root_progress.record_entry("object-0000");
root_progress.record_entry("object-0001");
scanner.raw_enumeration_progress.push(root_progress);
scanner.finish_raw_enumeration_parent("bucket");
assert_eq!(scanner.raw_enumeration_progress.len(), 1);
let root_index = scanner.raw_enumeration_progress[0]
.page_index()
.expect("completed scan root should retain its raw-page oracle");
assert_eq!(
root_index
.committed_entries()
.expect("retained root raw-page oracle should validate"),
vec!["object-0000".to_string(), "object-0001".to_string()]
);
let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
child_progress.record_entry("xl.meta");
scanner.raw_enumeration_progress.push(child_progress);
scanner.finish_raw_enumeration_parent("bucket/object-0000");
assert_eq!(
scanner
.raw_enumeration_progress
.iter()
.map(|progress| progress.parent.as_str())
.collect::<Vec<_>>(),
vec!["bucket"]
);
}
#[tokio::test]
async fn raw_enumeration_resume_state_keeps_largest_durable_quantum() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let mut root_progress = RawEnumerationProgress::new("bucket", None);
root_progress.record_entry("object-0000");
scanner.raw_enumeration_progress.push(root_progress);
let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
child_progress.record_entry("part-0000");
child_progress.record_entry("part-0001");
child_progress.record_entry("part-0002");
scanner.raw_enumeration_progress.push(child_progress);
let (cursor, page_index) = scanner.take_raw_enumeration_resume_state();
assert_eq!(
cursor.as_ref().expect("largest raw quantum should include a cursor").parent,
"bucket/object-0000"
);
assert_eq!(
page_index
.as_ref()
.expect("largest raw quantum should include a page index")
.indexed_entries()
.expect("selected page index should validate")
.len(),
3
);
assert!(scanner.raw_enumeration_progress.is_empty());
}
#[test]
fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappear() {
let mut index = RawEnumerationPageIndex::new("bucket", 2).expect("raw page index should initialize");