mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-08 04:58:12 +00:00
fix(scanner): retain raw enumeration quantum
Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
@@ -846,6 +846,16 @@ impl RawEnumerationProgress {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn has_checkpointable_page_index(&self) -> bool {
|
||||
self.page_index().is_some()
|
||||
}
|
||||
|
||||
fn checkpointable_entry_count(&self) -> usize {
|
||||
self.page_index()
|
||||
.and_then(|index| index.indexed_entries().ok())
|
||||
.map_or(0, |entries| entries.len())
|
||||
}
|
||||
}
|
||||
|
||||
fn update_raw_enumeration_digest(digest: &mut Sha256, label: &[u8], value: &[u8]) {
|
||||
@@ -1165,20 +1175,33 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
fn finish_raw_enumeration_parent(&mut self, parent: &str) {
|
||||
let scan_root = self.old_cache.info.name.as_str();
|
||||
self.raw_enumeration_progress.retain(|progress| {
|
||||
progress.parent != parent
|
||||
&& !progress
|
||||
.parent
|
||||
.strip_prefix(parent)
|
||||
.is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
|
||||
if progress.parent == parent {
|
||||
return parent == scan_root && progress.has_checkpointable_page_index();
|
||||
}
|
||||
|
||||
!progress
|
||||
.parent
|
||||
.strip_prefix(parent)
|
||||
.is_some_and(|suffix| suffix.starts_with(SLASH_SEPARATOR))
|
||||
});
|
||||
}
|
||||
|
||||
fn take_raw_enumeration_resume_state(&mut self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
match self.raw_enumeration_progress.drain(..).next() {
|
||||
Some(progress) => (progress.cursor(), progress.page_index()),
|
||||
None => (None, None),
|
||||
if self.raw_enumeration_progress.is_empty() {
|
||||
return (None, None);
|
||||
}
|
||||
let progress_index = self
|
||||
.raw_enumeration_progress
|
||||
.iter()
|
||||
.enumerate()
|
||||
.max_by_key(|(index, progress)| (progress.checkpointable_entry_count(), std::cmp::Reverse(*index)))
|
||||
.map(|(index, _)| index)
|
||||
.unwrap_or(0);
|
||||
let progress = self.raw_enumeration_progress.swap_remove(progress_index);
|
||||
self.raw_enumeration_progress.clear();
|
||||
(progress.cursor(), progress.page_index())
|
||||
}
|
||||
|
||||
fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {
|
||||
|
||||
@@ -3512,6 +3512,80 @@ fn raw_enumeration_progress_checkpoint_commits_budgeted_page_for_oracle() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_enumeration_root_page_survives_child_partial_boundary() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(temp_dir),
|
||||
};
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
|
||||
let mut root_progress = RawEnumerationProgress::new("bucket", None);
|
||||
root_progress.record_entry("object-0000");
|
||||
root_progress.record_entry("object-0001");
|
||||
scanner.raw_enumeration_progress.push(root_progress);
|
||||
scanner.finish_raw_enumeration_parent("bucket");
|
||||
|
||||
assert_eq!(scanner.raw_enumeration_progress.len(), 1);
|
||||
let root_index = scanner.raw_enumeration_progress[0]
|
||||
.page_index()
|
||||
.expect("completed scan root should retain its raw-page oracle");
|
||||
assert_eq!(
|
||||
root_index
|
||||
.committed_entries()
|
||||
.expect("retained root raw-page oracle should validate"),
|
||||
vec!["object-0000".to_string(), "object-0001".to_string()]
|
||||
);
|
||||
|
||||
let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
|
||||
child_progress.record_entry("xl.meta");
|
||||
scanner.raw_enumeration_progress.push(child_progress);
|
||||
scanner.finish_raw_enumeration_parent("bucket/object-0000");
|
||||
|
||||
assert_eq!(
|
||||
scanner
|
||||
.raw_enumeration_progress
|
||||
.iter()
|
||||
.map(|progress| progress.parent.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["bucket"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_enumeration_resume_state_keeps_largest_durable_quantum() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(temp_dir),
|
||||
};
|
||||
|
||||
let mut root_progress = RawEnumerationProgress::new("bucket", None);
|
||||
root_progress.record_entry("object-0000");
|
||||
scanner.raw_enumeration_progress.push(root_progress);
|
||||
|
||||
let mut child_progress = RawEnumerationProgress::new("bucket/object-0000", None);
|
||||
child_progress.record_entry("part-0000");
|
||||
child_progress.record_entry("part-0001");
|
||||
child_progress.record_entry("part-0002");
|
||||
scanner.raw_enumeration_progress.push(child_progress);
|
||||
|
||||
let (cursor, page_index) = scanner.take_raw_enumeration_resume_state();
|
||||
assert_eq!(
|
||||
cursor.as_ref().expect("largest raw quantum should include a cursor").parent,
|
||||
"bucket/object-0000"
|
||||
);
|
||||
assert_eq!(
|
||||
page_index
|
||||
.as_ref()
|
||||
.expect("largest raw quantum should include a page index")
|
||||
.indexed_entries()
|
||||
.expect("selected page index should validate")
|
||||
.len(),
|
||||
3
|
||||
);
|
||||
assert!(scanner.raw_enumeration_progress.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappear() {
|
||||
let mut index = RawEnumerationPageIndex::new("bucket", 2).expect("raw page index should initialize");
|
||||
|
||||
Reference in New Issue
Block a user