gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-07 20:46:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): consume raw page owner resume oracle (#7391)

Browse Source 
Treat persisted raw page owner entries as a validated set, not a read_dir-order prefix, so restart scans can consume committed owner pages without recounting them against the raw enumeration budget. Commit checkpointed partial pages, validate owner parent/generation/digests before the skip oracle, and fail closed on duplicate/corrupt page state.

Extend the real scanner restart driver to enforce fixed raw-entry and object budgets across fresh OS processes, and report owner-index coverage in each worker round.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
2026-09-07 19:30:15 +08:00
committed by GitHub
parent fc76099280
commit 0c6314babc
8 changed files with 242 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder.rs
+48 -13
View File
@@ -833,9 +833,17 @@ impl RawEnumerationProgress {
}
fn page_index(&self) -> Option<RawEnumerationPageIndex> {
self.page_index.clone().and_then(|index| match index.indexed_entries() {
Ok(entries) if !entries.is_empty() => Some(index),
_ => None,
self.page_index.clone().and_then(|mut index| {
if let Some(generation) = index.generation()
&& matches!(index.status(), crate::raw_page_index::RawEnumerationPageOwnerStatus::Building { .. })
&& index.commit_building_page(generation).is_err()
{
return None;
}
match index.indexed_entries() {
Ok(entries) if !entries.is_empty() => Some(index),
_ => None,
}
})
}
}
@@ -1133,6 +1141,29 @@ impl FolderScanner {
}
}
fn raw_enumeration_committed_entry_oracle(&self, parent: &str) -> HashSet<String> {
let Some(index) = self.old_cache.validated_raw_enumeration_page_index() else {
return HashSet::new();
};
let generation_matches_parent = match index.status() {
crate::raw_page_index::RawEnumerationPageOwnerStatus::Building {
generation,
parent: index_parent,
..
}
| crate::raw_page_index::RawEnumerationPageOwnerStatus::Ready {
generation,
parent: index_parent,
..
} => generation > 0 && index_parent == parent,
crate::raw_page_index::RawEnumerationPageOwnerStatus::Unsupported => false,
};
if !generation_matches_parent {
return HashSet::new();
}
index.committed_entries().unwrap_or_default().into_iter().collect()
}
fn finish_raw_enumeration_parent(&mut self, parent: &str) {
self.raw_enumeration_progress.retain(|progress| {
progress.parent != parent
@@ -1481,6 +1512,7 @@ impl FolderScanner {
let mut pending_entry_progress = 0_u64;
let mut last_entry_progress = Instant::now();
let mut raw_enumeration_complete = false;
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
loop {
let entry = match dir_reader.next_entry().await {
@@ -1519,20 +1551,23 @@ impl FolderScanner {
}
Err(e) => return Err(ScannerError::Io(e)),
};
#[cfg(test)]
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
{
self.budget.record_entries_visited(pending_entry_progress);
pending_entry_progress = 0;
last_entry_progress = Instant::now();
}
let file_name = entry.file_name().to_string_lossy().to_string();
if file_name.is_empty() || file_name == "." || file_name == ".." {
continue;
}
let raw_entry_consumed_by_owner_index = raw_enumeration_committed_entries.contains(&file_name);
if !raw_entry_consumed_by_owner_index {
#[cfg(test)]
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
pending_entry_progress = pending_entry_progress.saturating_add(1);
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
{
self.budget.record_entries_visited(pending_entry_progress);
pending_entry_progress = 0;
last_entry_progress = Instant::now();
}
}
self.record_raw_enumeration_entry(&folder.name, &file_name);
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;