gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-08 21:25:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): reuse clean local bucket prefixes (#7208)

Browse Source
This commit is contained in:
Henry Guo
2026-09-07 05:26:56 +08:00
committed by GitHub
parent cf1c45eb91
commit 975983abdd
18 changed files with 577 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder/tests.rs
+110
View File
@@ -332,6 +332,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
heal_object_select: 0,
scan_mode: HealScanMode::Normal,
is_erasure_mode: false,
prefix_scan_scope: None,
failed_object_ttl_secs: u64::MAX,
failed_objects_max: usize::MAX,
sleeper: SCANNER_SLEEPER.clone(),
@@ -2384,6 +2385,115 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let bucket_dir = temp_dir.join("bucket");
tokio::fs::create_dir_all(bucket_dir.join("clean"))
.await
.expect("failed to create clean top-level directory");
tokio::fs::create_dir_all(bucket_dir.join("dirty"))
.await
.expect("failed to create dirty top-level directory");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/clean",
"bucket",
DataUsageEntry {
size: 17,
objects: 3,
..Default::default()
},
);
scanner.old_cache.replace(
"bucket/dirty",
"bucket",
DataUsageEntry {
size: 23,
objects: 4,
..Default::default()
},
);
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
.await
.expect("scoped root scan should finish successfully");
let clean = scanner
.new_cache
.size_recursive("bucket/clean")
.expect("clean entry should be copied from the complete cache");
assert_eq!((clean.size, clean.objects), (17, 3));
let dirty = scanner
.new_cache
.size_recursive("bucket/dirty")
.expect("dirty entry should be rescanned");
assert_eq!((dirty.size, dirty.objects), (0, 0));
let bucket = scanner
.new_cache
.size_recursive("bucket")
.expect("bucket root should include reused and rescanned entries");
assert_eq!((bucket.size, bucket.objects), (17, 3));
}
#[tokio::test]
#[serial]
async fn scoped_root_scan_preserves_erasure_health_walks() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let bucket_dir = temp_dir.join("bucket");
tokio::fs::create_dir_all(bucket_dir.join("clean"))
.await
.expect("failed to create clean top-level directory");
scanner.is_erasure_mode = true;
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.update_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/clean",
"bucket",
DataUsageEntry {
size: 17,
objects: 3,
..Default::default()
},
);
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
.await
.expect("erasure root scan should finish successfully");
let clean = scanner
.new_cache
.size_recursive("bucket/clean")
.expect("erasure scan should visit the clean entry");
assert_eq!((clean.size, clean.objects), (0, 0));
}
#[tokio::test]
#[serial]
async fn test_scan_folder_compacted_parent_sends_partial_update() {
crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs
+29 -9
View File
@@ -14,7 +14,7 @@
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
use crate::scanner_io::{ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk};
use crate::storage_api::scanner_io::ObjectIO;
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
use std::io::Cursor;
@@ -29,6 +29,13 @@ const MAX_CACHE_BYTES: u64 = 1024 * 1024;
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
fn scan_options(scan_mode: HealScanMode) -> ScannerDiskScanOptions {
ScannerDiskScanOptions {
scan_mode,
prefix_scan_scope: None,
}
}
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
#[derive(Debug)]
struct FixtureStore {
@@ -538,7 +545,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
vec![scanner.local_disk.clone()],
loaded,
None,
HealScanMode::Normal,
scan_options(HealScanMode::Normal),
)
.await
.expect("scan must revisit the prefix");
@@ -594,7 +601,7 @@ async fn checkpoint_fixture_failed_child_prevents_receipt_advancing_past_gap() {
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
scan_options(HealScanMode::Normal),
)
.await
.expect("scan with a known failed child");
@@ -645,7 +652,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
scan_options(HealScanMode::Normal),
)
.await
.expect("initial complete baseline");
@@ -690,7 +697,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
scan_options(HealScanMode::Normal),
)
.await
.expect("sampling interruption");
@@ -737,7 +744,14 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
let result = scanner
.local_disk
.clone()
.nsscanner_disk(budget.token(), budget, vec![scanner.local_disk.clone()], loaded, None, resume_mode)
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
loaded,
None,
scan_options(resume_mode),
)
.await
.expect("bounded recovery scan");
let (cache, complete) = match result {
@@ -853,7 +867,10 @@ async fn run_checkpoint_fixture(change_digest: bool) {
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
ScannerDiskScanOptions {
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
},
)
.await
.expect("budgeted local disk scan returns partial cache");
@@ -931,7 +948,10 @@ async fn run_checkpoint_fixture(change_digest: bool) {
vec![scanner.local_disk.clone()],
loaded.clone(),
None,
HealScanMode::Normal,
ScannerDiskScanOptions {
scan_mode: HealScanMode::Normal,
prefix_scan_scope: None,
},
)
.await;
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
@@ -982,7 +1002,7 @@ async fn run_checkpoint_fixture(change_digest: bool) {
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
scan_options(HealScanMode::Normal),
)
.await
.expect("bounded sweep outcome");
crates/scanner/src/scanner_folder/tests/checkpoint_fixture/deep_compacted.rs
+1 -1
View File
@@ -30,7 +30,7 @@ async fn scan(
);
let outcome = disk
.clone()
.nsscanner_disk(budget.token(), budget.clone(), vec![disk.clone()], cache, None, mode)
.nsscanner_disk(budget.token(), budget.clone(), vec![disk.clone()], cache, None, super::scan_options(mode))
.await
.expect("bounded real disk scan");
(outcome, budget)