gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-30 08:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence system metadata publication (#6444)

Browse Source 
* feat(scanner): fence usage publication during data movement

* fix(scanner): detect movement refresh state changes

* fix(scanner): fence publication during data movement

* fix(scanner): close movement epoch publication races

* fix(scanner): fence movement-sensitive publication paths

* fix(scanner): fence cache and heal recovery paths

* fix(scanner): carry publication epoch through scan cycle

* fix(scanner): recheck remote cache epoch after save

* fix(scanner): recheck local cache epoch before publish

* fix(scanner): fence data usage writers and baseline

* fix(scanner): expose decommission activity to publication fence

* fix(scanner): release publication gate before reads

* fix(scanner): complete publication fence integration

* fix(scanner): avoid empty usage baseline publication

* chore(scanner): gate test-only helpers

* fix: use decommission canceler in reload test

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-08-23 17:28:21 +08:00
committed by GitHub
parent 9cda615519
commit e196a134cc
26 changed files with 2465 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_io/io_cache.rs
+79 -8
View File
@@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks {
all_buckets,
digest: scan_plan_digest,
leader_epoch,
publication_epoch,
dirty_usage_buckets,
bucket_failures,
pending_maintenance_work,
@@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks {
let set_label = self.set_index.to_string();
let source = DataUsageCacheSource::new(self.pool_index, self.set_index);
let expected_publication_epoch = match publication_epoch {
Some(epoch) => epoch,
None => scanner_publication_epoch(self.clone())
.await
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await {
warn!(
@@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks {
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
}
reset_disk_bucket_scan_gauges(&pool_label, &set_label);
return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref())
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
return persist_and_publish_cache_snapshot(
self,
&updates,
cache,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await
.map(|_| ())
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
@@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks {
let pending_maintenance_work_clone = pending_maintenance_work.clone();
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let expected_publication_epoch_clone = expected_publication_epoch;
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
@@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks {
);
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_before_reuse",
"Current scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
@@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks {
{
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
error!(
@@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks {
false
} else {
match partial_cache
.save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
cache_name.as_str(),
&revisions,
expected_publication_epoch_clone,
)
.await
{
Ok(()) => true,
@@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks {
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache
.save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions)
.save_with_revisions_for_epoch(
store_clone_clone.clone(),
&cache_name,
&revisions,
expected_publication_epoch_clone,
)
.await
{
done_save();
@@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks {
continue;
}
if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone)
.await
.is_none()
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "publication_epoch_changed_after_save",
"Scanner bucket cache root publish skipped after movement epoch change"
);
continue;
}
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
@@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks {
cache.info.lkg_scan_plan_digest = None;
cache.clone()
};
let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await;
let _ = persist_and_publish_cache_snapshot(
self.clone(),
&updates,
cache_snapshot,
cache_cycle_floor.as_ref(),
expected_publication_epoch,
)
.await;
} else {
let mut incomplete_scope = cache_mutex.lock().await.clone();
incomplete_scope.info.name = DATA_USAGE_ROOT.to_string();